From bda50712a191b50064f6d65fe4ca1ae7f087a729 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 29 Jul 2026 18:37:48 +0300 Subject: [PATCH 1/4] chore(project): limit backtrace size (#670) * chore(project): limit backtrace size * Bump version up to 1.5.7 * Update project.js --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- package.json | 2 +- src/resolvers/project.js | 44 ++++++----- src/utils/eventPayloadLimits.js | 79 +++++++++++++++++++ .../project-daily-events-portion.test.ts | 67 ++++++++++++++++ test/utils/eventPayloadLimits.test.ts | 40 ++++++++++ 5 files changed, 212 insertions(+), 20 deletions(-) create mode 100644 src/utils/eventPayloadLimits.js create mode 100644 test/utils/eventPayloadLimits.test.ts diff --git a/package.json b/package.json index 1e9ebca4..6582959a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.5.6", + "version": "1.5.7", "main": "index.ts", "license": "BUSL-1.1", "scripts": { diff --git a/src/resolvers/project.js b/src/resolvers/project.js index 7e4bf7fe..029d30f4 100644 --- a/src/resolvers/project.js +++ b/src/resolvers/project.js @@ -23,16 +23,18 @@ const GROUPING_TIMESTAMP_AND_GROUP_HASH_INDEX_NAME = 'groupingTimestampAndGroupH const DAILY_EVENTS_GROUP_HASH_INDEX_NAME = 'groupHash'; const MAX_SEARCH_QUERY_LENGTH = 50; const FALLBACK_EVENT_TITLE = 'Unknown'; +const { limitBacktraceForDailyEventsList } = require('../utils/eventPayloadLimits'); /** - * Ensures each daily event has non-empty payload title - * and writes warning log with identifiers when fallback is used. + * Temporary list-response sanitizer: + * - fallback for empty payload.title + * - cap backtrace frames/sourceCode size (heavy Rails stacks) * * @param {object} dailyEventsPortion - portion returned by events factory * @param {string|ObjectId} projectId - project id for logs * @returns {object} */ -function normalizeDailyEventsPayloadTitle(dailyEventsPortion, projectId) { +function sanitizeDailyEventsPortion(dailyEventsPortion, projectId) { if (!dailyEventsPortion || !Array.isArray(dailyEventsPortion.dailyEvents)) { return dailyEventsPortion; } @@ -40,29 +42,35 @@ function normalizeDailyEventsPayloadTitle(dailyEventsPortion, projectId) { dailyEventsPortion.dailyEvents = dailyEventsPortion.dailyEvents.map((dailyEvent) => { const event = dailyEvent && dailyEvent.event ? dailyEvent.event : null; const payload = event && event.payload ? event.payload : null; - const hasValidTitle = payload && - typeof payload.title === 'string' && - payload.title.trim().length > 0; + const rawTitle = payload && typeof payload.title === 'string' ? payload.title : ''; + const hasValidTitle = rawTitle.trim().length > 0; + const title = hasValidTitle ? rawTitle : FALLBACK_EVENT_TITLE; + const backtrace = limitBacktraceForDailyEventsList(payload && payload.backtrace); + const titleChanged = !payload || payload.title !== title; + const backtraceChanged = !payload || payload.backtrace !== backtrace; + + if (!hasValidTitle) { + console.warn('πŸ”΄ [ProjectResolver.dailyEventsPortion] Missing event payload title. Fallback title applied.', { + projectId: projectId ? projectId.toString() : null, + dailyEventId: dailyEvent && dailyEvent.id ? dailyEvent.id.toString() : null, + dailyEventGroupHash: dailyEvent && dailyEvent.groupHash ? dailyEvent.groupHash.toString() : null, + eventOriginalId: event && event.originalEventId ? event.originalEventId.toString() : null, + eventId: event && event._id ? event._id.toString() : null, + }); + } - if (hasValidTitle) { + if (!titleChanged && !backtraceChanged) { return dailyEvent; } - console.warn('πŸ”΄πŸ”΄πŸ”΄ [ProjectResolver.dailyEventsPortion] Missing event payload title. Fallback title applied.', { - projectId: projectId ? projectId.toString() : null, - dailyEventId: dailyEvent && dailyEvent.id ? dailyEvent.id.toString() : null, - dailyEventGroupHash: dailyEvent && dailyEvent.groupHash ? dailyEvent.groupHash.toString() : null, - eventOriginalId: event && event.originalEventId ? event.originalEventId.toString() : null, - eventId: event && event._id ? event._id.toString() : null, - }); - return { ...dailyEvent, event: { ...(event || {}), payload: { ...(payload || {}), - title: FALLBACK_EVENT_TITLE, + title, + backtrace, }, }, }; @@ -667,9 +675,7 @@ module.exports = { assignee ); - normalizeDailyEventsPayloadTitle(dailyEventsPortion, project._id); - - return dailyEventsPortion; + return sanitizeDailyEventsPortion(dailyEventsPortion, project._id); }, /** diff --git a/src/utils/eventPayloadLimits.js b/src/utils/eventPayloadLimits.js new file mode 100644 index 00000000..817f075b --- /dev/null +++ b/src/utils/eventPayloadLimits.js @@ -0,0 +1,79 @@ +/** + * Temporary limits for heavy event payloads in list responses. + * Deep Rails backtraces with sourceCode were producing multi‑MB ProjectDailyEvents + * responses and 502s behind the API gateway. + */ + +const MAX_DAILY_EVENTS_BACKTRACE_FRAMES = + Number(process.env.MAX_DAILY_EVENTS_BACKTRACE_FRAMES) || 20; + +const MAX_DAILY_EVENTS_SOURCE_CODE_LINES = + Number(process.env.MAX_DAILY_EVENTS_SOURCE_CODE_LINES) || 21; + +const MAX_DAILY_EVENTS_CODE_LINE_LENGTH = + Number(process.env.MAX_DAILY_EVENTS_CODE_LINE_LENGTH) || 140; + +/** + * Trim a string to max length and append ellipsis when truncated. + * + * @param {unknown} content - source line content + * @param {number} maxLength - max characters to keep + * @returns {unknown} + */ +function trimCodeLine(content, maxLength = MAX_DAILY_EVENTS_CODE_LINE_LENGTH) { + if (typeof content !== 'string') { + return content; + } + + if (content.length <= maxLength) { + return content; + } + + return `${content.slice(0, maxLength)}…`; +} + +/** + * Cap backtrace frames and sourceCode size for list payloads. + * Keeps sourceCode for UI, but limits frames/lines so list responses stay small. + * + * @param {unknown} backtrace - event payload backtrace + * @returns {unknown} + */ +function limitBacktraceForDailyEventsList(backtrace) { + if (!Array.isArray(backtrace)) { + return backtrace; + } + + return backtrace.slice(0, MAX_DAILY_EVENTS_BACKTRACE_FRAMES).map((frame) => { + if (!frame || typeof frame !== 'object') { + return frame; + } + + if (!Array.isArray(frame.sourceCode)) { + return frame; + } + + return { + ...frame, + sourceCode: frame.sourceCode + .slice(0, MAX_DAILY_EVENTS_SOURCE_CODE_LINES) + .map((line) => { + if (!line || typeof line !== 'object') { + return line; + } + + return { + ...line, + content: trimCodeLine(line.content), + }; + }), + }; + }); +} + +module.exports = { + MAX_DAILY_EVENTS_BACKTRACE_FRAMES, + MAX_DAILY_EVENTS_SOURCE_CODE_LINES, + MAX_DAILY_EVENTS_CODE_LINE_LENGTH, + limitBacktraceForDailyEventsList, +}; diff --git a/test/resolvers/project-daily-events-portion.test.ts b/test/resolvers/project-daily-events-portion.test.ts index ba9d61c9..e399f001 100644 --- a/test/resolvers/project-daily-events-portion.test.ts +++ b/test/resolvers/project-daily-events-portion.test.ts @@ -220,4 +220,71 @@ describe('Project resolver dailyEventsPortion', () => { warnSpy.mockRestore(); }); + + it('should cap backtrace frames and sourceCode size in list response', async () => { + const longLine = 'x'.repeat(200); + const frames = Array.from({ length: 80 }, (_, index) => { + return { + file: `frame-${index}.rb`, + line: index + 1, + sourceCode: Array.from({ length: 30 }, (__, lineIndex) => { + return { + line: lineIndex + 1, + content: longLine, + }; + }), + }; + }); + const findDailyEventsPortion = jest.fn().mockResolvedValue({ + nextCursor: null, + dailyEvents: [ + { + id: 'daily-1', + groupHash: 'group-1', + event: { + _id: 'repetition-1', + originalEventId: 'event-1', + payload: { + title: 'PG::UniqueViolation', + backtrace: frames, + }, + }, + }, + ], + }); + (getEventsFactory as unknown as jest.Mock).mockReturnValue({ + findDailyEventsPortion, + }); + + const project = { _id: 'project-1' }; + const args = { + limit: 10, + nextCursor: null, + sort: 'BY_DATE', + filters: {}, + search: '', + }; + + const result = await projectResolver.Project.dailyEventsPortion(project, args, {}) as { + dailyEvents: Array<{ + event: { + payload: { + title: string; + backtrace: Array<{ + file: string; + sourceCode: Array<{ content: string }>; + }>; + }; + }; + }>; + }; + + const backtrace = result.dailyEvents[0].event.payload.backtrace; + + expect(backtrace).toHaveLength(20); + expect(backtrace[0].file).toBe('frame-0.rb'); + expect(backtrace[0].sourceCode).toHaveLength(21); + expect(backtrace[0].sourceCode[0].content.endsWith('…')).toBe(true); + expect(backtrace[19].file).toBe('frame-19.rb'); + }); }); diff --git a/test/utils/eventPayloadLimits.test.ts b/test/utils/eventPayloadLimits.test.ts new file mode 100644 index 00000000..95aa8b7a --- /dev/null +++ b/test/utils/eventPayloadLimits.test.ts @@ -0,0 +1,40 @@ +import { + MAX_DAILY_EVENTS_BACKTRACE_FRAMES, + MAX_DAILY_EVENTS_CODE_LINE_LENGTH, + MAX_DAILY_EVENTS_SOURCE_CODE_LINES, + limitBacktraceForDailyEventsList, +} from '../../src/utils/eventPayloadLimits'; + +describe('eventPayloadLimits', () => { + it('should return non-array backtrace as is', () => { + expect(limitBacktraceForDailyEventsList(null)).toBeNull(); + expect(limitBacktraceForDailyEventsList(undefined)).toBeUndefined(); + }); + + it('should cap frames and sourceCode size while keeping sourceCode', () => { + const longLine = 'x'.repeat(MAX_DAILY_EVENTS_CODE_LINE_LENGTH + 40); + const backtrace = Array.from({ length: MAX_DAILY_EVENTS_BACKTRACE_FRAMES + 10 }, (_, index) => { + return { + file: `file-${index}.rb`, + line: index, + sourceCode: Array.from({ length: MAX_DAILY_EVENTS_SOURCE_CODE_LINES + 5 }, (__, lineIndex) => { + return { + line: lineIndex, + content: longLine, + }; + }), + }; + }); + + const limited = limitBacktraceForDailyEventsList(backtrace) as Array<{ + file: string; + sourceCode: Array<{ content: string }>; + }>; + + expect(limited).toHaveLength(MAX_DAILY_EVENTS_BACKTRACE_FRAMES); + expect(limited[0].file).toBe('file-0.rb'); + expect(limited[0].sourceCode).toHaveLength(MAX_DAILY_EVENTS_SOURCE_CODE_LINES); + expect(limited[0].sourceCode[0].content.endsWith('…')).toBe(true); + expect(limited[0].sourceCode[0].content.length).toBe(MAX_DAILY_EVENTS_CODE_LINE_LENGTH + 1); + }); +}); From ec65469330895e70201a1ba362a9a4985272a2f8 Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 29 Jul 2026 18:59:06 +0300 Subject: [PATCH 2/4] chore(ci): skip tests on merge to main and prod (#672) * chore(ci): skip tests on merge to main and prod * Bump version up to 1.5.8 --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- .github/workflows/integration-tests.yml | 6 +++++- .github/workflows/tests.yml | 6 +++++- package.json | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e58e0441..1896cc0e 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,7 +1,11 @@ name: Run integration tests on push on: - - push + push: + branches-ignore: + - main + - master + - prod jobs: tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 211fe006..e4bbb5b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,11 @@ name: Run Jest tests on push on: - - push + push: + branches-ignore: + - main + - master + - prod jobs: build: diff --git a/package.json b/package.json index 6582959a..9b7c317a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.5.7", + "version": "1.5.8", "main": "index.ts", "license": "BUSL-1.1", "scripts": { From 4b11afa407b1610716f05aa49d564540aeeb915d Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Aug 2026 22:39:27 +0300 Subject: [PATCH 3/4] chore(metrics): notification about slow graphql operations (#676) * chore(metrics): notification about slow graphql operations * Bump version up to 1.5.7 * upd corner cases * Bump version up to 1.5.9 * Update graphqlRequestDetails.ts --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- package.json | 2 +- src/metrics/graphql.ts | 15 ++ src/metrics/graphqlRequestDetails.ts | 158 +++++++++++++++++++++ src/metrics/mongodb.ts | 40 +++++- src/metrics/slowOperationAlert.ts | 69 +++++++++ test/metrics/graphqlRequestDetails.test.ts | 79 +++++++++++ test/metrics/slowOperationAlert.test.ts | 54 +++++++ 7 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 src/metrics/graphqlRequestDetails.ts create mode 100644 src/metrics/slowOperationAlert.ts create mode 100644 test/metrics/graphqlRequestDetails.test.ts create mode 100644 test/metrics/slowOperationAlert.test.ts diff --git a/package.json b/package.json index 9b7c317a..c2a99968 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.5.8", + "version": "1.5.9", "main": "index.ts", "license": "BUSL-1.1", "scripts": { diff --git a/src/metrics/graphql.ts b/src/metrics/graphql.ts index 046d903a..3cec26cc 100644 --- a/src/metrics/graphql.ts +++ b/src/metrics/graphql.ts @@ -2,6 +2,8 @@ import client from 'prom-client'; import { ApolloServerPlugin, GraphQLRequestContext, GraphQLRequestListener } from 'apollo-server-plugin-base'; import { GraphQLError } from 'graphql'; import HawkCatcher from '@hawk.so/nodejs'; +import { notifySlowOperation } from './slowOperationAlert'; +import { buildGraphqlRequestContext, formatGraphqlErrorsForAlert } from './graphqlRequestDetails'; /** * GraphQL operation duration histogram * Tracks GraphQL operation duration by operation name and type @@ -93,6 +95,19 @@ export const graphqlMetricsPlugin: ApolloServerPlugin = { }, }); + notifySlowOperation( + `Slow GraphQL operation: ${operationType} ${operationName}`, + durationMs, + { + operationType, + operationName, + ...buildGraphqlRequestContext(ctx), + ...(hasErrors && { + errors: formatGraphqlErrorsForAlert(ctx.errors!), + }), + } + ); + // Track errors if any if (hasErrors) { ctx.errors!.forEach((error: GraphQLError) => { diff --git a/src/metrics/graphqlRequestDetails.ts b/src/metrics/graphqlRequestDetails.ts new file mode 100644 index 00000000..aacd09b8 --- /dev/null +++ b/src/metrics/graphqlRequestDetails.ts @@ -0,0 +1,158 @@ +import { GraphQLRequestContext } from 'apollo-server-plugin-base'; +import { GraphQLError } from 'graphql'; +import { ResolverContextBase } from '../types/graphql'; +import { truncateText } from './slowOperationAlert'; + +const MAX_ALERT_ERRORS = 10; +const MAX_ALERT_ERRORS_LENGTH = 1200; + +const SENSITIVE_VARIABLE_KEYS = new Set([ + 'password', + 'token', + 'accesstoken', + 'refreshtoken', + 'secret', + 'authorization', +]); + +const HIGHLIGHTED_VARIABLE_KEYS = new Set([ + 'projectid', + 'workspaceid', + 'eventid', + 'originaleventid', + 'release', + 'search', + 'assignee', + 'cursor', +]); + +/** + * Redact sensitive GraphQL variables before sending alerts. + * + * @param value - variable value + * @param key - variable key + * @returns sanitized value + */ +function sanitizeVariableValue(value: unknown, key: string): unknown { + if (SENSITIVE_VARIABLE_KEYS.has(key.toLowerCase())) { + return '[redacted]'; + } + + if (Array.isArray(value)) { + return value.map((item, index) => sanitizeVariableValue(item, `${key}[${index}]`)); + } + + if (value && typeof value === 'object') { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + return sanitizeVariables(value as Record); + } + + return value; +} + +/** + * Redact sensitive GraphQL variables before sending alerts. + * + * @param variables - GraphQL request variables + * @returns sanitized variables + */ +function sanitizeVariables( + variables: Record | null | undefined +): Record { + /** + * Null / non-object values are treated as empty β€” many clients send + * `variables: null` for operations without variables, and arrays are not a + * valid GraphQL variables map. + */ + if (variables == null || typeof variables !== 'object' || Array.isArray(variables)) { + return {}; + } + + return Object.fromEntries( + Object.entries(variables).map(([key, value]) => [key, sanitizeVariableValue(value, key)]) + ); +} + +/** + * Extract useful identifiers from nested GraphQL variables. + * + * @param value - variable value + * @param prefix - nested path prefix + * @param result - accumulator for extracted ids + * @returns extracted identifiers + */ +function collectHighlightedIds( + value: unknown, + prefix = '', + result: Record = {} +): Record { + if (!value || typeof value !== 'object') { + return result; + } + + for (const [key, nestedValue] of Object.entries(value as Record)) { + const path = prefix ? `${prefix}.${key}` : key; + + if ( + HIGHLIGHTED_VARIABLE_KEYS.has(key.toLowerCase()) && + (typeof nestedValue === 'string' || typeof nestedValue === 'number' || typeof nestedValue === 'boolean') + ) { + result[path] = nestedValue; + continue; + } + + if (nestedValue && typeof nestedValue === 'object' && !Array.isArray(nestedValue)) { + collectHighlightedIds(nestedValue, path, result); + } + } + + return result; +} + +/** + * Flatten GraphQL errors into a capped string for Hawk alert context. + * sanitizeContext() only truncates top-level strings, not values nested in arrays. + * Reserves space for an omitted-count suffix and truncateText()'s ellipsis. + * + * @param errors - GraphQL errors from the request + * @returns flattened and truncated error messages + */ +export function formatGraphqlErrorsForAlert(errors: readonly GraphQLError[]): string { + const messages = errors.slice(0, MAX_ALERT_ERRORS).map((error) => error.message); + const omittedCount = errors.length - messages.length; + const omittedSuffix = omittedCount > 0 ? `; …(+${omittedCount} more)` : ''; + const maxMessagesLength = Math.max(0, MAX_ALERT_ERRORS_LENGTH - omittedSuffix.length - 1); + + return `${truncateText(messages.join('; '), maxMessagesLength)}${omittedSuffix}`; +} + +/** + * Build request context for slow GraphQL operation alerts. + * + * @param ctx - GraphQL request context + * @returns alert context + */ +export function buildGraphqlRequestContext(ctx: GraphQLRequestContext): Record { + const context = ctx.context as ResolverContextBase | undefined; + const variables = sanitizeVariables( + ctx.request.variables as Record | null | undefined + ); + const highlightedIds = collectHighlightedIds(variables); + const alertContext: Record = {}; + + if (context?.user?.id) { + alertContext.userId = context.user.id; + } + + if (Object.keys(highlightedIds).length > 0) { + alertContext.ids = highlightedIds; + } + + const variablesJson = JSON.stringify(variables); + + if (variablesJson && variablesJson !== '{}') { + alertContext.variables = truncateText(variablesJson, 1200); + } + + return alertContext; +} diff --git a/src/metrics/mongodb.ts b/src/metrics/mongodb.ts index a5c1f51d..0ffed4fa 100644 --- a/src/metrics/mongodb.ts +++ b/src/metrics/mongodb.ts @@ -2,6 +2,7 @@ import promClient from 'prom-client'; import { MongoClient, MongoClientOptions } from 'mongodb'; import { Effect, sgr } from '../utils/ansi'; import HawkCatcher from '@hawk.so/nodejs'; +import { notifySlowOperation, truncateText } from './slowOperationAlert'; /** * MongoDB command duration histogram @@ -156,6 +157,7 @@ function colorizeDuration(duration: number): string { */ interface StoredCommandInfo { formattedCommand: string; + plainFormattedCommand: string; timestamp: number; } @@ -202,7 +204,8 @@ setInterval(cleanupStaleCommandInfo, COMMAND_INFO_TIMEOUT_MS); */ function storeCommandInfo(event: any): void { const collectionRaw = extractCollectionFromCommand(event.command, event.commandName); - const collection = sgr(normalizeCollectionName(collectionRaw), Effect.ForegroundGreen); + const collectionName = normalizeCollectionName(collectionRaw); + const collection = sgr(collectionName, Effect.ForegroundGreen); const db = event.databaseName || 'unknown db'; const commandName = sgr(event.commandName, Effect.ForegroundRed); const filter = event.command.filter; @@ -212,11 +215,12 @@ function storeCommandInfo(event: any): void { const params = filter || update || pipeline; const paramsStr = formatParams(params); const projectionStr = projection ? ` projection: ${formatParams(projection)}` : ''; - + const plainFormattedCommand = `[${event.requestId}] ${db}.${collectionName}.${event.commandName}(${paramsStr})${projectionStr}`; const formattedCommand = `[${event.requestId}] ${db}.${collection}.${commandName}(${paramsStr})${projectionStr}`; commandInfoMap.set(event.requestId, { formattedCommand, + plainFormattedCommand, timestamp: Date.now(), }); } @@ -232,9 +236,24 @@ function logCommandSucceeded(event: any): void { if (info) { console.log(`${info.formattedCommand} βœ“ ${durationStr}`); + notifySlowOperation( + `Slow MongoDB command: ${event.commandName}`, + event.duration, + { + requestId: event.requestId, + command: truncateText(info.plainFormattedCommand), + } + ); commandInfoMap.delete(event.requestId); } else { console.log(`[${event.requestId}] ${event.commandName} βœ“ ${durationStr}`); + notifySlowOperation( + `Slow MongoDB command: ${event.commandName}`, + event.duration, + { + requestId: event.requestId, + } + ); } } @@ -250,9 +269,26 @@ function logCommandFailed(event: any): void { if (info) { console.error(`${info.formattedCommand} βœ— ${errorMsg} ${durationStr}`); + notifySlowOperation( + `Slow MongoDB command: ${event.commandName}`, + event.duration, + { + requestId: event.requestId, + command: truncateText(info.plainFormattedCommand), + error: truncateText(errorMsg, 500), + } + ); commandInfoMap.delete(event.requestId); } else { console.error(`[${event.requestId}] ${event.commandName} βœ— ${errorMsg} ${durationStr}`); + notifySlowOperation( + `Slow MongoDB command: ${event.commandName}`, + event.duration, + { + requestId: event.requestId, + error: truncateText(errorMsg, 500), + } + ); } } diff --git a/src/metrics/slowOperationAlert.ts b/src/metrics/slowOperationAlert.ts new file mode 100644 index 00000000..c4ed7ae8 --- /dev/null +++ b/src/metrics/slowOperationAlert.ts @@ -0,0 +1,69 @@ +import HawkCatcher from '@hawk.so/nodejs'; + +export const SLOW_OPERATION_THRESHOLD_MS = 10000; +const MAX_CONTEXT_STRING_LENGTH = 2500; + +/** + * Truncate text for slow operation context fields. + * + * @param value - text to truncate + * @param maxLength - max allowed length + * @returns truncated text + */ +function truncateText(value: string, maxLength = MAX_CONTEXT_STRING_LENGTH): string { + if (value.length <= maxLength) { + return value; + } + + return `${value.slice(0, maxLength)}…`; +} + +/** + * Truncate long string values in alert context. + * + * @param context - alert context + * @returns sanitized context + */ +function sanitizeContext(context: Record): Record { + return Object.fromEntries( + Object.entries(context).map(([key, value]) => { + if (typeof value === 'string') { + return [key, truncateText(value)]; + } + + return [key, value]; + }) + ); +} + +/** + * Send slow operation alert to Hawk via HawkCatcher. + * + * @param message - short alert message + * @param durationMs - operation duration in milliseconds + * @param context - additional alert context + */ +export function notifySlowOperation( + message: string, + durationMs: number, + context: Record = {} +): void { + if ( + process.env.NODE_ENV === 'test' || + process.env.NODE_ENV === 'e2e' || + durationMs < SLOW_OPERATION_THRESHOLD_MS + ) { + return; + } + + try { + HawkCatcher.send(new Error(message), { + durationMs, + ...sanitizeContext(context), + }); + } catch (error) { + console.log('Couldn\'t send slow operation alert to Hawk', error); + } +} + +export { truncateText }; diff --git a/test/metrics/graphqlRequestDetails.test.ts b/test/metrics/graphqlRequestDetails.test.ts new file mode 100644 index 00000000..f7a3542f --- /dev/null +++ b/test/metrics/graphqlRequestDetails.test.ts @@ -0,0 +1,79 @@ +import { GraphQLError } from 'graphql'; +import { + buildGraphqlRequestContext, + formatGraphqlErrorsForAlert, +} from '../../src/metrics/graphqlRequestDetails'; + +describe('buildGraphqlRequestContext', () => { + it('should include user, highlighted ids and sanitized variables', () => { + const context = buildGraphqlRequestContext({ + context: { + user: { + id: 'user-1', + accessTokenExpired: false, + }, + }, + request: { + variables: { + projectId: '6989be3a0bc03531cc430c72', + input: { + workspaceId: 'workspace-1', + password: 'secret', + }, + search: 'TypeError', + }, + }, + } as never); + + expect(context).toEqual({ + userId: 'user-1', + ids: { + projectId: '6989be3a0bc03531cc430c72', + 'input.workspaceId': 'workspace-1', + search: 'TypeError', + }, + variables: '{"projectId":"6989be3a0bc03531cc430c72","input":{"workspaceId":"workspace-1","password":"[redacted]"},"search":"TypeError"}', + }); + }); + + it('should tolerate null GraphQL variables', () => { + const context = buildGraphqlRequestContext({ + context: { + user: { + id: 'user-1', + accessTokenExpired: false, + }, + }, + request: { + variables: null, + }, + } as never); + + expect(context).toEqual({ + userId: 'user-1', + }); + }); +}); + +describe('formatGraphqlErrorsForAlert', () => { + it('should flatten error messages into a single string', () => { + const text = formatGraphqlErrorsForAlert([ + new GraphQLError('First error'), + new GraphQLError('Second error'), + ]); + + expect(text).toBe('First error; Second error'); + }); + + it('should cap the number of errors and truncate long payloads', () => { + const errors = Array.from({ length: 12 }, (_, index) => { + return new GraphQLError(`validation failed on field_${index}: ${'x'.repeat(200)}`); + }); + const text = formatGraphqlErrorsForAlert(errors); + + expect(text.startsWith('validation failed on field_0:')).toBe(true); + expect(text).toContain('…(+2 more)'); + expect(text).toContain('…; …(+2 more)'); + expect(text.length).toBeLessThanOrEqual(1200); + }); +}); diff --git a/test/metrics/slowOperationAlert.test.ts b/test/metrics/slowOperationAlert.test.ts new file mode 100644 index 00000000..0fa90d91 --- /dev/null +++ b/test/metrics/slowOperationAlert.test.ts @@ -0,0 +1,54 @@ +import HawkCatcher from '@hawk.so/nodejs'; +import { notifySlowOperation, SLOW_OPERATION_THRESHOLD_MS } from '../../src/metrics/slowOperationAlert'; + +jest.mock('@hawk.so/nodejs', () => ({ + __esModule: true, + default: { + send: jest.fn(), + }, +})); + +describe('slowOperationAlert', () => { + const originalNodeEnv = process.env.NODE_ENV; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.NODE_ENV = 'development'; + }); + + afterAll(() => { + process.env.NODE_ENV = originalNodeEnv; + }); + + it('should send Hawk alert for slow operations', () => { + notifySlowOperation('Slow GraphQL operation: query ProjectDailyEvents', SLOW_OPERATION_THRESHOLD_MS, { + projectId: 'project-1', + }); + + expect(HawkCatcher.send).toHaveBeenCalledWith( + expect.objectContaining({ message: 'Slow GraphQL operation: query ProjectDailyEvents' }), + { + durationMs: SLOW_OPERATION_THRESHOLD_MS, + projectId: 'project-1', + } + ); + }); + + it('should not send Hawk alert for fast operations', () => { + notifySlowOperation('fast op', SLOW_OPERATION_THRESHOLD_MS - 1, { + projectId: 'project-1', + }); + + expect(HawkCatcher.send).not.toHaveBeenCalled(); + }); + + it('should not send Hawk alert in test environment', () => { + process.env.NODE_ENV = 'test'; + + notifySlowOperation('slow op', SLOW_OPERATION_THRESHOLD_MS, { + projectId: 'project-1', + }); + + expect(HawkCatcher.send).not.toHaveBeenCalled(); + }); +}); From 453ab1fe6b5e3f2c9b0c24067844f571521b1bf7 Mon Sep 17 00:00:00 2001 From: Peter Date: Mon, 10 Aug 2026 22:46:58 +0300 Subject: [PATCH 4/4] chore(billing): improve notifications (#675) * chore(billing): improve notifications * Bump version up to 1.5.9 * Bump version up to 1.5.10 * Update billingNew.ts --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- package.json | 2 +- src/billing/cloudpayments.ts | 75 +++++++++++++++++++++++++----------- src/resolvers/billingNew.ts | 6 ++- 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/package.json b/package.json index c2a99968..6f67f5d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "hawk.api", - "version": "1.5.9", + "version": "1.5.10", "main": "index.ts", "license": "BUSL-1.1", "scripts": { diff --git a/src/billing/cloudpayments.ts b/src/billing/cloudpayments.ts index bcee09e0..c24588aa 100644 --- a/src/billing/cloudpayments.ts +++ b/src/billing/cloudpayments.ts @@ -202,7 +202,7 @@ export default class CloudPaymentsWebhooks { return; } - telegram.sendMessage(`πŸ€— [Billing / Check] All checks passed successfully Β«${workspace.name}Β»`, TelegramBotURLs.Money) + telegram.sendMessage(`πŸ€— [Billing / Check] All checks passed successfully ${this.formatWorkspaceForTelegram(workspace)}`, TelegramBotURLs.Money) .catch(e => console.error('Error while sending message to Telegram: ' + e)); HawkCatcher.send(new Error('[Billing / Check] All checks passed successfully'), body as any); @@ -425,7 +425,7 @@ export default class CloudPaymentsWebhooks { this.handleSendingToTelegramError(telegram.sendMessage(`βœ… [Billing / Pay] Card linked -workspace id: ${workspace._id} +${this.formatWorkspaceForTelegram(workspace)} date of operation: ${body.DateTime} first payment date: ${data.cloudPayments?.recurrent.startDate} card link charge: ${+body.Amount} ${body.Currency} @@ -451,14 +451,14 @@ plan monthly charge: ${data.cloudPayments?.recurrent.amount} ${body.Currency}` amount: ${+body.Amount} ${body.Currency} next payment date: ${data.cloudPayments?.recurrent.startDate} -workspace id: ${workspace._id} +${this.formatWorkspaceForTelegram(workspace)} date of operation: ${body.DateTime} subscription id: ${body.SubscriptionId}`; } else { messageText = `βœ… [Billing / Pay] New Recurrent payment amount: ${+body.Amount} ${body.Currency} -workspace id: ${workspace._id} +${this.formatWorkspaceForTelegram(workspace)} date of operation: ${body.DateTime} subscription id: ${body.SubscriptionId}`; } @@ -491,10 +491,12 @@ subscription id: ${body.SubscriptionId}`; console.log('πŸ’Ž CloudPayments /fail request', body); + const failReasonLine = `reason: ${body.Reason} (${body.ReasonCode})`; + try { data = await this.getDataFromRequest(req); } catch (e) { - this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Invalid request`, body); + this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Invalid request\n${failReasonLine}`, body); return; } @@ -508,7 +510,7 @@ subscription id: ${body.SubscriptionId}`; */ if (!data.workspaceId || !data.userId || !data.tariffPlanId) { - this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] No workspace or user id or plan id in request body`, body); + this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] No workspace or user id or plan id in request body\n${failReasonLine}`, body); return; } @@ -520,7 +522,7 @@ subscription id: ${body.SubscriptionId}`; } catch (e) { const error = e as Error; - this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] ${error.toString()}`, body); + this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] ${error.toString()}\n${failReasonLine}`, body); return; } @@ -530,7 +532,7 @@ subscription id: ${body.SubscriptionId}`; } catch (e) { const error = e as Error; - this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Can't update business operation status ${error.toString()}`, body); + this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Can't update business operation status ${error.toString()}\n${failReasonLine}`, body); return; } @@ -548,12 +550,18 @@ subscription id: ${body.SubscriptionId}`; } catch (e) { const error = e as Error; - this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Error while sending notification to the user ${error.toString()}`, body); + this.sendError(res, FailCodes.SUCCESS, `[Billing / Fail] Error while sending notification to the user ${error.toString()}\n${failReasonLine}`, body); return; } - this.handleSendingToTelegramError(telegram.sendMessage(`❌ [Billing / Fail] Transaction failed for Β«${workspace.name}Β»`, TelegramBotURLs.Money)); + this.handleSendingToTelegramError(telegram.sendMessage(`❌ [Billing / Fail] Transaction failed for ${this.formatWorkspaceForTelegram(workspace)} + +amount: ${+body.Amount} ${body.Currency} +${failReasonLine} +transaction id: ${body.TransactionId} +subscription id: ${body.SubscriptionId || 'none'}` + , TelegramBotURLs.Money)); HawkCatcher.send(new Error('[Billing / Fail] Transaction failed'), body as any); @@ -577,11 +585,21 @@ subscription id: ${body.SubscriptionId}`; const emoji = [SubscriptionStatus.CANCELLED, SubscriptionStatus.REJECTED].includes(body.Status) ? '❌' : 'βœ…'; + let workspace: WorkspaceModel | null = null; + let workspaceLookupError: Error | null = null; + + try { + workspace = await context.factories.workspacesFactory.findBySubscriptionId(body.Id); + } catch (e) { + workspaceLookupError = e as Error; + } + this.handleSendingToTelegramError(telegram.sendMessage(`${emoji} [Billing / Recurrent] New recurrent event amount: ${+body.Amount} ${body.Currency} next payment date: ${body.NextTransactionDate} -workspace id: ${body.AccountId} +${this.formatWorkspaceForTelegram(workspace)} +user id: ${body.AccountId} subscription id: ${body.Id} status: ${body.Status}` , TelegramBotURLs.Money)); @@ -591,18 +609,8 @@ status: ${body.Status}` switch (body.Status) { case SubscriptionStatus.CANCELLED: case SubscriptionStatus.REJECTED: { - let workspace; - - try { - /** - * If there is a workspace with subscription id then subscription was cancelled via CloudPayments admin panel (or other no garage way) - * We need to remove subscription id from workspace - */ - workspace = await context.factories.workspacesFactory.findBySubscriptionId(body.Id); - } catch (e) { - const error = e as Error; - - this.sendError(res, RecurrentCodes.SUCCESS, `[Billing / Recurrent] Can't get data from database: ${error.toString()}`, { + if (workspaceLookupError) { + this.sendError(res, RecurrentCodes.SUCCESS, `[Billing / Recurrent] Can't get data from database: ${workspaceLookupError.toString()}`, { body, workspace, }); @@ -624,6 +632,10 @@ status: ${body.Status}` } try { + /** + * Subscription was cancelled via CloudPayments admin panel (or other no-garage way). + * Remove subscription id from workspace. + */ await workspace.setSubscriptionId(null); } catch (e) { const error = e as Error; @@ -641,6 +653,23 @@ status: ${body.Status}` } as RecurrentResponse); } + /** + * Formats workspace identity for Telegram money notifications + * + * @param workspace - workspace model or null when it was not found + */ + private formatWorkspaceForTelegram(workspace: Pick | null): string { + if (!workspace) { + return 'workspace: unknown'; + } + + if (workspace.name) { + return `workspace: Β«${workspace.name}Β» (${workspace._id})`; + } + + return `workspace id: ${workspace._id}`; + } + /** * Get workspace by workspace id * diff --git a/src/resolvers/billingNew.ts b/src/resolvers/billingNew.ts index ee973d22..d5d96873 100644 --- a/src/resolvers/billingNew.ts +++ b/src/resolvers/billingNew.ts @@ -126,7 +126,9 @@ export default { isCardLinkOperation = true; } - // Calculate next payment date + /** + * Calculate next payment date + */ const lastChargeDate = workspace.lastChargeDate ? new Date(workspace.lastChargeDate) : now; const nextPaymentDate = isCardLinkOperation ? new Date(lastChargeDate) : new Date(now); @@ -163,7 +165,7 @@ card link operation: ${isCardLinkOperation} amount: ${+plan.monthlyCharge} RUB last charge date: ${workspace.lastChargeDate?.toISOString()} next payment date: ${nextPaymentDate.toISOString()} -workspace id: ${workspace._id.toString()} +workspace: Β«${workspace.name}Β» (${workspace._id.toString()}) debug: ${Boolean(workspace.isDebug)}` , TelegramBotURLs.Money) .catch(e => console.error('Error while sending message to Telegram: ' + e));