From cd18665349150d14f5eb603d92120538c4842071 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:07:14 +0530 Subject: [PATCH 1/3] feat: migrate crustdata to live person enrich API (CM-1354) Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/sources/crustdata/service.ts | 161 +++++++++--------- .../src/sources/crustdata/types.ts | 100 ++++++----- 2 files changed, 145 insertions(+), 116 deletions(-) diff --git a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts index 44022ed299..80a2e8a814 100644 --- a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts @@ -1,6 +1,6 @@ import axios from 'axios' -import { isEmail, replaceDoubleQuotes } from '@crowd/common' +import { replaceDoubleQuotes } from '@crowd/common' import { Logger, LoggerBase } from '@crowd/logging' import { IMemberEnrichmentCache, @@ -25,8 +25,8 @@ import { import { normalizeAttributes, normalizeSocialIdentity } from '../../utils/common' import { - IMemberEnrichmentCrustdataAPIErrorResponse, - IMemberEnrichmentCrustdataAPIResponse, + IMemberEnrichmentCrustdataEnrichResponse, + IMemberEnrichmentCrustdataPersonData, IMemberEnrichmentCrustdataRemainingCredits, IMemberEnrichmentDataCrustdata, } from './types' @@ -51,38 +51,41 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE public attributeSettings: IMemberEnrichmentAttributeSettings = { [MemberAttributeName.AVATAR_URL]: { - fields: ['profile_picture_url'], + // Fallback order: stable permalink, then CDN url. + fields: [ + 'professional_network.profile_picture_permalink', + 'basic_profile.profile_picture_permalink', + 'professional_network.profile_picture_url', + ], }, [MemberAttributeName.JOB_TITLE]: { - fields: ['title'], + fields: ['basic_profile.current_title'], }, [MemberAttributeName.BIO]: { - fields: ['summary', 'headline'], + fields: ['basic_profile.summary', 'basic_profile.headline'], }, [MemberAttributeName.SKILLS]: { - fields: ['skills'], - // Note: Crustdata API docs specify skills as string, but API returns string[] - // So we're handling both cases in the transformer. - transform: (skills: string | string[]) => { + fields: ['skills.professional_network_skills'], + transform: (skills: string[]) => { if (!skills) { return [] } - const arr = Array.isArray(skills) ? skills : skills.split(',') - - return arr + return skills .map((s) => s.trim()) .filter(Boolean) .sort() }, }, [MemberAttributeName.LANGUAGES]: { - fields: ['languages'], - transform: (languages: string[]) => languages.sort(), + fields: ['basic_profile.languages'], + transform: (languages: string[]) => (languages || []).sort(), }, [MemberAttributeName.SCHOOLS]: { - fields: ['all_schools'], - transform: (schools: string[]) => schools.sort(), + // Same school can appear multiple times (different degrees). + fields: ['education.schools'], + transform: (schools: Array<{ school?: string }>) => + [...new Set((schools || []).map((s) => s.school?.trim()).filter(Boolean))].sort(), }, } @@ -134,16 +137,17 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE try { const config = { method: 'get', - url: `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/user/credits`, + url: `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/account/credits`, headers: { - Authorization: `Token ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + Authorization: `Bearer ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + 'x-api-version': '2025-11-01', }, } const response: IMemberEnrichmentCrustdataRemainingCredits = (await axios(config)).data - // realtime linkedin enrichment costs 5 credits - return response.credits > 5 + // Live enrich costs 7 credits per profile. + return response.account.credits > 7 } catch (error) { this.log.error('Error while checking Crustdata account usage', error) throw error @@ -174,40 +178,47 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE private async getDataUsingLinkedinHandle( handle: string, - ): Promise { + ): Promise { const config = { - method: 'get', - url: `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/screener/person/enrich`, - params: { - linkedin_profile_url: `https://linkedin.com/in/${encodeURIComponent(handle)}`, - enrich_realtime: true, - }, + method: 'post', + url: `${process.env['CROWD_ENRICHMENT_CRUSTDATA_URL']}/person/professional_network/enrich/live`, headers: { - Authorization: `Token ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + Authorization: `Bearer ${process.env['CROWD_ENRICHMENT_CRUSTDATA_API_KEY']}`, + 'x-api-version': '2025-11-01', + 'content-type': 'application/json', + }, + data: { + professional_network_profile_urls: [`https://www.linkedin.com/in/${handle}`], + // Default response is only basic_profile + social_handles. + fields: [ + 'basic_profile', + 'social_handles', + 'professional_network', + 'experience', + 'education', + 'skills', + ], }, validateStatus: function (status) { - return (status >= 200 && status < 300) || status === 404 || status === 422 + return (status >= 200 && status < 300) || status === 404 }, } - const response = await axios(config) + const response = await axios(config) - if (response.status === 404 || response.status === 422) { + if (response.status === 404) { this.log.debug({ source: this.source, handle }, 'No data found for linkedin handle!') return null } - if (response.data.length === 0 || this.isErrorResponse(response.data[0])) { + // No match returns 200 with empty matches[]. + const match = response.data?.[0]?.matches?.[0] + if (!match?.person_data) { + this.log.debug({ source: this.source, handle }, 'No data found for linkedin handle!') return null } - return response.data[0] - } - - private isErrorResponse( - response: IMemberEnrichmentCrustdataAPIResponse, - ): response is IMemberEnrichmentCrustdataAPIErrorResponse { - return (response as IMemberEnrichmentCrustdataAPIErrorResponse).error !== undefined + return match.person_data } private async findDistinctScrapableLinkedinIdentities( @@ -288,8 +299,8 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE normalized = normalizeAttributes(data, normalized, this.attributeSettings, this.platform) normalized = this.normalizeEmployment(data, normalized) - if (data.num_of_connections) { - normalized.reach[this.platform] = data.num_of_connections + if (data.professional_network?.connections) { + normalized.reach[this.platform] = data.professional_network.connections } return normalized @@ -307,34 +318,17 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE normalized.attributes = {} } - if (data.name) { - normalized.displayName = data.name - } - - if (data.email) { - let emails: string[] - - if (Array.isArray(data.email)) { - emails = data.email - } else { - emails = data.email.split(',').filter(isEmail) - } - - for (const email of emails) { - normalized.identities.push({ - type: MemberIdentityType.EMAIL, - platform: this.platform, - value: email.trim(), - verified: false, - source: 'enrichment', - }) - } + if (data.basic_profile?.name) { + normalized.displayName = data.basic_profile.name } - if (data.twitter_handle) { + // Crustdata social_handles use generic identifiers: + // professional_network = LinkedIn, dev_platform = GitHub. + const twitterHandle = data.social_handles?.twitter_identifier?.slug + if (twitterHandle) { normalized = normalizeSocialIdentity( { - handle: data.twitter_handle, + handle: twitterHandle, platform: PlatformType.TWITTER, }, MemberIdentityType.USERNAME, @@ -342,10 +336,11 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE ) } - if (data.linkedin_flagship_url) { + const linkedinUrl = data.social_handles?.professional_network_identifier?.profile_url + if (linkedinUrl) { normalized = normalizeSocialIdentity( { - handle: data.linkedin_flagship_url.split('/').pop(), + handle: linkedinUrl.split('/').filter(Boolean).pop(), platform: PlatformType.LINKEDIN, }, MemberIdentityType.USERNAME, @@ -353,6 +348,21 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE ) } + const githubUrl = data.social_handles?.dev_platform_identifier?.profile_url + if (githubUrl) { + const handle = githubUrl.split('/').filter(Boolean).pop() + if (handle) { + normalized = normalizeSocialIdentity( + { + handle, + platform: PlatformType.GITHUB, + }, + MemberIdentityType.USERNAME, + normalized, + ) + } + } + return normalized } @@ -364,15 +374,17 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE normalized.memberOrganizations = [] } - const employmentInformation = (data.past_employers || []).concat(data.current_employers || []) + const employmentInformation = (data.experience?.employment_details?.past || []).concat( + data.experience?.employment_details?.current || [], + ) if (employmentInformation.length > 0) { for (const workExperience of employmentInformation) { const identities = [] - if (workExperience.employer_linkedin_id) { + if (workExperience.professional_network_id) { identities.push({ platform: PlatformType.LINKEDIN, - value: `company:${workExperience.employer_linkedin_id}`, + value: `company:${workExperience.professional_network_id}`, type: OrganizationIdentityType.USERNAME, verified: true, source: 'enrichment', @@ -380,15 +392,12 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE } normalized.memberOrganizations.push({ - name: replaceDoubleQuotes(workExperience.employer_name), + name: replaceDoubleQuotes(workExperience.name), source: OrganizationSource.ENRICHMENT_CRUSTDATA, identities, - title: replaceDoubleQuotes(workExperience.employee_title), + title: replaceDoubleQuotes(workExperience.title), startDate: workExperience?.start_date ?? null, endDate: workExperience?.end_date ?? null, - organizationDescription: replaceDoubleQuotes( - workExperience.employer_linkedin_description, - ), }) } } diff --git a/services/apps/members_enrichment_worker/src/sources/crustdata/types.ts b/services/apps/members_enrichment_worker/src/sources/crustdata/types.ts index 7893df81d1..f612feda52 100644 --- a/services/apps/members_enrichment_worker/src/sources/crustdata/types.ts +++ b/services/apps/members_enrichment_worker/src/sources/crustdata/types.ts @@ -1,53 +1,73 @@ import { IMemberEnrichmentLinkedinScraperMetadata } from '../../types' export interface IMemberEnrichmentCrustdataEmployer { - employer_name: string - employer_linkedin_id: string - employer_linkedin_description?: string - employer_company_id: number[] - employee_position_id?: number - employee_title: string - employee_description?: string - employee_location: string - start_date: string - end_date: string + name?: string + title?: string + professional_network_id?: string + start_date?: string + end_date?: string } -export interface IMemberEnrichmentDataCrustdata { - linkedin_profile_url: string - linkedin_flagship_url: string - name: string - email: string | string[] - title: string - last_updated: string - headline: string - summary: string - num_of_connections: number - skills: string - profile_picture_url: string - twitter_handle: string - languages: string[] - all_employers: string[] - past_employers: IMemberEnrichmentCrustdataEmployer[] - current_employers: IMemberEnrichmentCrustdataEmployer[] - all_employers_company_id: number[] - all_titles: string[] - all_schools: string[] - all_degrees: string[] +export interface IMemberEnrichmentCrustdataPersonData { + basic_profile?: { + name?: string + current_title?: string + headline?: string + summary?: string + languages?: string[] + profile_picture_permalink?: string + } + social_handles?: { + professional_network_identifier?: { + profile_url?: string + } + twitter_identifier?: { + slug?: string + } + dev_platform_identifier?: { + profile_url?: string + } + } + professional_network?: { + connections?: number + profile_picture_url?: string + profile_picture_permalink?: string + } + skills?: { + professional_network_skills?: string[] + } + education?: { + schools?: Array<{ + school?: string + }> + } + experience?: { + employment_details?: { + current?: IMemberEnrichmentCrustdataEmployer[] + past?: IMemberEnrichmentCrustdataEmployer[] + } + } +} + +export interface IMemberEnrichmentDataCrustdata extends IMemberEnrichmentCrustdataPersonData { metadata: IMemberEnrichmentLinkedinScraperMetadata } -export type IMemberEnrichmentCrustdataAPIResponse = - | IMemberEnrichmentDataCrustdata - | IMemberEnrichmentCrustdataAPIErrorResponse +export interface IMemberEnrichmentCrustdataEnrichMatch { + confidence_score: number + person_data: IMemberEnrichmentCrustdataPersonData +} -export interface IMemberEnrichmentCrustdataAPIErrorResponse { - error: string - linkedin_profile_url: string - last_tried_linkedin_enrichment_date: string - did_last_linkedin_enrichment_succeed: boolean +export interface IMemberEnrichmentCrustdataEnrichResult { + matched_on: string + match_type: string + matches: IMemberEnrichmentCrustdataEnrichMatch[] } +export type IMemberEnrichmentCrustdataEnrichResponse = IMemberEnrichmentCrustdataEnrichResult[] + export interface IMemberEnrichmentCrustdataRemainingCredits { - credits: number + account: { + credits: number + } } From 6bdca306400caebc2fcd5d4c1b37ce6d4aa6cce0 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:19:29 +0530 Subject: [PATCH 2/3] fix: use inclusive credit gate for live enrich (CM-1354) Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/sources/crustdata/service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts index 80a2e8a814..c798a6b1ed 100644 --- a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts @@ -82,7 +82,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE transform: (languages: string[]) => (languages || []).sort(), }, [MemberAttributeName.SCHOOLS]: { - // Same school can appear multiple times (different degrees). + // education.schools[] is one entry per degree; CDP stores school names only. fields: ['education.schools'], transform: (schools: Array<{ school?: string }>) => [...new Set((schools || []).map((s) => s.school?.trim()).filter(Boolean))].sort(), @@ -147,7 +147,7 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE const response: IMemberEnrichmentCrustdataRemainingCredits = (await axios(config)).data // Live enrich costs 7 credits per profile. - return response.account.credits > 7 + return response.account.credits >= 7 } catch (error) { this.log.error('Error while checking Crustdata account usage', error) throw error From 30681a1e668294e48ec33a7bffcf5f33b07557f6 Mon Sep 17 00:00:00 2001 From: Yeganathan S <63534555+skwowet@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:20:15 +0530 Subject: [PATCH 3/3] fix: guard linkedin handle before creating identity (CM-1354) Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com> --- .../src/sources/crustdata/service.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts index c798a6b1ed..b41d038782 100644 --- a/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts +++ b/services/apps/members_enrichment_worker/src/sources/crustdata/service.ts @@ -338,14 +338,17 @@ export default class EnrichmentServiceCrustdata extends LoggerBase implements IE const linkedinUrl = data.social_handles?.professional_network_identifier?.profile_url if (linkedinUrl) { - normalized = normalizeSocialIdentity( - { - handle: linkedinUrl.split('/').filter(Boolean).pop(), - platform: PlatformType.LINKEDIN, - }, - MemberIdentityType.USERNAME, - normalized, - ) + const handle = linkedinUrl.split('/').filter(Boolean).pop() + if (handle) { + normalized = normalizeSocialIdentity( + { + handle, + platform: PlatformType.LINKEDIN, + }, + MemberIdentityType.USERNAME, + normalized, + ) + } } const githubUrl = data.social_handles?.dev_platform_identifier?.profile_url