feat: improve dates parser & catch empty response (ASG-4721 #IN_PROGRESS) (#4)

Author: rebecamendez

README.md

@@ -50,7 +50,5 @@ Make sure you have all necessary environment variables and dependencies set up b
   - error handling, detect "empty" response (not yet connected)
   - measure time and return to storage in DB for metabase stats
   - more tests, integration test?
-- Github action to deploy in aws
 - Send result to SQS queue
 - Script to get linkedin token
-- Review dates parser: fallback MMM yyyy -> yyyy ?

src/index.spec.ts

@@ -25,7 +25,10 @@ describe('Linkedin lambda handler', () => {
     const event = createMockedSqsSEvent();
     const expectedLinkedinProfile = createMockedLinkedinProfile();
 
-    linkedinProfileService.getLinkedinProfile.calledWith('fake-token').mockResolvedValue(expectedLinkedinProfile);
+    linkedinProfileService.getLinkedinProfile.calledWith('fake-token').mockResolvedValue({
+      linkedinProfile: expectedLinkedinProfile,
+      isEmptyProfile: false
+    });
 
     const response = await handler(event, {} as Context, () => {});
 
@@ -58,7 +61,10 @@ describe('Linkedin lambda handler', () => {
     const expectedErrorString =
       '[LinkedinProfileResponseMapper] MAC Validation failed: ["property: mac.aboutMe.profile.title errors: title should not be empty"]';
 
-    linkedinProfileService.getLinkedinProfile.calledWith('fake-token').mockResolvedValue(expectedLinkedinProfile);
+    linkedinProfileService.getLinkedinProfile.calledWith('fake-token').mockResolvedValue({
+      linkedinProfile: expectedLinkedinProfile,
+      isEmptyProfile: false
+    });
 
     await expect(handler(event, {} as Context, () => {})).rejects.toThrow(new Error(expectedErrorString));
   });

src/index.ts

@@ -7,20 +7,27 @@ import { LinkedinProfileService } from './services/linkedin-profile.service';
 import { Environment } from './util/environment';
 import { logger } from './util/logger';
 
-export const handler: Handler = async (event: SQSEvent): Promise<LinkedinProfileResponse> => {
+export const handler: Handler = async (event: SQSEvent): Promise<LinkedinProfileResponse | undefined> => {
   try {
     const linkedinProfileRequest = LinkedinProfileRequestMapper.toDomain(event);
     Environment.setupEnvironment(linkedinProfileRequest);
+    logger.info(`⌛️ [handler] Starting Linkedin profile request for linkedinUrl: ${linkedinProfileRequest.linkedinUrl}`, linkedinProfileRequest);
 
-    const linkedinProfile = await new LinkedinProfileService().getLinkedinProfile(linkedinProfileRequest.profileApiToken);
-    const linkedinProfileResponse = LinkedinProfileResponseMapper.toResponse(linkedinProfile);
+    const { linkedinProfile, isEmptyProfile } = await new LinkedinProfileService().getLinkedinProfile(linkedinProfileRequest.profileApiToken);
 
-    logger.info(`✅ [handler] Linkedin profile response with MAC: ${JSON.stringify(linkedinProfileResponse.mac)}`);
+    if (isEmptyProfile) {
+      logger.warn(`👻 [handler] Linkedin profile is not synced for linkedinUrl: ${linkedinProfileRequest.linkedinUrl}`, linkedinProfileRequest);
+      // TODO: send response to SQS queue again if no max retries
+      return undefined;
+    }
 
-    // TODO: send response to SQS queue
+    const linkedinProfileResponse = LinkedinProfileResponseMapper.toResponse(linkedinProfile);
+    logger.info(`✅ [handler] Linkedin profile response with MAC: ${JSON.stringify(linkedinProfileResponse.mac)}`, linkedinProfileRequest);
+    // TODO: send response to result SQS queue
 
     return linkedinProfileResponse;
   } catch (error) {
+    logger.error(`❌ [handler] Error processing Linkedin profile request: ${error}`, error);
     throw error;
   }
 };

src/mappers/linkedin-profile.response.mapper.ts

@@ -64,8 +64,8 @@ export class LinkedinProfileResponseMapper {
 
         const role = new LinkedinProfileResponseMacJobRole();
         role.name = position.Title || '';
-        role.startDate = DateUtilities.toIsoDate(position['Started On'], 'MMM yyyy') || new Date().toISOString();
-        role.finishDate = position['Finished On'] ? DateUtilities.toIsoDate(position['Finished On'], 'MMM yyyy') : undefined;
+        role.startDate = DateUtilities.toIsoDate(position['Started On']);
+        role.finishDate = position['Finished On'] ? DateUtilities.toIsoDate(position['Finished On']) : undefined;
 
         const challenge = new LinkedinProfileResponseMacExperienceJobChallenge();
         challenge.description = position.Description || '';
@@ -95,8 +95,8 @@ export class LinkedinProfileResponseMapper {
       const study = new LinkedinProfileResponseMacStudy();
       study.studyType = 'officialDegree';
       study.name = educationItem['Degree Name'] || educationItem['School Name'] || '';
-      study.startDate = DateUtilities.toIsoDate(educationItem['Start Date'], 'yyyy') || new Date().toISOString();
-      study.finishDate = educationItem['End Date'] ? DateUtilities.toIsoDate(educationItem['End Date'], 'yyyy') : undefined;
+      study.startDate = DateUtilities.toIsoDate(educationItem['Start Date']);
+      study.finishDate = educationItem['End Date'] ? DateUtilities.toIsoDate(educationItem['End Date']) : undefined;
       study.degreeAchieved = !!educationItem['End Date'];
       study.description = educationItem['Degree Name'] || educationItem['School Name'];
       if (educationItem['School Name']) {
@@ -111,7 +111,6 @@ export class LinkedinProfileResponseMapper {
   private static validate(response: LinkedinProfileResponse): void {
     const errors = validateSync(response);
     if (errors.length > 0) {
-      console.log(JSON.stringify(response.mac, null, 2));
       logger.error(`[LinkedinProfileResponseMapper] MAC Validation failed: ${JSON.stringify(errors)}`, { errors, response });
       const formattedErrors = ValidationUtilities.formatErrors(errors);
       throw new Error(`[LinkedinProfileResponseMapper] MAC Validation failed: ${JSON.stringify(formattedErrors)}`);

src/services/linkedin-api.client.spec.ts

@@ -29,7 +29,7 @@ describe('A linkedin api client', () => {
       return undefined;
     });
 
-    const education = await client.fetchProfileDomainData('fake_token', 'EDUCATION');
+    const education = await client.fetchProfileDomainData('fake_token', 'EDUCATION', 'ARRAY');
 
     expect(education).toEqual(expectedEducation);
   });

src/services/linkedin-api.client.ts

@@ -2,17 +2,15 @@ import axios, { AxiosError } from 'axios';
 import { logger } from '../util/logger';
 
 export type LinkedinDomain = 'PROFILE' | 'SKILLS' | 'POSITIONS' | 'EDUCATION';
+export type LinkedinDomainResponseType = 'OBJECT' | 'ARRAY';
 
 export class LinkedinAPIClient {
   public constructor() {}
 
-  /* TODO: 
-    - refactor to not check for domain type
-    - improve error handling: detect not provided data
-  */
-  public async fetchProfileDomainData<A>(token: string, domain: LinkedinDomain): Promise<A> {
+  public async fetchProfileDomainData<A>(token: string, domain: LinkedinDomain, responseType: LinkedinDomainResponseType): Promise<A> {
     const url = `https://api.linkedin.com/rest/memberSnapshotData?q=criteria&domain=${domain}`;
-    const isResponseArray = domain !== 'PROFILE';
+    const isArrayData = responseType === 'ARRAY';
+
     logger.debug(`🔍 [LinkedinAPIClient] Fetching ${domain} domain profile data ...`);
     try {
       const headers = {
@@ -22,10 +20,10 @@ export class LinkedinAPIClient {
 
       const response = await axios.get(url, { headers });
 
-      const data = isResponseArray ? response.data.elements[0].snapshotData : response.data.elements[0].snapshotData[0];
+      const data = isArrayData ? response.data.elements[0].snapshotData : response.data.elements[0].snapshotData[0];
       return data;
     } catch (error: unknown) {
-      if (error instanceof AxiosError && error.response?.status === 404) return (isResponseArray ? [] : {}) as A;
+      if (error instanceof AxiosError && error.response?.status === 404) return (isArrayData ? [] : {}) as A;
       if (error instanceof AxiosError) {
         const responseData = JSON.stringify(error.response?.data);
         logger.error(`🚨 [LinkedinAPIClient] Error fetching ${domain} profile data: ${responseData}`, error.stack);

src/services/linkedin-profile.service.spec.ts

@@ -19,13 +19,23 @@ describe('A linkedin profile service', () => {
   it('should get linkedin profile using client', async () => {
     const expectedProfile = createMockedLinkedinProfile();
 
-    client.fetchProfileDomainData.calledWith('fake_token', 'PROFILE').mockResolvedValue(expectedProfile.profile);
-    client.fetchProfileDomainData.calledWith('fake_token', 'SKILLS').mockResolvedValue(expectedProfile.skills);
-    client.fetchProfileDomainData.calledWith('fake_token', 'POSITIONS').mockResolvedValue(expectedProfile.positions);
-    client.fetchProfileDomainData.calledWith('fake_token', 'EDUCATION').mockResolvedValue(expectedProfile.education);
+    client.fetchProfileDomainData.calledWith('fake_token', 'PROFILE', 'OBJECT').mockResolvedValue(expectedProfile.profile);
+    client.fetchProfileDomainData.calledWith('fake_token', 'SKILLS', 'ARRAY').mockResolvedValue(expectedProfile.skills);
+    client.fetchProfileDomainData.calledWith('fake_token', 'POSITIONS', 'ARRAY').mockResolvedValue(expectedProfile.positions);
+    client.fetchProfileDomainData.calledWith('fake_token', 'EDUCATION', 'ARRAY').mockResolvedValue(expectedProfile.education);
 
-    const profile = await service.getLinkedinProfile('fake_token');
+    const { linkedinProfile, isEmptyProfile } = await service.getLinkedinProfile('fake_token');
+    expect(linkedinProfile).toEqual(expectedProfile);
+    expect(isEmptyProfile).toBe(false);
+  });
+
+  it('should return empty profile flag', async () => {
+    client.fetchProfileDomainData.calledWith('fake_token', 'PROFILE', 'OBJECT').mockResolvedValue({});
+    client.fetchProfileDomainData.calledWith('fake_token', 'SKILLS', 'ARRAY').mockResolvedValue([]);
+    client.fetchProfileDomainData.calledWith('fake_token', 'POSITIONS', 'ARRAY').mockResolvedValue([]);
+    client.fetchProfileDomainData.calledWith('fake_token', 'EDUCATION', 'ARRAY').mockResolvedValue([]);
 
-    expect(profile).toEqual(expectedProfile);
+    const { isEmptyProfile } = await service.getLinkedinProfile('fake_token');
+    expect(isEmptyProfile).toBe(true);
   });
 });

src/services/linkedin-profile.service.ts

@@ -13,15 +13,23 @@ export class LinkedinProfileService {
 
   public constructor() {}
 
-  public async getLinkedinProfile(token: string): Promise<LinkedinProfile> {
-    const profile = await this.client.fetchProfileDomainData<LinkedinProfileProfile>(token, 'PROFILE');
-    const skills = await this.client.fetchProfileDomainData<LinkedinProfileSkill[]>(token, 'SKILLS');
-    const positions = await this.client.fetchProfileDomainData<LinkedinProfilePosition[]>(token, 'POSITIONS');
-    const education = await this.client.fetchProfileDomainData<LinkedinProfileEducation[]>(token, 'EDUCATION');
+  public async getLinkedinProfile(token: string): Promise<{ linkedinProfile: LinkedinProfile; isEmptyProfile: boolean }> {
+    const profile = await this.client.fetchProfileDomainData<LinkedinProfileProfile>(token, 'PROFILE', 'OBJECT');
+    const skills = await this.client.fetchProfileDomainData<LinkedinProfileSkill[]>(token, 'SKILLS', 'ARRAY');
+    const positions = await this.client.fetchProfileDomainData<LinkedinProfilePosition[]>(token, 'POSITIONS', 'ARRAY');
+    const education = await this.client.fetchProfileDomainData<LinkedinProfileEducation[]>(token, 'EDUCATION', 'ARRAY');
 
     const linkedinProfile = { profile, skills, positions, education };
+    const isEmptyProfile = this.isEmptyProfile(profile);
+    logger.debug(`🧐 [LinkedinProfileService] Linkedin profile retrieved: ${JSON.stringify(linkedinProfile)}`);
 
-    logger.debug(`🧐 [LinkedinProfileService] Linkedin profile retrieved: ${JSON.stringify(linkedinProfile)}`, { linkedinProfile });
-    return linkedinProfile;
+    return { linkedinProfile, isEmptyProfile };
+  }
+
+  // --- 🔐 Private methods
+
+  private isEmptyProfile(profile: LinkedinProfileProfile): boolean {
+    const json = JSON.stringify(profile);
+    return json === '{}';
   }
 }

src/util/date.spec.ts

@@ -1,35 +1,45 @@
 import { DateUtilities } from './date';
 
-describe('date utities', () => {
+describe('date utilities', () => {
+  beforeAll(() => {
+    jest.useFakeTimers().setSystemTime(new Date('2023-10-01'));
+  });
+
+  afterAll(() => {
+    jest.useRealTimers();
+  });
+
   describe('when calling toIsoDate', () => {
-    it('should return the date in ISO format with `MMM yyyy` format', () => {
+    it('should correctly format a date in `MMM yyyy` format to ISO format', () => {
       const date = 'Mar 2026';
       const expected = '2026-03-01';
 
-      const result = DateUtilities.toIsoDate(date, 'MMM yyyy');
+      const result = DateUtilities.toIsoDate(date);
       expect(result).toBe(expected);
     });
 
-    it('should return the date in ISO format with `yyyy` format', () => {
+    it('should correctly format a date in `yyyy` format to ISO format', () => {
       const date = '2026';
       const expected = '2026-01-01';
 
-      const result = DateUtilities.toIsoDate(date, 'yyyy');
+      const result = DateUtilities.toIsoDate(date);
       expect(result).toBe(expected);
     });
 
-    it('should return undefined when the date is not provided', () => {
+    it("should return today's date in ISO format when the date is not provided", () => {
       const date = undefined;
+      const expected = '2023-10-01';
 
-      const result = DateUtilities.toIsoDate(date, 'MMM yyyy');
-      expect(result).toBeUndefined();
+      const result = DateUtilities.toIsoDate(date);
+      expect(result).toBe(expected);
     });
 
-    it('should return undefined when the date is not valid', () => {
+    it("should return today's date in ISO format when the date is invalid", () => {
       const date = 'invalid date';
+      const expected = '2023-10-01';
 
-      const result = DateUtilities.toIsoDate(date, 'MMM yyyy');
-      expect(result).toBeUndefined();
+      const result = DateUtilities.toIsoDate(date);
+      expect(result).toBe(expected);
     });
   });
 });

src/util/date.ts

@@ -1,15 +1,27 @@
 import { format, parse } from 'date-fns';
 import { enUS } from 'date-fns/locale';
+import { logger } from './logger';
 
 export class DateUtilities {
-  public static toIsoDate(date: string | undefined, dateFormat: 'MMM yyyy' | 'yyyy'): string | undefined {
-    if (!date) return undefined;
-
-    try {
-      const parsedDate = parse(date, dateFormat, new Date(), { locale: enUS });
-      return format(parsedDate, 'yyyy-MM-dd');
-    } catch {
-      return undefined;
+  public static toIsoDate(date: string | undefined): string {
+    const dateFormats = ['MMM yyyy', 'yyyy'];
+    const now = format(new Date(), 'yyyy-MM-dd');
+
+    if (!date) {
+      logger.warn(`[DateUtilities] Date is not provided, so returning today's date`);
+      return now;
+    }
+
+    for (const dateFormat of dateFormats) {
+      try {
+        const parsedDate = parse(date, dateFormat, new Date(), { locale: enUS });
+        if (!isNaN(parsedDate.getTime())) return format(parsedDate, 'yyyy-MM-dd');
+      } catch {
+        continue;
+      }
     }
+
+    logger.warn(`[DateUtilities] Could not parse date: ${date}, so returning today's date`);
+    return now;
   }
 }