chore: Add checksum for file uploads (#6266)

initial setup

Author: timarney

app/(gcforms)/[locale]/(form filler)/id/[...props]/actions.ts

@@ -31,7 +31,8 @@ export async function submitForm(
   values: Responses,
   language: string,
   formRecordOrId: PublicFormRecord | string,
-  captchaToken?: string | undefined
+  captchaToken?: string | undefined,
+  fileChecksums?: Record<string, string>
 ): Promise<{
   id: string;
   submissionId?: string;
@@ -114,6 +115,7 @@ export async function submitForm(
       securityAttribute: template.securityAttribute,
       formId,
       language,
+      fileChecksums,
     });
 
     sendNotifications(formId, template.form.titleEn, template.form.titleFr);

app/(gcforms)/[locale]/(form filler)/id/[...props]/lib/client/fileUploader.ts

@@ -4,6 +4,7 @@ import axios, { AxiosError, AxiosProgressEvent } from "axios";
 import { Responses, FileInputResponse, FileInput } from "@gcforms/types";
 import { FileUploadError } from "../client/exceptions";
 import { isMimeTypeValid } from "@gcforms/core";
+import { generateFileChecksums } from "@lib/utils/fileChecksum";
 
 const isFileInput = (response: unknown): response is FileInput => {
   return (
@@ -63,7 +64,11 @@ export const copyObjectExcludingFileContent = (
     return filteredState as unknown as T;
   };
   filterFileContent(originalObject, formValuesWithoutFileContent);
-  return { formValuesWithoutFileContent, fileObjsRef };
+
+  // Calculate checksums for all files key / value pairs
+  const fileChecksums = generateFileChecksums(fileObjsRef);
+
+  return { formValuesWithoutFileContent, fileObjsRef, fileChecksums };
 };
 
 export const uploadFile = async (

app/(gcforms)/[locale]/(form filler)/id/[...props]/lib/server/invokeSubmissionLambda.ts

@@ -9,21 +9,25 @@ export const invokeSubmissionLambda = async (
   formID: string,
   fields: Responses,
   language: string,
-  securityAttribute: string
+  securityAttribute: string,
+  fileChecksums?: Record<string, string>
 ): Promise<{
   submissionId: string;
   fileURLMap?: SignedURLMap;
 }> => {
   try {
+    const payload = {
+      formID,
+      language,
+      responses: fields,
+      securityAttribute,
+      ...(fileChecksums && Object.keys(fileChecksums).length > 0 && { fileChecksums }),
+    };
+
     const lambdaInvokeResponse = await lambdaClient.send(
       new InvokeCommand({
         FunctionName: "Submission",
-        Payload: JSON.stringify({
-          formID,
-          language,
-          responses: fields,
-          securityAttribute,
-        }),
+        Payload: JSON.stringify(payload),
       })
     );
 

app/(gcforms)/[locale]/(form filler)/id/[...props]/lib/server/processFormData.ts

@@ -10,13 +10,15 @@ type ProcessFormDataParams = {
   securityAttribute?: string;
   formId: string;
   language?: string;
+  fileChecksums?: Record<string, string>;
 };
 
 export const processFormData = async ({
   responses,
   securityAttribute,
   formId,
   language,
+  fileChecksums,
 }: ProcessFormDataParams): Promise<{
   submissionId: string;
   fileURLMap?: SignedURLMap;
@@ -56,7 +58,8 @@ export const processFormData = async ({
       form.id,
       responses,
       language ? language : "en",
-      securityAttribute ? securityAttribute : "Protected A"
+      securityAttribute ? securityAttribute : "Protected A",
+      fileChecksums
     );
 
     logMessage.info(`Response submitted for Form ID: ${form.id}`);

components/clientComponents/forms/Form/Form.tsx

@@ -300,7 +300,7 @@ export const Form = withFormik<FormProps, Responses>({
       );
 
       // Extract file content from formValues so they are not part of the submission call to the submit action
-      const { formValuesWithoutFileContent, fileObjsRef } =
+      const { formValuesWithoutFileContent, fileObjsRef, fileChecksums } =
         copyObjectExcludingFileContent(formValues);
 
       let submitProgress = 0;
@@ -328,7 +328,8 @@ export const Form = withFormik<FormProps, Responses>({
         formValuesWithoutFileContent,
         formikBag.props.language,
         formikBag.props.formRecord.id,
-        formikBag.props.captchaToken?.current
+        formikBag.props.captchaToken?.current,
+        fileChecksums
       );
 
       clearInterval(progressInterval);

lib/utils/fileChecksum.ts

@@ -0,0 +1,40 @@
+import { createHash } from "crypto";
+import { FileInput } from "@gcforms/types";
+import { logMessage } from "@lib/logger";
+
+export function calculateMD5Checksum(content: ArrayBuffer): string {
+  const buffer = Buffer.from(content);
+  const hash = createHash("md5");
+  hash.update(buffer);
+  return hash.digest("base64");
+}
+
+export function calculateFileChecksum(file: FileInput): string {
+  // Ensure file content is available
+  if (!file.content) {
+    throw new Error(`Unable to calculate checksum for file ${file.name}: no content available`);
+  }
+  return calculateMD5Checksum(file.content);
+}
+
+/**
+ * Take our fileObjsRefs and generate file checksums map for each file
+ * @param fileObjsRef - Map of file IDs to FileInput objects
+ * @returns Map of file IDs to their MD5 checksums
+ */
+export function generateFileChecksums(
+  fileObjsRef: Record<string, FileInput>
+): Record<string, string> {
+  const checksums: Record<string, string> = {};
+
+  for (const [fileId, file] of Object.entries(fileObjsRef)) {
+    try {
+      checksums[fileId] = calculateFileChecksum(file);
+    } catch (error) {
+      logMessage.warn(`Failed to calculate checksum for file ${file.name}: ${error}`);
+      // Continue with other files, don't fail the entire process
+    }
+  }
+
+  return checksums;
+}

lib/utils/tests/fileChecksum.vitest.ts

@@ -0,0 +1,126 @@
+import { describe, it, expect } from 'vitest';
+import { calculateMD5Checksum, calculateFileChecksum, generateFileChecksums } from '../fileChecksum';
+import { FileInput } from '@gcforms/types';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+
+describe('fileChecksum', () => {
+  // Helper function to load CSV fixtures
+  const loadCsvFixture = (filename: string): ArrayBuffer => {
+    const csvPath = join(__dirname, 'fixtures', filename);
+    return readFileSync(csvPath).buffer;
+  };
+
+  describe('calculateMD5Checksum', () => {
+    it('should calculate MD5 checksum for file content', () => {
+      const content = loadCsvFixture('sample.csv');
+      const checksum = calculateMD5Checksum(content);
+
+      // This checksum should be consistent for the sample.csv file
+      expect(checksum).toBeTruthy();
+      expect(typeof checksum).toBe('string');
+    });
+
+    it('should return different checksums for different content', () => {
+      const content1 = loadCsvFixture('sample.csv');
+      const content2 = new TextEncoder().encode('Different content').buffer;
+
+      const checksum1 = calculateMD5Checksum(content1);
+      const checksum2 = calculateMD5Checksum(content2);
+
+      expect(checksum1).not.toBe(checksum2);
+    });
+
+    it('should return same checksum for same content', () => {
+      const content1 = loadCsvFixture('sample.csv');
+      const content2 = loadCsvFixture('sample-copy.csv');
+
+      const checksum1 = calculateMD5Checksum(content1);
+      const checksum2 = calculateMD5Checksum(content2);
+
+      expect(checksum1).toBe(checksum2);
+    });
+  });
+
+  describe('calculateFileChecksum', () => {
+    it('should calculate checksum for FileInput object', () => {
+      const csvContent = loadCsvFixture('sample.csv');
+      const file: FileInput = {
+        name: 'sample.csv',
+        size: csvContent.byteLength,
+        content: csvContent,
+      };
+
+      const checksum = calculateFileChecksum(file);
+      expect(checksum).toBeTruthy();
+      expect(typeof checksum).toBe('string');
+    });
+
+    it('should throw error for file without content', () => {
+      const file = {
+        name: 'test.txt',
+        size: 0,
+        content: null,
+      } as unknown as FileInput;
+
+      expect(() => calculateFileChecksum(file)).toThrow(
+        'Unable to calculate checksum for file test.txt: no content available'
+      );
+    });
+  });
+
+  describe('generateFileChecksums', () => {
+    it('should generate checksums for multiple files', () => {
+      const csvContent1 = loadCsvFixture('sample.csv');
+      const csvContent2 = loadCsvFixture('sample-copy.csv');
+
+      const fileObjsRef = {
+        'file1': {
+          name: 'sample.csv',
+          size: csvContent1.byteLength,
+          content: csvContent1,
+        } as FileInput,
+        'file2': {
+          name: 'sample-copy.csv',
+          size: csvContent2.byteLength,
+          content: csvContent2,
+        } as FileInput,
+      };
+
+      const checksums = generateFileChecksums(fileObjsRef);
+
+      expect(checksums).toHaveProperty('file1');
+      expect(checksums).toHaveProperty('file2');
+      // Both files have identical content, so checksums should be the same
+      expect(checksums.file1).toBe(checksums.file2);
+    });
+
+    it('should handle empty file objects', () => {
+      const checksums = generateFileChecksums({});
+      expect(checksums).toEqual({});
+    });
+
+    it('should continue processing when one file fails', () => {
+      const csvContent = loadCsvFixture('sample.csv');
+
+      const fileObjsRef = {
+        'file1': {
+          name: 'sample.csv',
+          size: csvContent.byteLength,
+          content: csvContent,
+        } as FileInput,
+        'file2': {
+          name: 'broken.csv',
+          size: 0,
+          content: null,
+        } as unknown as FileInput,
+      };
+
+      const checksums = generateFileChecksums(fileObjsRef);
+
+      expect(checksums).toHaveProperty('file1');
+      expect(checksums).not.toHaveProperty('file2');
+      expect(typeof checksums.file1).toBe('string');
+    });
+  });
+});

lib/utils/tests/fixtures/sample-copy.csv

@@ -0,0 +1,6 @@
+name,email,age,department
+John Doe,[email protected],30,Engineering
+Jane Smith,[email protected],25,Marketing
+Bob Johnson,[email protected],35,Sales
+Alice Brown,[email protected],28,HR
+Charlie Wilson,[email protected],32,Finance

lib/utils/tests/fixtures/sample.csv

@@ -0,0 +1,6 @@
+name,email,age,department
+John Doe,[email protected],30,Engineering
+Jane Smith,[email protected],25,Marketing
+Bob Johnson,[email protected],35,Sales
+Alice Brown,[email protected],28,HR
+Charlie Wilson,[email protected],32,Finance