Fallback to approx create time check to catch replicated ttl removes.

Author: petermyers

src/from/dynamodb.js

@@ -165,14 +165,23 @@ export const outGlobalTableExtraModify = (record) => {
 //--------------------------------------------
 
 export const outTtlExpiredEvents = (ignoreTtlExpiredEvents) => (record) => {
-  const { eventName, userIdentity } = record;
+  const { eventName, userIdentity, dynamodb: { OldImage, ApproximateCreationDateTime } } = record;
   // this is not a REMOVE event or we're not ignoring the ttl expired events anyway.
   if (eventName !== 'REMOVE' || !ignoreTtlExpiredEvents) return true;
 
-  // See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_Record.html
-  // We trust dynamodb that the ttl expired if its a remove and has the ttl expiry indicating
-  // identity attributes.
-  return !(userIdentity?.type === 'Service' && userIdentity?.principalId === 'dynamodb.amazonaws.com');
+  if (userIdentity) {
+    // See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_Record.html
+    // We trust dynamodb that the ttl expired if its a remove and has the ttl expiry indicating
+    // identity attributes.
+    return !(userIdentity?.type === 'Service' && userIdentity?.principalId === 'dynamodb.amazonaws.com');
+  } else if (OldImage.ttl?.N) {
+    // If no user identity attribute is present, this may be a replicated TTL delete, but we still
+    // want to honor it because filtering out replica region events may be disabled.
+    const ttlSec = Number(OldImage.ttl.N);
+    return !(ttlSec <= ApproximateCreationDateTime);
+  } else {
+    return true;
+  }
 };
 
 // test helper
@@ -185,6 +194,9 @@ export const toDynamodbRecords = (events, { removeUndefinedValues = true } = {})
       eventSource: 'aws:dynamodb',
       awsRegion: e.newImage?.awsregion || process.env.AWS_REGION || /* istanbul ignore next */ 'us-west-2',
       dynamodb: {
+        // TODO - Fix this in next major version bump. ApproximateCreationDateTime is meant to be in seconds, not millis.
+        // Didn't want to fix until a major version bump to avoid compatibility issues for consumers
+        // upgrading the lib. This should be e.timestamp / 1000
         ApproximateCreationDateTime: e.timestamp,
         Keys: e.keys ? marshall(e.keys, { removeUndefinedValues }) : /* istanbul ignore next */ undefined,
         NewImage: e.newImage ? marshall(e.newImage, { removeUndefinedValues }) : undefined,

test/unit/from/dynamodb.test.js

@@ -946,6 +946,47 @@ describe('from/dynamodb.js', () => {
       .done(done);
   });
 
+  it('should ignore replicated ttl', (done) => {
+    const events = toDynamodbRecords([
+      {
+        timestamp: 1573005491,
+        keys: {
+          pk: '1',
+          sk: 'thing',
+        },
+        oldImage: {
+          pk: '1',
+          sk: 'thing',
+          name: 'N1',
+          ttl: 1573005490,
+          timestamp: 1573005490000,
+        },
+      },
+      {
+        timestamp: 1573005490,
+        keys: {
+          pk: '1',
+          sk: 'thing',
+        },
+        oldImage: {
+          pk: '1',
+          sk: 'thing',
+          name: 'N1',
+          ttl: 1573015490, // expired, has no identity attributes to indicate ttl delete
+          timestamp: 1573005490000,
+        },
+      },
+    ]);
+
+    fromDynamodb(events, { ignoreTtlExpiredEvents: true })
+      .collect()
+      .tap((collected) => {
+        // console.log(JSON.stringify(collected, null, 2));
+        expect(collected.length).to.equal(1);
+      })
+      .done(done);
+  });
+
   it('should keep replica records if ignoreReplicas is false', (done) => {
     const events = toDynamodbRecords([
       {