Fix issues with SF ContentDocument restrictions (#13)

* Fix issues with SF ContentDocument restrictions

Workaround which fixes issues with SF malformed queries which worked previously. Errors that were occurring:

> Implementation restriction: ContentDocumentLink requires a filter by a single Id on ContentDocumentId or LinkedEntityId using the equals operator or multiple Id's using the IN operator.

> Implementation restriction: filtering on non-id fields is only permitted when filtering by ContentDocumentLink.LinkedEntityId using the equals operator.

* Update tests for salesforce.py

Author: piotrekkr

src/salesforce_archivist/salesforce/salesforce.py

@@ -47,25 +47,37 @@ def _init_tmp_dir(self) -> str:
         return tmp_dir
 
     def _get_content_document_list_query(self) -> str:
-        select_list = ["LinkedEntityId", "ContentDocumentId"]
+        select_list = ["LinkedEntityId", "ContentDocumentId", "LinkedEntity.Type"]
         if self._archivist_obj.dir_name_field is not None and self._archivist_obj.dir_name_field not in select_list:
             select_list.append(self._archivist_obj.dir_name_field)
-        where_list = ["LinkedEntity.Type = '{obj_type}'".format(obj_type=self._archivist_obj.obj_type)]
+        where_conditions = []
         if self._archivist_obj.modified_date_lt is not None:
-            where_list.append(
+            where_conditions.append(
                 "ContentDocument.ContentModifiedDate < {date}".format(
                     date=self._archivist_obj.modified_date_lt.strftime("%Y-%m-%dT%H:%M:%SZ")
                 )
             )
         if self._archivist_obj.modified_date_gt is not None:
-            where_list.append(
+            where_conditions.append(
                 "ContentDocument.ContentModifiedDate > {date}".format(
                     date=self._archivist_obj.modified_date_gt.strftime("%Y-%m-%dT%H:%M:%SZ")
                 )
             )
-        return "SELECT {fields} FROM ContentDocumentLink WHERE {where}".format(
-            fields=", ".join(select_list), where=" AND ".join(where_list)
-        )
+        where = ""
+        if len(where_conditions):
+            where = "WHERE {}".format(" AND ".join(where_conditions))
+        # Using WHERE IN and not using filter on `LinkedEntity.Type` is done because of SF restrictions like:
+        #
+        #   Implementation restriction: ContentDocumentLink requires a filter by a single Id on ContentDocumentId
+        #   or LinkedEntityId using the equals operator or multiple Id's using the IN operator.
+        #
+        #   Implementation restriction: filtering on non-id fields is only permitted when filtering
+        #   by ContentDocumentLink.LinkedEntityId using the equals operator.
+
+        return (
+            "SELECT {fields} FROM ContentDocumentLink "
+            "WHERE ContentDocumentId IN (SELECT Id FROM ContentDocument {where})"
+        ).format(fields=", ".join(select_list), where=where)
 
     def download_content_document_link_list(
         self,
@@ -81,10 +93,14 @@ def download_content_document_link_list(
                 reader = csv.reader(file)
                 next(reader)
                 for row in reader:
+                    # If type is not the same as the object type, skip.
+                    # This is a workaround for restriction on ContentDocumentLink filtering directly in query.
+                    if row[2] != self._archivist_obj.obj_type:
+                        continue
                     link = ContentDocumentLink(
                         linked_entity_id=row[0],
                         content_document_id=row[1],
-                        download_dir_name=row[2] if self._archivist_obj.dir_name_field is not None else None,
+                        download_dir_name=row[3] if self._archivist_obj.dir_name_field is not None else None,
                     )
                     document_link_list.add_link(link)
 

test/salesforce/test_salesforce.py

@@ -22,7 +22,7 @@
             None,
             None,
             None,
-            "SELECT LinkedEntityId, ContentDocumentId FROM ContentDocumentLink WHERE LinkedEntity.Type = 'User'",
+            "SELECT LinkedEntityId, ContentDocumentId, LinkedEntity.Type FROM ContentDocumentLink WHERE ContentDocumentId IN (SELECT Id FROM ContentDocument )",
         ),
         (
             datetime(
@@ -38,10 +38,9 @@
             None,
             None,
             (
-                "SELECT LinkedEntityId, ContentDocumentId "
+                "SELECT LinkedEntityId, ContentDocumentId, LinkedEntity.Type "
                 "FROM ContentDocumentLink "
-                "WHERE LinkedEntity.Type = 'User' "
-                "AND ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z"
+                "WHERE ContentDocumentId IN (SELECT Id FROM ContentDocument WHERE ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z)"
             ),
         ),
         (
@@ -67,11 +66,11 @@
             ),
             None,
             (
-                "SELECT LinkedEntityId, ContentDocumentId "
+                "SELECT LinkedEntityId, ContentDocumentId, LinkedEntity.Type "
                 "FROM ContentDocumentLink "
-                "WHERE LinkedEntity.Type = 'User' "
-                "AND ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z "
-                "AND ContentDocument.ContentModifiedDate > 2023-01-01T00:00:00Z"
+                "WHERE ContentDocumentId IN ("
+                "SELECT Id FROM ContentDocument WHERE ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z AND ContentDocument.ContentModifiedDate > 2023-01-01T00:00:00Z"
+                ")"
             ),
         ),
         (
@@ -97,11 +96,11 @@
             ),
             "DirField",
             (
-                "SELECT LinkedEntityId, ContentDocumentId, DirField "
+                "SELECT LinkedEntityId, ContentDocumentId, LinkedEntity.Type, DirField "
                 "FROM ContentDocumentLink "
-                "WHERE LinkedEntity.Type = 'User' "
-                "AND ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z "
-                "AND ContentDocument.ContentModifiedDate > 2023-01-01T00:00:00Z"
+                "WHERE ContentDocumentId IN ("
+                "SELECT Id FROM ContentDocument WHERE ContentDocument.ContentModifiedDate < 2024-01-01T00:00:00Z AND ContentDocument.ContentModifiedDate > 2023-01-01T00:00:00Z"
+                ")"
             ),
         ),
     ],
@@ -141,35 +140,35 @@ def test_download_content_document_link_list_queries(
         [],
         # no results from query (file with only header)
         [
-            [["LinkedEntityId", "ContentDocumentId"]],
+            [["LinkedEntityId", "ContentDocumentId", "Type"]],
         ],
         # results without custom field for dir name
         [
             [
-                ["LinkedEntityId", "ContentDocumentId"],
-                ["LinkedEntityId_1", "ContentDocumentId_1"],
-                ["LinkedEntityId_2", "ContentDocumentId_2"],
+                ["LinkedEntityId", "ContentDocumentId", "User"],
+                ["LinkedEntityId_1", "ContentDocumentId_1", "User"],
+                ["LinkedEntityId_2", "ContentDocumentId_2", "User"],
             ]
         ],
         # results with custom field for dir name
         [
             [
-                ["LinkedEntityId", "ContentDocumentId", "CustomFieldForDirName"],
-                ["LinkedEntityId_1", "ContentDocumentId_1", "CustomFieldForDirName_1"],
-                ["LinkedEntityId_2", "ContentDocumentId_2", "CustomFieldForDirName_2"],
+                ["LinkedEntityId", "ContentDocumentId", "User", "CustomFieldForDirName"],
+                ["LinkedEntityId_1", "ContentDocumentId_1", "User", "CustomFieldForDirName_1"],
+                ["LinkedEntityId_2", "ContentDocumentId_2", "User", "CustomFieldForDirName_2"],
             ]
         ],
         # results with custom field for dir name in multiple csv files
         [
             [
-                ["LinkedEntityId", "ContentDocumentId", "CustomFieldForDirName"],
-                ["LinkedEntityId_1", "ContentDocumentId_1", "CustomFieldForDirName_1"],
-                ["LinkedEntityId_2", "ContentDocumentId_2", "CustomFieldForDirName_2"],
+                ["LinkedEntityId", "ContentDocumentId", "User", "CustomFieldForDirName"],
+                ["LinkedEntityId_1", "ContentDocumentId_1", "User", "CustomFieldForDirName_1"],
+                ["LinkedEntityId_2", "ContentDocumentId_2", "User", "CustomFieldForDirName_2"],
             ],
             [
-                ["LinkedEntityId", "ContentDocumentId", "CustomFieldForDirName"],
-                ["LinkedEntityId_3", "ContentDocumentId_3", "CustomFieldForDirName_3"],
-                ["LinkedEntityId_4", "ContentDocumentId_4", "CustomFieldForDirName_4"],
+                ["LinkedEntityId", "ContentDocumentId", "User", "CustomFieldForDirName"],
+                ["LinkedEntityId_3", "ContentDocumentId_3", "User", "CustomFieldForDirName_3"],
+                ["LinkedEntityId_4", "ContentDocumentId_4", "User", "CustomFieldForDirName_4"],
             ],
         ],
     ],
@@ -182,7 +181,7 @@ def test_download_content_document_link_list_csv_reading(
         archivist_obj = ArchivistObject(
             data_dir=tmp_dir,
             obj_type="User",
-            dir_name_field=(csv_files_data[0][0][2] if len(csv_files_data) and len(csv_files_data[0][0]) > 2 else None),
+            dir_name_field=(csv_files_data[0][0][3] if len(csv_files_data) and len(csv_files_data[0][0]) > 3 else None),
         )
         client.bulk2 = Mock(
             side_effect=lambda *args, **kwargs: gen_temp_csv_files(
@@ -196,7 +195,7 @@ def test_download_content_document_link_list_csv_reading(
                 doc_link = ContentDocumentLink(
                     linked_entity_id=row[0],
                     content_document_id=row[1],
-                    download_dir_name=row[2] if len(row) > 2 else row[0],
+                    download_dir_name=row[3] if len(row) > 3 else row[0],
                 )
                 add_link_calls.append(call(doc_link))