fix(db-mongodb): improve compatability with Firestore database

.github/workflows/main.yml

@@ -168,6 +168,7 @@ jobs:
       matrix:
         database:
           - mongodb
+          - firestore
           - postgres
           - postgres-custom-schema
           - postgres-uuid

docs/database/mongodb.mdx

@@ -30,17 +30,21 @@ export default buildConfig({
 
 ## Options
 
-| Option                     | Description                                                                                                                                                                                                                                                                                                  |
-| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `autoPluralization`        | Tell Mongoose to auto-pluralize any collection names if it encounters any singular words used as collection `slug`s.                                                                                                                                                                                         |
-| `connectOptions`           | Customize MongoDB connection options. Payload will connect to your MongoDB database using default options which you can override and extend to include all the [options](https://mongoosejs.com/docs/connections.html#options) available to mongoose.                                                        |
-| `collectionsSchemaOptions` | Customize Mongoose schema options for collections.                                                                                                                                                                                                                                                           |
-| `disableIndexHints`        | Set to true to disable hinting to MongoDB to use 'id' as index. This is currently done when counting documents for pagination, as it increases the speed of the count function used in that query. Disabling this optimization might fix some problems with AWS DocumentDB. Defaults to false                |
-| `migrationDir`             | Customize the directory that migrations are stored.                                                                                                                                                                                                                                                          |
-| `transactionOptions`       | An object with configuration properties used in [transactions](https://www.mongodb.com/docs/manual/core/transactions/) or `false` which will disable the use of transactions.                                                                                                                                |
-| `collation`                | Enable language-specific string comparison with customizable options. Available on MongoDB 3.4+. Defaults locale to "en". Example: `{ strength: 3 }`. For a full list of collation options and their definitions, see the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/collation/). |
-| `allowAdditionalKeys`      | By default, Payload strips all additional keys from MongoDB data that don't exist in the Payload schema. If you have some data that you want to include to the result but it doesn't exist in Payload, you can set this to `true`. Be careful as Payload access control _won't_ work for this data.          |
-| `allowIDOnCreate`          | Set to `true` to use the `id` passed in data on the create API operations without using a custom ID field.                                                                                                                                                                                                   |
+| Option                       | Description                                                                                                                                                                                                                                                                                                  |
+| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `autoPluralization`          | Tell Mongoose to auto-pluralize any collection names if it encounters any singular words used as collection `slug`s.                                                                                                                                                                                         |
+| `connectOptions`             | Customize MongoDB connection options. Payload will connect to your MongoDB database using default options which you can override and extend to include all the [options](https://mongoosejs.com/docs/connections.html#options) available to mongoose.                                                        |
+| `collectionsSchemaOptions`   | Customize Mongoose schema options for collections.                                                                                                                                                                                                                                                           |
+| `disableIndexHints`          | Set to true to disable hinting to MongoDB to use 'id' as index. This is currently done when counting documents for pagination, as it increases the speed of the count function used in that query. Disabling this optimization might fix some problems with AWS DocumentDB. Defaults to false                |
+| `migrationDir`               | Customize the directory that migrations are stored.                                                                                                                                                                                                                                                          |
+| `transactionOptions`         | An object with configuration properties used in [transactions](https://www.mongodb.com/docs/manual/core/transactions/) or `false` which will disable the use of transactions.                                                                                                                                |
+| `collation`                  | Enable language-specific string comparison with customizable options. Available on MongoDB 3.4+. Defaults locale to "en". Example: `{ strength: 3 }`. For a full list of collation options and their definitions, see the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/collation/). |
+| `allowAdditionalKeys`        | By default, Payload strips all additional keys from MongoDB data that don't exist in the Payload schema. If you have some data that you want to include to the result but it doesn't exist in Payload, you can set this to `true`. Be careful as Payload access control _won't_ work for this data.          |
+| `allowIDOnCreate`            | Set to `true` to use the `id` passed in data on the create API operations without using a custom ID field.                                                                                                                                                                                                   |
+| `useAlternativeDropDatabase` | Set to `true` to use an alternative `dropDatabase` implementation that calls `collection.deleteMany({})` on every collection instead of sending a raw `dropDatabase` command. Payload only uses `dropDatabase` for testing purposes. Defaults to `false`.                                                    |
+| `useBigIntForNumberIDs`      | Set to `true` to use `BigInt` for custom ID fields of type `'number'`. Useful for databases that don't support `double` or `int32` IDs. Defaults to `false`.                                                                                                                                                 |
+| `useJoinAggregations`        | Set to `false` to disable join aggregations (which use correlated subqueries) and instead populate join fields via multiple `find` queries. Defaults to `true`.                                                                                                                                              |
+| `usePipelineInSortLookup`    | Set to `false` to disable the use of `pipeline` in the `$lookup` aggregation in sorting. Defaults to `true`.                                                                                                                                                                                                 |
 
 ## Access to Mongoose models
 
@@ -55,9 +59,21 @@ You can access Mongoose models as follows:
 
 ## Using other MongoDB implementations
 
-Limitations with [DocumentDB](https://aws.amazon.com/documentdb/) and [Azure Cosmos DB](https://azure.microsoft.com/en-us/products/cosmos-db):
+You can import the `compatabilityOptions` object to get the recommended settings for other MongoDB implementations. Since these databases aren't officially supported by payload, you may still encounter issues even with these settings (please create an issue or PR if you believe these options should be updated):
 
-- For Azure Cosmos DB you must pass `transactionOptions: false` to the adapter options. Azure Cosmos DB does not support transactions that update two and more documents in different collections, which is a common case when using Payload (via hooks).
-- For Azure Cosmos DB the root config property `indexSortableFields` must be set to `true`.
-- The [Join Field](../fields/join) is not supported in DocumentDB and Azure Cosmos DB, as we internally use MongoDB aggregations to query data for that field, which are limited there. This can be changed in the future.
-- For DocumentDB pass `disableIndexHints: true` to disable hinting to the DB to use `id` as index which can cause problems with DocumentDB.
+```ts
+import { mongooseAdapter, compatabilityOptions } from '@payloadcms/db-mongodb'
+
+export default buildConfig({
+  db: mongooseAdapter({
+    url: process.env.DATABASE_URI,
+    // For example, if you're using firestore:
+    ...compatabilityOptions.firestore,
+  }),
+})
+```
+
+We export compatability options for [DocumentDB](https://aws.amazon.com/documentdb/), [Azure Cosmos DB](https://azure.microsoft.com/en-us/products/cosmos-db) and [Firestore](https://cloud.google.com/firestore/mongodb-compatibility/docs/overview). Known limitations:
+
+- Azure Cosmos DB does not support transactions that update two or more documents in different collections, which is a common case when using Payload (via hooks).
+- Azure Cosmos DB the root config property `indexSortableFields` must be set to `true`.

package.json

@@ -107,6 +107,7 @@
     "test:e2e:prod:ci": "pnpm prepare-run-test-against-prod:ci && pnpm runts ./test/runE2E.ts --prod",
     "test:e2e:prod:ci:noturbo": "pnpm prepare-run-test-against-prod:ci && pnpm runts ./test/runE2E.ts --prod --no-turbo",
     "test:int": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 DISABLE_LOGGING=true jest --forceExit --detectOpenHandles --config=test/jest.config.js --runInBand",
+    "test:int:firestore": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PAYLOAD_DATABASE=firestore DISABLE_LOGGING=true jest --forceExit --detectOpenHandles --config=test/jest.config.js --runInBand",
     "test:int:postgres": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PAYLOAD_DATABASE=postgres DISABLE_LOGGING=true jest --forceExit --detectOpenHandles --config=test/jest.config.js --runInBand",
     "test:int:sqlite": "cross-env NODE_OPTIONS=\"--no-deprecation --no-experimental-strip-types\" NODE_NO_WARNINGS=1 PAYLOAD_DATABASE=sqlite DISABLE_LOGGING=true jest --forceExit --detectOpenHandles --config=test/jest.config.js --runInBand",
     "test:types": "tstyche",

packages/db-mongodb/src/connect.ts

@@ -36,6 +36,25 @@ export const connect: Connect = async function connect(
 
   try {
     this.connection = (await mongoose.connect(urlToConnect, connectionOptions)).connection
+    if (this.useAlternativeDropDatabase) {
+      if (this.connection.db) {
+        // Firestore doesn't support dropDatabase, so we monkey patch
+        // dropDatabase to delete all documents from all collections instead
+        this.connection.db.dropDatabase = async function (): Promise<boolean> {
+          const existingCollections = await this.listCollections().toArray()
+          await Promise.all(
+            existingCollections.map(async (collectionInfo) => {
+              const collection = this.collection(collectionInfo.name)
+              await collection.deleteMany({})
+            }),
+          )
+          return true
+        }
+        this.connection.dropDatabase = async function () {
+          await this.db?.dropDatabase()
+        }
+      }
+    }
 
     // If we are running a replica set with MongoDB Memory Server,
     // wait until the replica set elects a primary before proceeding

packages/db-mongodb/src/find.ts

@@ -12,6 +12,7 @@ import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
 import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { getCollection } from './utilities/getEntity.js'
 import { getSession } from './utilities/getSession.js'
+import { resolveJoins } from './utilities/resolveJoins.js'
 import { transform } from './utilities/transform.js'
 
 export const find: Find = async function find(
@@ -155,6 +156,16 @@ export const find: Find = async function find(
     result = await Model.paginate(query, paginationOptions)
   }
 
+  if (!this.useJoinAggregations) {
+    await resolveJoins({
+      adapter: this,
+      collectionSlug,
+      docs: result.docs as Record<string, unknown>[],
+      joins,
+      locale,
+    })
+  }
+
   transform({
     adapter: this,
     data: result.docs,

packages/db-mongodb/src/findOne.ts

@@ -10,6 +10,7 @@ import { buildJoinAggregation } from './utilities/buildJoinAggregation.js'
 import { buildProjectionFromSelect } from './utilities/buildProjectionFromSelect.js'
 import { getCollection } from './utilities/getEntity.js'
 import { getSession } from './utilities/getSession.js'
+import { resolveJoins } from './utilities/resolveJoins.js'
 import { transform } from './utilities/transform.js'
 
 export const findOne: FindOne = async function findOne(
@@ -67,6 +68,16 @@ export const findOne: FindOne = async function findOne(
     doc = await Model.findOne(query, {}, options)
   }
 
+  if (doc && !this.useJoinAggregations) {
+    await resolveJoins({
+      adapter: this,
+      collectionSlug,
+      docs: [doc] as Record<string, unknown>[],
+      joins,
+      locale,
+    })
+  }
+
   if (!doc) {
     return null
   }

packages/db-mongodb/src/index.ts

@@ -135,6 +135,29 @@ export interface Args {
 
   /** The URL to connect to MongoDB or false to start payload and prevent connecting */
   url: false | string
+
+  /**
+   * Set to `true` to use an alternative `dropDatabase` implementation that calls `collection.deleteMany({})` on every collection instead of sending a raw `dropDatabase` command.
+   * Payload only uses `dropDatabase` for testing purposes.
+   * @default false
+   */
+  useAlternativeDropDatabase?: boolean
+  /**
+   * Set to `true` to use `BigInt` for custom ID fields of type `'number'`.
+   * Useful for databases that don't support `double` or `int32` IDs.
+   * @default false
+   */
+  useBigIntForNumberIDs?: boolean
+  /**
+   * Set to `false` to disable join aggregations (which use correlated subqueries) and instead populate join fields via multiple `find` queries.
+   * @default true
+   */
+  useJoinAggregations?: boolean
+  /**
+   * Set to `false` to disable the use of `pipeline` in the `$lookup` aggregation in sorting.
+   * @default true
+   */
+  usePipelineInSortLookup?: boolean
 }
 
 export type MongooseAdapter = {
@@ -151,6 +174,10 @@ export type MongooseAdapter = {
     up: (args: MigrateUpArgs) => Promise<void>
   }[]
   sessions: Record<number | string, ClientSession>
+  useAlternativeDropDatabase: boolean
+  useBigIntForNumberIDs: boolean
+  useJoinAggregations: boolean
+  usePipelineInSortLookup: boolean
   versions: {
     [slug: string]: CollectionModel
   }
@@ -186,6 +213,10 @@ declare module 'payload' {
     updateVersion: <T extends TypeWithID = TypeWithID>(
       args: { options?: QueryOptions } & UpdateVersionArgs<T>,
     ) => Promise<TypeWithVersion<T>>
+    useAlternativeDropDatabase: boolean
+    useBigIntForNumberIDs: boolean
+    useJoinAggregations: boolean
+    usePipelineInSortLookup: boolean
     versions: {
       [slug: string]: CollectionModel
     }
@@ -205,6 +236,10 @@ export function mongooseAdapter({
   prodMigrations,
   transactionOptions = {},
   url,
+  useAlternativeDropDatabase = false,
+  useBigIntForNumberIDs = false,
+  useJoinAggregations = true,
+  usePipelineInSortLookup = true,
 }: Args): DatabaseAdapterObj {
   function adapter({ payload }: { payload: Payload }) {
     const migrationDir = findMigrationDir(migrationDirArg)
@@ -269,6 +304,10 @@ export function mongooseAdapter({
       updateOne,
       updateVersion,
       upsert,
+      useAlternativeDropDatabase,
+      useBigIntForNumberIDs,
+      useJoinAggregations,
+      usePipelineInSortLookup,
     })
   }
 
@@ -280,6 +319,8 @@ export function mongooseAdapter({
   }
 }
 
+export { compatabilityOptions } from './utilities/compatabilityOptions.js'
+
 /**
  * Attempt to find migrations directory.
  *

packages/db-mongodb/src/models/buildSchema.ts

@@ -143,7 +143,12 @@ export const buildSchema = (args: {
     const idField = schemaFields.find((field) => fieldAffectsData(field) && field.name === 'id')
     if (idField) {
       fields = {
-        _id: idField.type === 'number' ? Number : String,
+        _id:
+          idField.type === 'number'
+            ? payload.db.useBigIntForNumberIDs
+              ? mongoose.Schema.Types.BigInt
+              : Number
+            : String,
       }
       schemaFields = schemaFields.filter(
         (field) => !(fieldAffectsData(field) && field.name === 'id'),
@@ -900,7 +905,11 @@ const getRelationshipValueType = (field: RelationshipField | UploadField, payloa
     }
 
     if (customIDType === 'number') {
-      return mongoose.Schema.Types.Number
+      if (payload.db.useBigIntForNumberIDs) {
+        return mongoose.Schema.Types.BigInt
+      } else {
+        return mongoose.Schema.Types.Number
+      }
     }
 
     return mongoose.Schema.Types.String