perf: flattenedFields collection/global property, remove deep copyi…

…ng in `validateQueryPaths` (#9299)

### What?
Improves querying performance of the Local API, reduces copying overhead

### How?

Adds `flattenedFields` property for sanitized collection/global config
which contains fields in database schema structure.
For example, It removes rows / collapsible / unnamed tabs and merges
them to the parent `fields`, ignores UI fields, named tabs are added as
`type: 'tab'`.
This simplifies code in places like Drizzle `transform/traverseFields`.
Also, now we can avoid calling `flattenTopLevelFields` which adds some
overhead and use `collection.flattenedFields` / `field.flattenedFields`.

By refactoring `configToJSONSchema.ts` with `flattenedFields` it also
1. Fixes #9467
2. Fixes types for UI fields were generated for `select`



Removes this deep copying for each `where` query constraint
https://github.com/payloadcms/payload/blob/58ac784425b411fc28c91dbb3e7df06cc26b6320/packages/payload/src/database/queryValidation/validateQueryPaths.ts#L69-L73
which potentially can add overhead if you have a large collection/global
config

UPD:
The overhead is even much more than in the benchmark below if you have
Lexical with blocks.

Benchmark in `relationships/int.spec.ts`:
```ts
const now = Date.now()
for (let i = 0; i < 10; i++) {
  const now = Date.now()
  for (let i = 0; i < 300; i++) {
    const query = await payload.find({
      collection: 'chained',
      where: {
        'relation.relation.name': {
          equals: 'third',
        },
        and: [
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
          {
            'relation.relation.name': {
              equals: 'third',
            },
          },
        ],
      },
    })
  }

  payload.logger.info(`#${i + 1} ${Date.now() - now}`)
}
payload.logger.info(`Total ${Date.now() - now}`)
```

Before:
```
[02:11:48] INFO: #1 3682
[02:11:50] INFO: #2 2199
[02:11:54] INFO: #3 3483
[02:11:56] INFO: #4 2516
[02:11:59] INFO: #5 2467
[02:12:01] INFO: #6 1987
[02:12:03] INFO: #7 1986
[02:12:05] INFO: #8 2375
[02:12:07] INFO: #9 2040
[02:12:09] INFO: #10 1920
    [PASS] Relationships > Querying > Nested Querying > should allow querying two levels deep (24667ms)
[02:12:09] INFO: Total 24657
```

After:
```
[02:12:36] INFO: #1 2113
[02:12:38] INFO: #2 1854
[02:12:40] INFO: #3 1700
[02:12:42] INFO: #4 1797
[02:12:44] INFO: #5 2121
[02:12:46] INFO: #6 1774
[02:12:47] INFO: #7 1670
[02:12:49] INFO: #8 1610
[02:12:50] INFO: #9 1596
[02:12:52] INFO: #10 1576
    [PASS] Relationships > Querying > Nested Querying > should allow querying two levels deep (17828ms)
[02:12:52] INFO: Total 17818
```

packages/db-mongodb/src/deleteOne.ts

@@ -22,7 +22,7 @@ export const deleteOne: DeleteOne = async function deleteOne(
     ...options,
     projection: buildProjectionFromSelect({
       adapter: this,
-      fields: this.payload.collections[collection].config.fields,
+      fields: this.payload.collections[collection].config.flattenedFields,
       select,
     }),
   }).lean()

packages/db-mongodb/src/find.ts

@@ -42,7 +42,7 @@ export const find: Find = async function find(
   if (!hasNearConstraint) {
     sort = buildSortParam({
       config: this.payload.config,
-      fields: collectionConfig.fields,
+      fields: collectionConfig.flattenedFields,
       locale,
       sort: sortArg || collectionConfig.defaultSort,
       timestamps: true,
@@ -71,7 +71,7 @@ export const find: Find = async function find(
   if (select) {
     paginationOptions.projection = buildProjectionFromSelect({
       adapter: this,
-      fields: collectionConfig.fields,
+      fields: collectionConfig.flattenedFields,
       select,
     })
   }

packages/db-mongodb/src/findGlobal.ts

@@ -18,7 +18,7 @@ export const findGlobal: FindGlobal = async function findGlobal(
     lean: true,
     select: buildProjectionFromSelect({
       adapter: this,
-      fields: this.payload.globals.config.find((each) => each.slug === slug).fields,
+      fields: this.payload.globals.config.find((each) => each.slug === slug).flattenedFields,
       select,
     }),
   }

packages/db-mongodb/src/findGlobalVersions.ts

@@ -29,6 +29,7 @@ export const findGlobalVersions: FindGlobalVersions = async function findGlobalV
   const versionFields = buildVersionGlobalFields(
     this.payload.config,
     this.payload.globals.config.find(({ slug }) => slug === global),
+    true,
   )
   const options = {
     ...(await withSession(this, req)),

packages/db-mongodb/src/findOne.ts

@@ -27,7 +27,7 @@ export const findOne: FindOne = async function findOne(
 
   const projection = buildProjectionFromSelect({
     adapter: this,
-    fields: collectionConfig.fields,
+    fields: collectionConfig.flattenedFields,
     select,
   })
 

packages/db-mongodb/src/findVersions.ts

@@ -44,7 +44,7 @@ export const findVersions: FindVersions = async function findVersions(
   if (!hasNearConstraint) {
     sort = buildSortParam({
       config: this.payload.config,
-      fields: collectionConfig.fields,
+      fields: collectionConfig.flattenedFields,
       locale,
       sort: sortArg || '-updatedAt',
       timestamps: true,
@@ -68,7 +68,7 @@ export const findVersions: FindVersions = async function findVersions(
     pagination,
     projection: buildProjectionFromSelect({
       adapter: this,
-      fields: buildVersionCollectionFields(this.payload.config, collectionConfig),
+      fields: buildVersionCollectionFields(this.payload.config, collectionConfig, true),
       select,
     }),
     sort,

packages/db-mongodb/src/init.ts

@@ -37,7 +37,7 @@ export const init: Init = function init(this: MongooseAdapter) {
       versionSchema.plugin<any, PaginateOptions>(paginate, { useEstimatedCount: true }).plugin(
         getBuildQueryPlugin({
           collectionSlug: collection.slug,
-          versionsFields: versionCollectionFields,
+          versionsFields: buildVersionCollectionFields(this.payload.config, collection, true),
         }),
       )
 
@@ -84,9 +84,11 @@ export const init: Init = function init(this: MongooseAdapter) {
         },
       })
 
-      versionSchema
-        .plugin<any, PaginateOptions>(paginate, { useEstimatedCount: true })
-        .plugin(getBuildQueryPlugin({ versionsFields: versionGlobalFields }))
+      versionSchema.plugin<any, PaginateOptions>(paginate, { useEstimatedCount: true }).plugin(
+        getBuildQueryPlugin({
+          versionsFields: buildVersionGlobalFields(this.payload.config, global, true),
+        }),
+      )
 
       this.versions[global.slug] = mongoose.model(
         versionModelName,

packages/db-mongodb/src/queries/buildAndOrConditions.ts

@@ -1,4 +1,4 @@
-import type { Field, Payload, Where } from 'payload'
+import type { FlattenedField, Payload, Where } from 'payload'
 
 import { parseParams } from './parseParams.js'
 
@@ -11,7 +11,7 @@ export async function buildAndOrConditions({
   where,
 }: {
   collectionSlug?: string
-  fields: Field[]
+  fields: FlattenedField[]
   globalSlug?: string
   locale?: string
   payload: Payload

packages/db-mongodb/src/queries/buildQuery.ts

@@ -1,12 +1,12 @@
-import type { Field, Payload, Where } from 'payload'
+import type { FlattenedField, Payload, Where } from 'payload'
 
 import { QueryError } from 'payload'
 
 import { parseParams } from './parseParams.js'
 
 type GetBuildQueryPluginArgs = {
   collectionSlug?: string
-  versionsFields?: Field[]
+  versionsFields?: FlattenedField[]
 }
 
 export type BuildQueryArgs = {
@@ -34,11 +34,11 @@ export const getBuildQueryPlugin = ({
       if (!fields) {
         if (globalSlug) {
           const globalConfig = payload.globals.config.find(({ slug }) => slug === globalSlug)
-          fields = globalConfig.fields
+          fields = globalConfig.flattenedFields
         }
         if (collectionSlug) {
           const collectionConfig = payload.collections[collectionSlug].config
-          fields = collectionConfig.fields
+          fields = collectionConfig.flattenedFields
         }
       }
       const errors = []

packages/db-mongodb/src/queries/buildSearchParams.ts

@@ -1,4 +1,4 @@
-import type { Field, Operator, PathToQuery, Payload } from 'payload'
+import type { FlattenedField, Operator, PathToQuery, Payload } from 'payload'
 
 import { Types } from 'mongoose'
 import { getLocalizedPaths } from 'payload'
@@ -34,7 +34,7 @@ export async function buildSearchParam({
   val,
 }: {
   collectionSlug?: string
-  fields: Field[]
+  fields: FlattenedField[]
   globalSlug?: string
   incomingPath: string
   locale?: string
@@ -68,11 +68,11 @@ export async function buildSearchParam({
       field: {
         name: 'id',
         type: idFieldType,
-      } as Field,
+      } as FlattenedField,
       path: '_id',
     })
   } else {
-    paths = await getLocalizedPaths({
+    paths = getLocalizedPaths({
       collectionSlug,
       fields,
       globalSlug,

packages/db-mongodb/src/queries/buildSortParam.ts

@@ -1,11 +1,11 @@
 import type { PaginateOptions } from 'mongoose'
-import type { Field, SanitizedConfig, Sort } from 'payload'
+import type { FlattenedField, SanitizedConfig, Sort } from 'payload'
 
 import { getLocalizedSortProperty } from './getLocalizedSortProperty.js'
 
 type Args = {
   config: SanitizedConfig
-  fields: Field[]
+  fields: FlattenedField[]
   locale: string
   sort: Sort
   timestamps: boolean

packages/db-mongodb/src/queries/getLocalizedSortProperty.spec.ts

@@ -1,6 +1,6 @@
 import type { Config, SanitizedConfig } from 'payload'
 
-import { sanitizeConfig } from 'payload'
+import { flattenAllFields, sanitizeConfig } from 'payload'
 
 import { getLocalizedSortProperty } from './getLocalizedSortProperty.js'
 
@@ -69,19 +69,21 @@ describe('get localized sort property', () => {
   it('properly localizes nested sort properties', () => {
     const result = getLocalizedSortProperty({
       config,
-      fields: [
-        {
-          name: 'group',
-          type: 'group',
-          fields: [
-            {
-              name: 'title',
-              type: 'text',
-              localized: true,
-            },
-          ],
-        },
-      ],
+      fields: flattenAllFields({
+        fields: [
+          {
+            name: 'group',
+            type: 'group',
+            fields: [
+              {
+                name: 'title',
+                type: 'text',
+                localized: true,
+              },
+            ],
+          },
+        ],
+      }),
       locale: 'en',
       segments: ['group', 'title'],
     })
@@ -92,19 +94,21 @@ describe('get localized sort property', () => {
   it('keeps requested locale with nested sort properties', () => {
     const result = getLocalizedSortProperty({
       config,
-      fields: [
-        {
-          name: 'group',
-          type: 'group',
-          fields: [
-            {
-              name: 'title',
-              type: 'text',
-              localized: true,
-            },
-          ],
-        },
-      ],
+      fields: flattenAllFields({
+        fields: [
+          {
+            name: 'group',
+            type: 'group',
+            fields: [
+              {
+                name: 'title',
+                type: 'text',
+                localized: true,
+              },
+            ],
+          },
+        ],
+      }),
       locale: 'en',
       segments: ['group', 'title', 'es'],
     })
@@ -115,18 +119,20 @@ describe('get localized sort property', () => {
   it('properly localizes field within row', () => {
     const result = getLocalizedSortProperty({
       config,
-      fields: [
-        {
-          type: 'row',
-          fields: [
-            {
-              name: 'title',
-              type: 'text',
-              localized: true,
-            },
-          ],
-        },
-      ],
+      fields: flattenAllFields({
+        fields: [
+          {
+            type: 'row',
+            fields: [
+              {
+                name: 'title',
+                type: 'text',
+                localized: true,
+              },
+            ],
+          },
+        ],
+      }),
       locale: 'en',
       segments: ['title'],
     })
@@ -137,23 +143,25 @@ describe('get localized sort property', () => {
   it('properly localizes field within named tab', () => {
     const result = getLocalizedSortProperty({
       config,
-      fields: [
-        {
-          type: 'tabs',
-          tabs: [
-            {
-              name: 'tab',
-              fields: [
-                {
-                  name: 'title',
-                  type: 'text',
-                  localized: true,
-                },
-              ],
-            },
-          ],
-        },
-      ],
+      fields: flattenAllFields({
+        fields: [
+          {
+            type: 'tabs',
+            tabs: [
+              {
+                name: 'tab',
+                fields: [
+                  {
+                    name: 'title',
+                    type: 'text',
+                    localized: true,
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }),
       locale: 'en',
       segments: ['tab', 'title'],
     })
@@ -164,23 +172,25 @@ describe('get localized sort property', () => {
   it('properly localizes field within unnamed tab', () => {
     const result = getLocalizedSortProperty({
       config,
-      fields: [
-        {
-          type: 'tabs',
-          tabs: [
-            {
-              fields: [
-                {
-                  name: 'title',
-                  type: 'text',
-                  localized: true,
-                },
-              ],
-              label: 'Tab',
-            },
-          ],
-        },
-      ],
+      fields: flattenAllFields({
+        fields: [
+          {
+            type: 'tabs',
+            tabs: [
+              {
+                fields: [
+                  {
+                    name: 'title',
+                    type: 'text',
+                    localized: true,
+                  },
+                ],
+                label: 'Tab',
+              },
+            ],
+          },
+        ],
+      }),
       locale: 'en',
       segments: ['title'],
     })