diff --git a/spec/GridFSBucketStorageAdapter.spec.js b/spec/GridFSBucketStorageAdapter.spec.js index 6a274125bc..033292063c 100644 --- a/spec/GridFSBucketStorageAdapter.spec.js +++ b/spec/GridFSBucketStorageAdapter.spec.js @@ -29,6 +29,7 @@ describe_only_db('mongo')('GridFSBucket', () => { enableSchemaHooks: true, schemaCacheTtl: 5000, maxTimeMS: 30000, + batchSize: 500, disableIndexFieldValidation: true, logClientEvents: [{ name: 'commandStarted' }], createIndexUserUsername: true, @@ -46,6 +47,13 @@ describe_only_db('mongo')('GridFSBucket', () => { expect(db.options?.retryWrites).toEqual(true); }); + it('should store batchSize and filter it from MongoClient options', async () => { + const gfsAdapter = new GridFSBucketAdapter(databaseURI, { batchSize: 500 }); + expect(gfsAdapter._batchSize).toEqual(500); + // Verify batchSize is filtered from MongoClient options + expect(gfsAdapter._mongoOptions.batchSize).toBeUndefined(); + }); + it('should save an encrypted file that can only be decrypted by a GridFS adapter with the encryptionKey', async () => { const unencryptedAdapter = new GridFSBucketAdapter(databaseURI); const encryptedAdapter = new GridFSBucketAdapter( diff --git a/spec/MongoStorageAdapter.spec.js b/spec/MongoStorageAdapter.spec.js index 1deaa169d8..ffaaf94c98 100644 --- a/spec/MongoStorageAdapter.spec.js +++ b/spec/MongoStorageAdapter.spec.js @@ -108,6 +108,58 @@ describe_only_db('mongo')('MongoStorageAdapter', () => { ); }); + it('passes batchSize to the MongoDB driver find() call', async () => { + const batchSize = 50; + const adapter = new MongoStorageAdapter({ + uri: databaseURI, + mongoOptions: { batchSize }, + }); + await adapter.createObject('BatchTest', { fields: {} }, { objectId: 'obj1' }); + + // Spy on the MongoDB driver's Collection.prototype.find to verify batchSize is forwarded + const originalFind = Collection.prototype.find; + let capturedOptions; + spyOn(Collection.prototype, 'find').and.callFake(function (query, options) { + capturedOptions = options; + return originalFind.call(this, query, options); + }); + + await adapter.find('BatchTest', { fields: {} }, {}, {}); + expect(capturedOptions).toBeDefined(); + expect(capturedOptions.batchSize).toEqual(50); + }); + + it('passes batchSize to the MongoDB driver aggregate() call', async () => { + const batchSize = 50; + const adapter = new MongoStorageAdapter({ + uri: databaseURI, + mongoOptions: { batchSize }, + }); + await adapter.createObject('AggBatchTest', { fields: { count: { type: 'Number' } } }, { objectId: 'obj1', count: 1 }); + + // Spy on the MongoDB driver's Collection.prototype.aggregate to verify batchSize is forwarded + const originalAggregate = Collection.prototype.aggregate; + let capturedOptions; + spyOn(Collection.prototype, 'aggregate').and.callFake(function (pipeline, options) { + capturedOptions = options; + return originalAggregate.call(this, pipeline, options); + }); + + await adapter.aggregate('AggBatchTest', { fields: { count: { type: 'Number' } } }, [{ $match: {} }]); + expect(capturedOptions).toBeDefined(); + expect(capturedOptions.batchSize).toEqual(50); + }); + + it('defaults batchSize to 1000', async () => { + await reconfigureServer({ + databaseURI: databaseURI, + collectionPrefix: 'test_', + databaseAdapter: undefined, + }); + const adapter = Config.get(Parse.applicationId).database.adapter; + expect(adapter._batchSize).toEqual(1000); + }); + it('stores pointers with a _p_ prefix', done => { const obj = { objectId: 'bar', diff --git a/src/Adapters/Files/GridFSBucketAdapter.js b/src/Adapters/Files/GridFSBucketAdapter.js index 6157b8e7b2..0236bec219 100644 --- a/src/Adapters/Files/GridFSBucketAdapter.js +++ b/src/Adapters/Files/GridFSBucketAdapter.js @@ -38,6 +38,7 @@ export class GridFSBucketAdapter extends FilesAdapter { const defaultMongoOptions = {}; const _mongoOptions = Object.assign(defaultMongoOptions, mongoOptions); this._clientMetadata = mongoOptions.clientMetadata; + this._batchSize = mongoOptions.batchSize; // Remove Parse Server-specific options that should not be passed to MongoDB client for (const key of ParseServerDatabaseOptions) { delete _mongoOptions[key]; @@ -135,7 +136,7 @@ export class GridFSBucketAdapter extends FilesAdapter { async deleteFile(filename: string) { const bucket = await this._getBucket(); - const documents = await bucket.find({ filename }).toArray(); + const documents = await bucket.find({ filename }, { batchSize: this._batchSize }).toArray(); if (documents.length === 0) { throw new Error('FileNotFound'); } @@ -196,7 +197,7 @@ export class GridFSBucketAdapter extends FilesAdapter { if (options.fileNames !== undefined) { fileNames = options.fileNames; } else { - const fileNamesIterator = await bucket.find().toArray(); + const fileNamesIterator = await bucket.find({}, { batchSize: this._batchSize }).toArray(); fileNamesIterator.forEach(file => { fileNames.push(file.filename); }); @@ -226,7 +227,7 @@ export class GridFSBucketAdapter extends FilesAdapter { async getMetadata(filename) { const bucket = await this._getBucket(); - const files = await bucket.find({ filename }).toArray(); + const files = await bucket.find({ filename }, { batchSize: this._batchSize }).toArray(); if (files.length === 0) { return {}; } @@ -236,7 +237,7 @@ export class GridFSBucketAdapter extends FilesAdapter { async handleFileStream(filename: string, req, res, contentType) { const bucket = await this._getBucket(); - const files = await bucket.find({ filename }).toArray(); + const files = await bucket.find({ filename }, { batchSize: this._batchSize }).toArray(); if (files.length === 0) { throw new Error('FileNotFound'); } diff --git a/src/Adapters/Storage/Mongo/MongoCollection.js b/src/Adapters/Storage/Mongo/MongoCollection.js index 4f02c5c8fa..04f8ca1dc2 100644 --- a/src/Adapters/Storage/Mongo/MongoCollection.js +++ b/src/Adapters/Storage/Mongo/MongoCollection.js @@ -21,6 +21,7 @@ export default class MongoCollection { sort, keys, maxTimeMS, + batchSize, readPreference, hint, caseInsensitive, @@ -39,6 +40,7 @@ export default class MongoCollection { sort, keys, maxTimeMS, + batchSize, readPreference, hint, caseInsensitive, @@ -68,6 +70,7 @@ export default class MongoCollection { sort, keys, maxTimeMS, + batchSize, readPreference, hint, caseInsensitive, @@ -94,6 +97,7 @@ export default class MongoCollection { sort, keys, maxTimeMS, + batchSize, readPreference, hint, caseInsensitive, @@ -108,6 +112,7 @@ export default class MongoCollection { readPreference, hint, comment, + batchSize, }); if (keys) { @@ -153,9 +158,9 @@ export default class MongoCollection { return this._mongoCollection.distinct(field, query); } - aggregate(pipeline, { maxTimeMS, readPreference, hint, explain, comment } = {}) { + aggregate(pipeline, { maxTimeMS, batchSize, readPreference, hint, explain, comment } = {}) { return this._mongoCollection - .aggregate(pipeline, { maxTimeMS, readPreference, hint, explain, comment }) + .aggregate(pipeline, { maxTimeMS, batchSize, readPreference, hint, explain, comment }) .toArray(); } diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 59101bba72..9d8ef47941 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -170,6 +170,7 @@ export class MongoStorageAdapter implements StorageAdapter { database: any; client: MongoClient; _maxTimeMS: ?number; + _batchSize: ?number; canSortOnJoinTables: boolean; enableSchemaHooks: boolean; schemaCacheTtl: ?number; @@ -182,6 +183,8 @@ export class MongoStorageAdapter implements StorageAdapter { // MaxTimeMS is not a global MongoDB client option, it is applied per operation. this._maxTimeMS = mongoOptions.maxTimeMS; + // BatchSize is not a global MongoDB client option, it is applied per cursor operation. + this._batchSize = mongoOptions.batchSize; this.canSortOnJoinTables = true; this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks; this.schemaCacheTtl = mongoOptions.schemaCacheTtl; @@ -735,6 +738,7 @@ export class MongoStorageAdapter implements StorageAdapter { sort: mongoSort, keys: mongoKeys, maxTimeMS: this._maxTimeMS, + batchSize: this._batchSize, readPreference, hint, caseInsensitive, @@ -820,6 +824,7 @@ export class MongoStorageAdapter implements StorageAdapter { .then(collection => collection.find(query, { maxTimeMS: this._maxTimeMS, + batchSize: this._batchSize, }) ) .catch(err => this.handleError(err)); @@ -909,6 +914,7 @@ export class MongoStorageAdapter implements StorageAdapter { collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS, + batchSize: this._batchSize, hint, explain, comment, diff --git a/src/Options/Definitions.js b/src/Options/Definitions.js index 9715af12de..323cd19a9b 100644 --- a/src/Options/Definitions.js +++ b/src/Options/Definitions.js @@ -1097,6 +1097,12 @@ module.exports.DatabaseOptions = { help: 'The MongoDB driver option to specify the amount of time in milliseconds to wait for a connection attempt to finish before trying the next address when using the autoSelectFamily option. If set to a positive integer less than 10, the value 10 is used instead.', action: parsers.numberParser('autoSelectFamilyAttemptTimeout'), }, + batchSize: { + env: 'PARSE_SERVER_DATABASE_BATCH_SIZE', + help: 'The number of documents per batch for MongoDB cursor `getMore` operations. A lower value reduces memory usage per batch; a higher value reduces the number of network round-trips.', + action: parsers.numberParser('batchSize'), + default: 1000, + }, clientMetadata: { env: 'PARSE_SERVER_DATABASE_CLIENT_METADATA', help: "Custom metadata to append to database client connections for identifying Parse Server instances in database logs. If set, this metadata will be visible in database logs during connection handshakes. This can help with debugging and monitoring in deployments with multiple database clients. Set `name` to identify your application (e.g., 'MyApp') and `version` to your application's version. Leave undefined (default) to disable this feature and avoid the additional data transfer overhead.", diff --git a/src/Options/docs.js b/src/Options/docs.js index 0c2a296a31..13aaa00f17 100644 --- a/src/Options/docs.js +++ b/src/Options/docs.js @@ -265,6 +265,7 @@ * @property {String} authSource The MongoDB driver option to specify the database name associated with the user's credentials. * @property {Boolean} autoSelectFamily The MongoDB driver option to set whether the socket attempts to connect to IPv6 and IPv4 addresses until a connection is established. If available, the driver will select the first IPv6 address. * @property {Number} autoSelectFamilyAttemptTimeout The MongoDB driver option to specify the amount of time in milliseconds to wait for a connection attempt to finish before trying the next address when using the autoSelectFamily option. If set to a positive integer less than 10, the value 10 is used instead. + * @property {Number} batchSize The number of documents per batch for MongoDB cursor `getMore` operations. A lower value reduces memory usage per batch; a higher value reduces the number of network round-trips. * @property {DatabaseOptionsClientMetadata} clientMetadata Custom metadata to append to database client connections for identifying Parse Server instances in database logs. If set, this metadata will be visible in database logs during connection handshakes. This can help with debugging and monitoring in deployments with multiple database clients. Set `name` to identify your application (e.g., 'MyApp') and `version` to your application's version. Leave undefined (default) to disable this feature and avoid the additional data transfer overhead. * @property {Union} compressors The MongoDB driver option to specify an array or comma-delimited string of compressors to enable network compression for communication between this client and a mongod/mongos instance. * @property {Number} connectTimeoutMS The MongoDB driver option to specify the amount of time, in milliseconds, to wait to establish a single TCP socket connection to the server before raising an error. Specifying 0 disables the connection timeout. diff --git a/src/Options/index.js b/src/Options/index.js index eb1439538d..1539abd283 100644 --- a/src/Options/index.js +++ b/src/Options/index.js @@ -669,6 +669,9 @@ export interface DatabaseOptions { schemaCacheTtl: ?number; /* The MongoDB driver option to set whether to retry failed writes. */ retryWrites: ?boolean; + /* The number of documents per batch for MongoDB cursor `getMore` operations. A lower value reduces memory usage per batch; a higher value reduces the number of network round-trips. + :DEFAULT: 1000 */ + batchSize: ?number; /* The MongoDB driver option to set a cumulative time limit in milliseconds for processing operations on a cursor. */ maxTimeMS: ?number; /* The MongoDB driver option to set the maximum replication lag for reads from secondary nodes.*/ diff --git a/src/ParseServer.ts b/src/ParseServer.ts index 1e916efe61..8b1c6cdf8a 100644 --- a/src/ParseServer.ts +++ b/src/ParseServer.ts @@ -9,7 +9,7 @@ var batch = require('./batch'), fs = require('fs'); import { ParseServerOptions, LiveQueryServerOptions } from './Options'; -import defaults from './defaults'; +import defaults, { DatabaseOptionDefaults } from './defaults'; import * as logging from './logger'; import Config from './Config'; import PromiseRouter from './PromiseRouter'; @@ -593,6 +593,22 @@ function injectDefaults(options: ParseServerOptions) { } }); + // Inject defaults for database options; only when no explicit database adapter is set, + // because an explicit adapter manages its own options and passing databaseOptions alongside + // it would cause a conflict error in getDatabaseController. + if (!options.databaseAdapter) { + if (options.databaseOptions == null) { + options.databaseOptions = {}; + } + if (typeof options.databaseOptions === 'object' && !Array.isArray(options.databaseOptions)) { + Object.keys(DatabaseOptionDefaults).forEach(key => { + if (!Object.prototype.hasOwnProperty.call(options.databaseOptions, key)) { + options.databaseOptions[key] = DatabaseOptionDefaults[key]; + } + }); + } + } + if (!Object.prototype.hasOwnProperty.call(options, 'serverURL')) { options.serverURL = `http://localhost:${options.port}${options.mountPath}`; } diff --git a/src/defaults.js b/src/defaults.js index ba959e22fc..ce4bd09766 100644 --- a/src/defaults.js +++ b/src/defaults.js @@ -1,5 +1,5 @@ import { nullParser } from './Options/parsers'; -const { ParseServerOptions } = require('./Options/Definitions'); +const { ParseServerOptions, DatabaseOptions } = require('./Options/Definitions'); const logsFolder = (() => { let folder = './logs/'; if (typeof process !== 'undefined' && process.env.TESTING === '1') { @@ -34,10 +34,19 @@ const computedDefaults = { export default Object.assign({}, DefinitionDefaults, computedDefaults); export const DefaultMongoURI = DefinitionDefaults.databaseURI; +export const DatabaseOptionDefaults = Object.keys(DatabaseOptions).reduce((memo, key) => { + const def = DatabaseOptions[key]; + if (Object.prototype.hasOwnProperty.call(def, 'default')) { + memo[key] = def.default; + } + return memo; +}, {}); + // Parse Server-specific database options that should be filtered out // before passing to MongoDB client export const ParseServerDatabaseOptions = [ 'allowPublicExplain', + 'batchSize', 'clientMetadata', 'createIndexRoleName', 'createIndexUserEmail', diff --git a/types/Options/index.d.ts b/types/Options/index.d.ts index b8bfa8a83d..49e58cc1df 100644 --- a/types/Options/index.d.ts +++ b/types/Options/index.d.ts @@ -244,6 +244,7 @@ export interface FileUploadOptions { export interface DatabaseOptions { // Parse Server custom options allowPublicExplain?: boolean; + batchSize?: number; createIndexRoleName?: boolean; createIndexUserEmail?: boolean; createIndexUserEmailCaseInsensitive?: boolean;