diff options
Diffstat (limited to 'jstests/concurrency/fsm_workloads')
7 files changed, 319 insertions, 194 deletions
diff --git a/jstests/concurrency/fsm_workloads/agg_out.js b/jstests/concurrency/fsm_workloads/agg_out.js index d320676a167..86ebca8801a 100644 --- a/jstests/concurrency/fsm_workloads/agg_out.js +++ b/jstests/concurrency/fsm_workloads/agg_out.js @@ -37,11 +37,10 @@ var $config = extendWorkload($config, function($config, $super) { $config.transitions = { query: { - query: 0.63, + query: 0.68, createIndexes: 0.1, dropIndex: 0.1, collMod: 0.1, - movePrimary: 0.05, // Converting the target collection to a capped collection or a sharded collection will // cause all subsequent aggregations to fail, so give these a low probability to make // sure they don't happen too early in the test. @@ -49,7 +48,6 @@ var $config = extendWorkload($config, function($config, $super) { shardCollection: 0.01, }, createIndexes: {query: 1}, - movePrimary: {query: 1}, dropIndex: {query: 1}, collMod: {query: 1}, convertToCapped: {query: 1}, @@ -60,7 +58,6 @@ var $config = extendWorkload($config, function($config, $super) { * Runs an aggregate with a $out into '$config.data.outputCollName'. */ $config.states.query = function query(db, collName) { - jsTestLog(`Running query: coll=${collName} out=${this.outputCollName}`); const res = db[collName].runCommand({ aggregate: collName, pipeline: [{$match: {flag: true}}, {$out: this.outputCollName}], @@ -68,27 +65,15 @@ var $config = extendWorkload($config, function($config, $super) { }); const allowedErrorCodes = [ - // Indexes of target collection changed during processing. - ErrorCodes.CommandFailed, - // $out is not supported to an existing *sharded* output collection. - ErrorCodes.IllegalOperation, - // Namespace is capped so it can't be used for $out. - 17152, - // $out collection cannot be sharded. - 28769, - // $out can't be executed while there is a move primary in progress. - ErrorCodes.MovePrimaryInProgress, - // (SERVER-78850) Move primary coordinator drops donor collections when it is done. - // This invalidates non - snapshot cursors, which causes $out to fail. - // Locally it fails with explicit collection dropped error. When doing remote reads, - // it fails with cursor not found error. - ErrorCodes.CursorNotFound, + ErrorCodes.CommandFailed, // indexes of target collection changed during processing. + ErrorCodes.IllegalOperation, // $out is not supported to an existing *sharded* output + // collection. + 17152, // namespace is capped so it can't be used for $out. + 28769, // $out collection cannot be sharded. ]; assertWhenOwnDB.commandWorkedOrFailedWithCode(res, allowedErrorCodes); if (res.ok) { - // No matter how many documents were in the original input stream, $out should never - // return any results. const cursor = new DBCommandCursor(db, res); assertAlways.eq(0, cursor.itcount()); // No matter how many documents were in the // original input stream, $out should never @@ -102,10 +87,7 @@ var $config = extendWorkload($config, function($config, $super) { */ $config.states.createIndexes = function createIndexes(db, unusedCollName) { for (var i = 0; i < this.indexSpecs; ++i) { - const indexSpecs = this.indexSpecs[i]; - jsTestLog(`Running createIndex: coll=${this.outputCollName} indexSpec=${indexSpecs}`); - assertWhenOwnDB.commandWorkedOrFailedWithCode( - db[this.outputCollName].createIndex(indexSpecs), ErrorCodes.MovePrimaryInProgress); + assertWhenOwnDB.commandWorked(db[this.outputCollName].createIndex(this.indexSpecs[i])); } }; @@ -114,7 +96,6 @@ var $config = extendWorkload($config, function($config, $super) { */ $config.states.dropIndex = function dropIndex(db, unusedCollName) { const indexSpec = this.indexSpecs[Random.randInt(this.indexSpecs.length)]; - jsTestLog(`Running dropIndex: coll=${this.outputCollName} indexSpec=${indexSpec}`); db[this.outputCollName].dropIndex(indexSpec); }; @@ -126,39 +107,20 @@ var $config = extendWorkload($config, function($config, $super) { // Change the validation level. const validationLevels = ['off', 'strict', 'moderate']; const newValidationLevel = validationLevels[Random.randInt(validationLevels.length)]; - jsTestLog(`Running collMod: coll=${this.outputCollName} validationLevel=${ - newValidationLevel}`); - assertWhenOwnDB.commandWorkedOrFailedWithCode( db.runCommand({collMod: this.outputCollName, validationLevel: newValidationLevel}), - [ErrorCodes.ConflictingOperationInProgress, ErrorCodes.MovePrimaryInProgress]); + ErrorCodes.ConflictingOperationInProgress); } else { // Change the validation action. - const validationAction = Random.rand() > 0.5 ? 'warn' : 'error'; - jsTestLog(`Running collMod: coll=${this.outputCollName} validationAction=${ - validationAction}`); - assertWhenOwnDB.commandWorkedOrFailedWithCode( - db.runCommand({collMod: this.outputCollName, validationAction: validationAction}), - [ErrorCodes.ConflictingOperationInProgress, ErrorCodes.MovePrimaryInProgress]); + db.runCommand({ + collMod: this.outputCollName, + validationAction: Random.rand() > 0.5 ? 'warn' : 'error' + }), + ErrorCodes.ConflictingOperationInProgress); } }; - $config.states.movePrimary = function movePrimary(db, collName) { - if (!isMongos(db)) { - return; - } - const toShard = this.shards[Random.randInt(this.shards.length)]; - jsTestLog(`Running movePrimary: db=${db} to=${toShard}`); - - assert.commandWorkedOrFailedWithCode( - db.adminCommand({movePrimary: db.getName(), to: toShard}), [ - // Caused by a concurrent movePrimary operation on the same database but a different - // destination shard. - ErrorCodes.ConflictingOperationInProgress, - ]); - }; - /** * Converts '$config.data.outputCollName' to a capped collection. This is never undone, and all * subsequent $out's to this collection should fail. @@ -168,10 +130,8 @@ var $config = extendWorkload($config, function($config, $super) { return; // convertToCapped can't be run against a mongos. } - jsTestLog(`Running convertToCapped: coll=${this.outputCollName}`); - assertWhenOwnDB.commandWorkedOrFailedWithCode( - db.runCommand({convertToCapped: this.outputCollName, size: 100000}), - ErrorCodes.MovePrimaryInProgress); + assertWhenOwnDB.commandWorked( + db.runCommand({convertToCapped: this.outputCollName, size: 100000})); }; /** @@ -182,7 +142,6 @@ var $config = extendWorkload($config, function($config, $super) { */ $config.states.shardCollection = function shardCollection(db, unusedCollName) { if (isMongos(db) && this.tid === 0) { - jsTestLog(`Running shardCollection: coll=${this.outputCollName} key=${this.shardKey}`); assertWhenOwnDB.commandWorked(db.adminCommand({enableSharding: db.getName()})); assertWhenOwnDB.commandWorked(db.adminCommand( {shardCollection: db[this.outputCollName].getFullName(), key: {_id: 'hashed'}})); @@ -198,10 +157,6 @@ var $config = extendWorkload($config, function($config, $super) { // `shardCollection()` requires a shard key index to be in place on the output collection, // as we may be sharding a non-empty collection. assertWhenOwnDB.commandWorked(db[this.outputCollName].createIndex({_id: 'hashed'})); - - if (isMongos(db)) { - this.shards = Object.keys(cluster.getSerializedCluster().shards); - } }; return $config; diff --git a/jstests/concurrency/fsm_workloads/collection_uuid_sharded.js b/jstests/concurrency/fsm_workloads/collection_uuid_sharded.js index 7fbaba0357f..40cc51e3219 100644 --- a/jstests/concurrency/fsm_workloads/collection_uuid_sharded.js +++ b/jstests/concurrency/fsm_workloads/collection_uuid_sharded.js @@ -34,7 +34,8 @@ var $config = extendWorkload($config, function($config, $super) { let reshardCollectionCmd = { reshardCollection: namespace, key: {a: 1}, - collectionUUID: this.collUUID + collectionUUID: this.collUUID, + numInitialChunks: 1, }; testCommand(db, namespace, "reshardCollection", reshardCollectionCmd, this); @@ -56,7 +57,8 @@ var $config = extendWorkload($config, function($config, $super) { reshardCollectionCmd = { reshardCollection: namespace, key: {_id: 1}, - collectionUUID: this.collUUID + collectionUUID: this.collUUID, + numInitialChunks: 1, }; testCommand(db, namespace, "reshardCollection", reshardCollectionCmd, this); } diff --git a/jstests/concurrency/fsm_workloads/insert_with_data_size_aware_balancing.js b/jstests/concurrency/fsm_workloads/insert_with_data_size_aware_balancing.js deleted file mode 100644 index 65315d99b36..00000000000 --- a/jstests/concurrency/fsm_workloads/insert_with_data_size_aware_balancing.js +++ /dev/null @@ -1,131 +0,0 @@ -'use strict'; - -/** - * - Shard several collections with different (random) configured maxChunkSize - * - Perform continuous inserts of random amounts of data into the collections - * - Verify that the balancer fairly redistributes data among available shards - * - * @tags: [ - * requires_sharding, - * assumes_balancer_on, - * featureFlagBalanceAccordingToDataSize, - * requires_fcv_60, - * ] - */ - -const bigString = 'X'.repeat(1024 * 1024 - 30); // Almost 1MB, to create documents of exactly 1MB -const minChunkSizeMB = 1; -const maxChunkSizeMB = 10; -const dbNames = ['db0', 'db1']; -const collNames = ['collA', 'collB', 'collC']; - -/* - * Get a random db/coll name from the test lists. - * - * Using the thread id to introduce more randomness: it has been observed that concurrent calls to - * Random.randInt(array.length) are returning too often the same number to different threads. - */ -function getRandomDbName(tid) { - return dbNames[Random.randInt(tid * tid) % dbNames.length]; -} -function getRandomCollName(tid) { - return collNames[Random.randInt(tid * tid) % collNames.length]; -} - -var $config = (function() { - let states = { - /* - * Insert into a test collection a random amount of documents (up to 10MB per iteration) - */ - insert: function(db, collName, connCache) { - const dbName = getRandomDbName(this.tid); - db = db.getSiblingDB(dbName); - collName = getRandomCollName(this.tid); - const coll = db[collName]; - - const numDocs = Random.randInt(maxChunkSizeMB - 1) + 1; - let insertBulkOp = coll.initializeUnorderedBulkOp(); - for (let i = 0; i < numDocs; ++i) { - insertBulkOp.insert({s: bigString}); - } - - assertAlways.commandWorked(insertBulkOp.execute()); - }, - }; - - /* - * Create sharded collections with random maxChunkSizeMB (betwen 1MB and 10MB) - */ - let setup = function(db, collName, cluster) { - const mongos = cluster.getDB('config').getMongo(); - const shardNames = Object.keys(cluster.getSerializedCluster().shards); - const numShards = shardNames.length; - - for (let i = 0; i < dbNames.length; i++) { - // Initialize database - const dbName = dbNames[i]; - const newDb = db.getSiblingDB(dbName); - newDb.adminCommand({enablesharding: dbName, primaryShard: shardNames[i % numShards]}); - - for (let j = 0; j < collNames.length; j++) { - // Shard collection - collName = collNames[j]; - const coll = newDb[collName]; - const ns = coll.getFullName(); - db.adminCommand({shardCollection: ns, key: {_id: 1}}); - - // Configure random maxChunkSize - const randomMaxChunkSizeMB = Random.randInt(maxChunkSizeMB - 1) + 1; - assert.commandWorked(mongos.adminCommand({ - configureCollectionBalancing: ns, - chunkSize: randomMaxChunkSizeMB, - })); - } - } - }; - - /* - * Verify that the balancer fairly redistributes data among available shards: the - * collection size difference between two shards must be at most 2 * maxChunkSize - */ - let teardown = function(db, collName, cluster) { - const mongos = cluster.getDB('config').getMongo(); - // Sentinel variable to make sure not all collections have been skipped - let testedAtLeastOneCollection = false; - for (let i = 0; i < dbNames.length; i++) { - const dbName = dbNames[i]; - for (let j = 0; j < collNames.length; j++) { - collName = collNames[j]; - const ns = dbName + '.' + collName; - - const coll = mongos.getCollection(ns); - if (coll.countDocuments({}) === 0) { - // Skip empty collections - continue; - } - testedAtLeastOneCollection = true; - - // Wait for collection to be considered balanced - sh.awaitCollectionBalance( - coll, 5 * 60000 /* 5min timeout */, 1000 /* 1s interval */); - sh.verifyCollectionIsBalanced(coll); - } - - assert(testedAtLeastOneCollection); - } - }; - - let transitions = {insert: {insert: 1.0}}; - - return { - threadCount: 5, - iterations: 8, - startState: 'insert', - states: states, - transitions: transitions, - data: {}, - setup: setup, - teardown: teardown, - passConnectionCache: true - }; -})(); diff --git a/jstests/concurrency/fsm_workloads/query_stats_concurrent.js b/jstests/concurrency/fsm_workloads/query_stats_concurrent.js new file mode 100644 index 00000000000..6d7b9456288 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/query_stats_concurrent.js @@ -0,0 +1,155 @@ +'use strict'; + +/** + * query_stats_concurrent.js + * + * Stresses $queryStats running concurrently with queries. + * + * @tags: [ + * requires_fcv_60, + * does_not_support_causal_consistency, + * ] + * + * + */ +load("jstests/concurrency/fsm_workload_helpers/set_parameter.js"); + +var $config = (function() { + var states = (function() { + function init(db, collName) { + } + + function reInit(db, collName) { + } + + // Runs one find query so that the queryStatsEntry is updated. + function findOneShape(db, collName) { + assertWhenOwnColl.gt(db[collName].find({i: {$lt: 50}}).itcount(), 0); + } + + // Runs one agg query so that the queryStatsEntry is updated. + function aggOneShape(db, collName) { + assertWhenOwnColl.gt(db[collName].aggregate([{$match: {i: {$gt: 900}}}]).itcount(), 0); + } + + // Runs many queries with different shapes to ensure eviction occurs in the queryStats + // store. + function multipleShapes(db, collName) { + for (var i = 0; i < 10000; i++) { + let query = {}; + query["foo" + i] = "bar"; + db[collName].aggregate([{$match: query}]).itcount(); + } + const evictedAfter = db.serverStatus().metrics.queryStats.numEvicted; + assertAlways.gt(evictedAfter, 0); + } + + // Runs queryStats with transformation. + function runQueryStatsWithHmac(db, collName) { + let response = db.adminCommand({ + aggregate: 1, + pipeline: [{ + $queryStats: { + transformIdentifiers: { + algorithm: "hmac-sha-256", + hmacKey: BinData(8, "MjM0NTY3ODkxMDExMTIxMzE0MTUxNjE3MTgxOTIwMjE=") + } + } + }], + // Use a small batch size to ensure these operations open up a cursor and use + // multiple getMores. + cursor: {batchSize: 1} + }); + assertAlways.commandWorked(response); + const cursor = new DBCommandCursor(db.getSiblingDB("admin"), response); + assertAlways.gt(cursor.itcount(), 0); + } + + // Runs queryStats without transformation. + function runQueryStatsWithoutHmac(db, collName) { + let response = db.adminCommand({ + aggregate: 1, + pipeline: [{$queryStats: {}}], + // Use a small batch size to ensure these operations open up a cursor and use + // multiple getMores. + cursor: {batchSize: 1} + }); + assertAlways.commandWorked(response); + const cursor = new DBCommandCursor(db.getSiblingDB("admin"), response); + assertAlways.gt(cursor.itcount(), 0); + } + + return { + init: init, + reInit: reInit, + findOneShape: findOneShape, + multipleShapes: multipleShapes, + aggOneShape: aggOneShape, + runQueryStatsWithHmac: runQueryStatsWithHmac, + runQueryStatsWithoutHmac: runQueryStatsWithoutHmac + }; + })(); + + var internalQueryStatsRateLimit; + var internalQueryStatsCacheSize; + + let setup = function(db, collName, cluster) { + internalQueryStatsRateLimit = setParameterOnAllNodes( + {cluster: cluster, paramName: "internalQueryStatsRateLimit", newValue: -1}); + internalQueryStatsCacheSize = setParameterOnAllNodes( + {cluster: cluster, paramName: "internalQueryStatsCacheSize", newValue: "1MB"}); + + assert.commandWorked(db[collName].createIndex({i: 1})); + const bulk = db[collName].initializeUnorderedBulkOp(); + for (let i = 0; i < 1000; ++i) { + bulk.insert({i: i}); + } + assert.commandWorked(bulk.execute()); + }; + + let teardown = function(db, collName, cluster) { + setParameterOnAllNodes({ + cluster: cluster, + paramName: "internalQueryStatsRateLimit", + newValue: internalQueryStatsRateLimit + }); + setParameterOnAllNodes({ + cluster: cluster, + paramName: "internalQueryStatsCacheSize", + newValue: internalQueryStatsCacheSize + }); + + db[collName].drop(); + }; + + let transitions = { + // To start, add some $queryStats data so that it is never empty. + init: { + aggOneShape: 0.33, + findOneShape: 0.33, + multipleShapes: 0.34, + }, + // From then on, choose evenly among all possibilities: + reInit: { + aggOneShape: 0.2, + findOneShape: 0.2, + multipleShapes: 0.2, + runQueryStatsWithHmac: 0.2, + runQueryStatsWithoutHmac: 0.2 + }, + findOneShape: {reInit: 1}, + multipleShapes: {reInit: 1}, + runQueryStatsWithHmac: {reInit: 1}, + runQueryStatsWithoutHmac: {reInit: 1}, + aggOneShape: {reInit: 1} + }; + + return { + threadCount: 10, + iterations: 10, + states: states, + setup: setup, + teardown: teardown, + transitions: transitions + }; +})(); diff --git a/jstests/concurrency/fsm_workloads/query_stats_enable_disable.js b/jstests/concurrency/fsm_workloads/query_stats_enable_disable.js new file mode 100644 index 00000000000..ff22b26efb0 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/query_stats_enable_disable.js @@ -0,0 +1,133 @@ +/** + * query_stats_enable_disable.js + * + * Stresses $queryStats being toggled on and off during execution. + * + * @tags: [ + * requires_fcv_60, + * # Cannot read query stats entries from a different node than issued the query. + * does_not_support_causal_consistency, + * # setParameter is not persistent. + * does_not_support_stepdowns, + * # Changing the query stats parameters messes with the query stats test which expects to see + * # query stats results. + * incompatible_with_concurrency_simultaneous, + * ] + * + */ + +load("jstests/concurrency/fsm_workload_helpers/set_parameter.js"); + +var $config = (function() { + function setCacheSize(db, cacheSize) { + assert.commandWorked( + db.adminCommand({setParameter: 1, internalQueryStatsCacheSize: cacheSize})); + } + + function setRateLimit(db, rateLimit) { + assert.commandWorked( + db.adminCommand({setParameter: 1, internalQueryStatsRateLimit: rateLimit})); + } + + function runQueryStats({db, options}) { + try { + // Use a small batch size to ensure these operations open up a cursor and use + // multiple getMores - this will better stress odd concurrency states. + const cursor = + db.getSiblingDB("admin").aggregate([{$queryStats: options}], {batchSize: 1}); + // Can't assert much in particular about the results. + assert.gte(cursor.itcount(), 0); + } catch (e) { + if (e.code === ErrorCodes.QueryFeatureNotAllowed) { + // This means query stats is disabled, which is expected to happen in this workload. + return; + } + throw e; + } + } + + const hmacOptions = { + transformIdentifiers: { + algorithm: "hmac-sha-256", + hmacKey: BinData(8, "MjM0NTY3ODkxMDExMTIxMzE0MTUxNjE3MTgxOTIwMjE=") + } + }; + const nDifferentShapes = 200; + const states = { + // The main operating states which toggle the feature on and off, since we've seen + // bugs like SERVER-84730 when this happens. + disableViaCacheSize: (db, collName) => setCacheSize(db, "0MB"), + enableViaCacheSize: (db, collName) => setCacheSize(db, "1MB"), + disableViaRateLimit: (db, collName) => setRateLimit(db, 0), + enableViaRateLimit: (db, collName) => setRateLimit(db, -1), + + runOneQuery: function runOneQuery(db, collName) { + assert.eq(null, + db[collName].findOne({["field" + Random.randInt(nDifferentShapes)]: 42})); + }, + + runQueryStatsWithHmac: (db, collName) => runQueryStats({db: db, options: hmacOptions}), + runQueryStatsWithoutHmac: (db, collName) => runQueryStats({db: db, options: {}}), + init: (db, collName) => { /* no op */ }, + }; + + const transitions = { + init: { + // Make the most common thing be running a query. + runOneQuery: 0.6, + // Followed by runing $queryStats, with half the probability. + runQueryStatsWithHmac: 0.15, + runQueryStatsWithoutHmac: 0.15, + // These sum up to 0.1: + disableViaCacheSize: 0.05, + disableViaRateLimit: 0.05, + }, + runOneQuery: {init: 1}, + runQueryStatsWithHmac: {init: 1}, + runQueryStatsWithoutHmac: {init: 1}, + // If you disable it, immediately re-enable it. + disableViaCacheSize: {enableViaCacheSize: 1}, + enableViaCacheSize: {init: 1}, + disableViaRateLimit: {enableViaRateLimit: 1}, + enableViaRateLimit: {init: 1}, + }; + + var internalQueryStatsRateLimit; + var internalQueryStatsCacheSize; + + const setup = function(db, collName, cluster) { + // TODO SERVER-85405 Make this pattern easier and repeated throughout multiple workloads, + // not just the query stats ones. + internalQueryStatsRateLimit = setParameterOnAllNodes( + {cluster: cluster, paramName: "internalQueryStatsRateLimit", newValue: -1}); + internalQueryStatsCacheSize = setParameterOnAllNodes( + {cluster: cluster, paramName: "internalQueryStatsCacheSize", newValue: "1MB"}); + }; + + const teardown = function(db, collName, cluster) { + setParameterOnAllNodes({ + cluster: cluster, + paramName: "internalQueryStatsRateLimit", + newValue: internalQueryStatsRateLimit, + // We were messing with the settings in this test by sending commands only to the + // primary. The secondaries are not expected to be involved and so cannot be expected to + // have the same settings. + assertAllSettingsWereIdentical: false, + }); + setParameterOnAllNodes({ + cluster: cluster, + paramName: "internalQueryStatsCacheSize", + newValue: internalQueryStatsCacheSize, + assertAllSettingsWereIdentical: false, + }); + }; + + return { + threadCount: 10, + iterations: 10, + states: states, + setup: setup, + teardown: teardown, + transitions: transitions + }; +})(); diff --git a/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js b/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js index d74d2454176..18fa771a945 100644 --- a/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js +++ b/jstests/concurrency/fsm_workloads/random_moveChunk_refine_collection_shard_key.js @@ -57,7 +57,17 @@ var $config = extendWorkload($config, function($config, $super) { // migrated back in. The particular error code is replaced with a more generic one, so this // is identified by the failed migration's error message. $config.data.isMoveChunkErrorAcceptable = (err) => { - const codes = [ErrorCodes.LockBusy, ErrorCodes.ShardKeyNotFound, ErrorCodes.LockTimeout]; + const codes = [ + // TODO SERVER-68551: Remove lockbusy error since the balancer won't acquire anymore the + // DDL lock for migrations + ErrorCodes.LockBusy, + ErrorCodes.ShardKeyNotFound, + ErrorCodes.LockTimeout, + // The refienCollectionCoordinator interrupt all migrations by setting `allowMigration` + // to false + ErrorCodes.Interrupted, + ErrorCodes.OrphanedRangeCleanUpFailed, + ]; return (err.code && codes.includes(err.code)) || (err.message && (err.message.includes("CommandFailed") || diff --git a/jstests/concurrency/fsm_workloads/random_moveChunk_timeseries_insert_many.js b/jstests/concurrency/fsm_workloads/random_moveChunk_timeseries_insert_many.js index 97b0cf8e5b4..80224a2725d 100644 --- a/jstests/concurrency/fsm_workloads/random_moveChunk_timeseries_insert_many.js +++ b/jstests/concurrency/fsm_workloads/random_moveChunk_timeseries_insert_many.js @@ -7,6 +7,7 @@ * assumes_balancer_off, * requires_non_retryable_writes, * does_not_support_transactions, + * requires_fcv_51, * ] */ load('jstests/concurrency/fsm_workload_helpers/chunks.js'); // for chunk helpers |