diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-18 17:02:53 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-18 17:02:53 -0300 |
| commit | 959575a5ca598bf5f37fb5cebe7ed1d80d3d71f7 (patch) | |
| tree | acc8d60aedb12b70048e676e8a7349deb0010db8 /jstests/sharding | |
| parent | 76588293975fc059cf076779e4283e6ffaf8afff (diff) | |
New upstream version 6.0.20upstream
Diffstat (limited to 'jstests/sharding')
41 files changed, 802 insertions, 327 deletions
diff --git a/jstests/sharding/agg_union_with_lookup.js b/jstests/sharding/agg_union_with_lookup.js new file mode 100644 index 00000000000..587f3f85d20 --- /dev/null +++ b/jstests/sharding/agg_union_with_lookup.js @@ -0,0 +1,76 @@ +/** + * SERVER-88439 - when run against a sharded collection, this query caused a use after free + * when running getMore against the returned cursor. + * @tags: [ + * # Needs $lookup to support sharded foreign collections. + * requires_fcv_51, + * ] + */ + +function insertDocs(coll) { + assert.commandWorked(coll.insert([ + {v: 1, y: -3}, + {v: 2, y: -2}, + {v: 3, y: -1}, + {v: 4, y: 1}, + {v: 5, y: 2}, + {v: 6, y: 3}, + {v: 7, y: 4} + ])); +} + +const st = new ShardingTest({shards: 2, mongos: 1}); + +const conn = st.s; + +// Setup the collection. +const coll = conn.getCollection("test." + jsTestName() + "_1"); +coll.drop(); +insertDocs(coll); +assert.commandWorked(coll.createIndex({y: 1})); +st.shardColl(coll, + /* key */ {y: 1}, + /* split at */ {y: 0}, + /* move chunk containing */ {y: 1}, + /* db */ coll.getDB().getName(), + /* waitForDelete */ true); + +// Construct the command that causes the problem. The problem was observed during the +// subsequent getMore. +const cmd = { + aggregate: coll.getName(), + pipeline: [ + {$unionWith: { + coll: coll.getName(), + pipeline: [ + {$unionWith: { + coll: coll.getName(), + pipeline: [ + {$lookup: { + from: coll.getName(), + as: "lookedUp", + localField: "y", + foreignField: "y", + }} + ], + }}, + ] + }}, + ], + cursor: {batchSize: 1}, +}; + +const initialResult = assert.commandWorked(coll.runCommand(cmd)); +const cursor = initialResult.cursor; + +const getMore = { + getMore: cursor.id, + collection: coll.getName(), + batchSize: 1000 +}; +const getMoreResult = assert.commandWorked(coll.runCommand(getMore)); +const allResults = cursor.firstBatch.concat(getMoreResult.cursor.nextBatch); + +assert.eq(allResults.length, 21); + +st.stop(); diff --git a/jstests/sharding/all_collection_stats.js b/jstests/sharding/all_collection_stats.js index 2fd89236e70..1635d6f0573 100644 --- a/jstests/sharding/all_collection_stats.js +++ b/jstests/sharding/all_collection_stats.js @@ -9,6 +9,47 @@ (function() { 'use strict'; +function checkResults(aggregationPipeline, checksToDo) { + assert.soon(() => { + const results = adminDb.aggregate(aggregationPipeline).toArray(); + assert.lte(numCollections, results.length); + + for (let i = 0; i < numCollections; i++) { + try { + const coll = "coll" + i; + + // To check that the data retrieve from $_internalAllCollectionStats is correct we + // will call $collStats for each namespace to retrieve its storage stats and compare + // the two outputs. + const expectedResults = testDb.getCollection(coll) + .aggregate([{$collStats: {storageStats: {}}}]) + .toArray(); + assert.neq(null, expectedResults); + assert.eq(1, expectedResults.length); + + let exists = false; + for (const data of results) { + const ns = data.ns; + if (dbName + "." + coll === ns) { + checksToDo(data, expectedResults); + exists = true; + break; + } + } + assert(exists, + "Expected to have $_internalAllCollectionStats results for coll" + i); + } catch (e) { + // As we perform two logical executions of $collStats they might return different + // storageSizes since WT may have rewritten the file during a checkpoint or + // background compaction. We retry the operation as it is a transient error. + jsTest.log(e); + return false; + } + } + return true; + }); +} + // Configure initial sharding cluster const st = new ShardingTest({shards: 2}); const mongos = st.s; @@ -16,56 +57,41 @@ const mongos = st.s; const dbName = "test"; const testDb = mongos.getDB(dbName); const adminDb = mongos.getDB("admin"); +const numCollections = 4; // Insert sharded collections to validate the aggregation stage -for (let i = 0; i < 10; i++) { +for (let i = 0; i < (numCollections / 2); i++) { const coll = "coll" + i; assert(st.adminCommand({shardcollection: dbName + "." + coll, key: {skey: 1}})); assert.commandWorked(testDb.getCollection(coll).insert({skey: i})); } // Insert some unsharded collections to validate the aggregation stage -for (let i = 10; i < 20; i++) { +for (let i = numCollections / 2; i < numCollections; i++) { const coll = "coll" + i; assert.commandWorked(testDb.getCollection(coll).insert({skey: i})); } -// Get output data -const outputData = - adminDb.aggregate([{$_internalAllCollectionStats: {stats: {storageStats: {}}}}]).toArray(); -assert.gte(outputData.length, 20); - // Testing for comparing each collection returned from $_internalAllCollectionStats to $collStats -for (let i = 0; i < 20; i++) { - const coll = "coll" + i; - const expectedResults = - testDb.getCollection(coll).aggregate([{$collStats: {storageStats: {}}}]).toArray(); - assert.neq(null, expectedResults); - assert.eq(expectedResults.length, 1); - - let exists = false; - for (const data of outputData) { - const ns = data.ns; - if (dbName + "." + coll === ns) { - assert.eq(data.host, expectedResults[0].host); - assert.eq(data.shard, expectedResults[0].shard); - assert.eq(data.storageStats.size, expectedResults[0].storageStats.size); - assert.eq(data.storageStats.count, expectedResults[0].storageStats.count); - assert.eq(data.storageStats.avgObjSize, expectedResults[0].storageStats.avgObjSize); - assert.eq(data.storageStats.storageSize, expectedResults[0].storageStats.storageSize); - assert.eq(data.storageStats.freeStorageSize, - expectedResults[0].storageStats.freeStorageSize); - assert.eq(data.storageStats.nindexes, expectedResults[0].storageStats.nindexes); - assert.eq(data.storageStats.totalIndexSize, - expectedResults[0].storageStats.totalIndexSize); - assert.eq(data.storageStats.totalSize, expectedResults[0].storageStats.totalSize); - exists = true; - break; - } - } +(function testInternalAllCollectionStats() { + const aggregationPipeline = [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}]; - assert(exists); -} + const checksToDo = (left, right) => { + const msg = "Expected same output from $_internalAllCollectionStats and $collStats " + + "for same namespace"; + assert.eq(left.host, right[0].host, msg); + assert.eq(left.shard, right[0].shard, msg); + assert.eq(left.storageStats.size, right[0].storageStats.size, msg); + assert.eq(left.storageStats.count, right[0].storageStats.count, msg); + assert.eq(left.storageStats.avgObjSize, right[0].storageStats.avgObjSize, msg); + assert.eq(left.storageStats.storageSize, right[0].storageStats.storageSize, msg); + assert.eq(left.storageStats.freeStorageSize, right[0].storageStats.freeStorageSize, msg); + assert.eq(left.storageStats.nindexes, right[0].storageStats.nindexes, msg); + assert.eq(left.storageStats.totalIndexSize, right[0].storageStats.totalIndexSize, msg); + assert.eq(left.storageStats.totalSize, right[0].storageStats.totalSize, msg); + }; + checkResults(aggregationPipeline, checksToDo); +})(); // Test valid query with empty specification assert.commandWorked( diff --git a/jstests/sharding/balance_random_data_distribution.js b/jstests/sharding/balance_random_data_distribution.js new file mode 100644 index 00000000000..36c58f47ab9 --- /dev/null +++ b/jstests/sharding/balance_random_data_distribution.js @@ -0,0 +1,117 @@ +/* + * Test that the balancer redistributes data from multiple tracked collections across the + * cluster and it is able to converge within a limited amount of time. + * (Data amount & distribution, as well as per-collection maxChunkSize, are randomly chosen). + * + * @tags: [ + * requires_fcv_60, + * does_not_support_stepdowns, # TODO SERVER-89797 remove this tag. + * ] + * */ + +(function() { +"use strict"; + +load("jstests/libs/parallel_shell_helpers.js"); + +const numShards = 2; +Random.setRandomSeed(); + +const st = new ShardingTest({shards: numShards}); + +const clusterMaxChunkSizeMB = 8; +const collectionBalancedTimeoutMS = 10 * 60 * 1000 /* 10min */; + +const numDatabases = numShards; +const numCollInDB = 3; +const dbNamePrefix = 'test_db_'; +const collNamePrefix = 'coll_'; + +// 1. Setup an initial set of collections. +for (let i = 0; i < numDatabases; ++i) { + const dbName = dbNamePrefix + `${i}`; + const primaryShardId = st[`shard${i}`].shardName; + // Avoid assigning the same primary shard for every collection. + assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: primaryShardId})); + for (let j = 0; j < numCollInDB; ++j) { + let collName = collNamePrefix + `${j}`; + const coll = st.s.getDB(dbName)[collName]; + const ns = coll.getFullName(); + // Use {_id: 1} as shard key to allow room for imbalance as documents get later inserted. + st.s.adminCommand({shardCollection: ns, key: {_id: 1}}); + const collMaxChunkSizeMB = Random.randInt(clusterMaxChunkSizeMB - 1) + 1; + assert.commandWorked(st.s.adminCommand({ + configureCollectionBalancing: ns, + chunkSize: collMaxChunkSizeMB, + })); + } +} + +// 2. Launch the balancer and start multiple workers inserting random data into the existing +// collections. +st.startBalancer(); + +function doBatchInserts( + numDatabases, dbNamePrefix, numCollInDB, collNamePrefix, clusterMaxChunkSizeMB) { + Random.setRandomSeed(); + const numOfBatchInserts = 8; + const bigString = + 'X'.repeat(1024 * 1024 - 30); // Almost 1MB, to create documents of exactly 1MB + + for (let i = 0; i < numOfBatchInserts; ++i) { + const dbName = dbNamePrefix + `${Random.randInt(numDatabases)}`; + let randomDB = db.getSiblingDB(dbName); + const collName = collNamePrefix + `${Random.randInt(numCollInDB)}`; + const coll = randomDB[collName]; + + const numDocs = Random.randInt(clusterMaxChunkSizeMB - 1) + 1; + let insertBulkOp = coll.initializeUnorderedBulkOp(); + for (let i = 0; i < numDocs; ++i) { + insertBulkOp.insert({s: bigString}); + } + + assert.commandWorked(insertBulkOp.execute()); + } +} + +const numBackgroundBatchInserters = 5; +let backgroundBatchInserters = []; +for (let i = 0; i < numBackgroundBatchInserters; ++i) { + backgroundBatchInserters.push(startParallelShell(funWithArgs(doBatchInserts, + numDatabases, + dbNamePrefix, + numCollInDB, + collNamePrefix, + clusterMaxChunkSizeMB), + st.s.port)); +} + +// 3. Once the insertion workers are done, verify that the balancer may bring each tracked +// collection to a "balanced" state within the deadline. +for (let joinInserter of backgroundBatchInserters) { + joinInserter(); +} + +let testedAtLeastOneCollection = false; +for (let i = 0; i < numDatabases; i++) { + const dbName = dbNamePrefix + `${i}`; + for (let j = 0; j < numCollInDB; j++) { + const ns = dbName + '.' + collNamePrefix + `${j}`; + + const coll = st.s.getCollection(ns); + if (coll.countDocuments({}) === 0) { + // Skip empty collections + continue; + } + testedAtLeastOneCollection = true; + + // Wait for collection to be considered balanced + sh.awaitCollectionBalance(coll, collectionBalancedTimeoutMS, 1000 /* 1s interval */); + sh.verifyCollectionIsBalanced(coll); + } + + assert(testedAtLeastOneCollection); +} + +st.stop(); +}()); diff --git a/jstests/sharding/balancer_defragmentation_merge_chunks.js b/jstests/sharding/balancer_defragmentation_merge_chunks.js index 8bb0cb601d2..dc287f9772d 100644 --- a/jstests/sharding/balancer_defragmentation_merge_chunks.js +++ b/jstests/sharding/balancer_defragmentation_merge_chunks.js @@ -286,7 +286,8 @@ jsTest.log("Changed uuid causes defragmentation to restart"); })); st.startBalancer(); // Reshard collection - assert.commandWorked(db.adminCommand({reshardCollection: nss, key: {key2: 1}})); + assert.commandWorked( + db.adminCommand({reshardCollection: nss, key: {key2: 1}, numInitialChunks: 1})); // Let defragementation run clearFailPointOnConfigNodes("afterBuildingNextDefragmentationPhase"); defragmentationUtil.waitForEndOfDefragmentation(st.s, nss); diff --git a/jstests/sharding/balancer_warns_draining_shards_blocked.js b/jstests/sharding/balancer_warns_draining_shards_blocked.js new file mode 100644 index 00000000000..18af7a8cf59 --- /dev/null +++ b/jstests/sharding/balancer_warns_draining_shards_blocked.js @@ -0,0 +1,68 @@ +/* + * Verifies the balancer emits a warning when a removed shard cannot be drained due to balancing + * being disabled. There are two cases: + * - Balancer is disabled. + * - Balancing is disabled for a collection, which has chunks on draining shards. + */ + +load("jstests/libs/fail_point_util.js"); +load("jstests/libs/parallel_shell_helpers.js"); +load('jstests/sharding/libs/find_chunks_util.js'); + +const st = new ShardingTest({ + shards: 2, + config: 1, + rs: {nodes: 1}, + other: { + enableBalancer: false, + } +}); + +const mongos = st.s0; +const configDB = st.getDB('config'); + +const dbName = 'test'; +const collName = 'collToDrain'; +const ns = dbName + '.' + collName; +const testDB = st.getDB(dbName); +const coll = testDB.getCollection(collName); + +// Shard collection with shard0 as db primary. +assert.commandWorked( + mongos.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); +assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {x: 1}})); + +assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); +assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: -1}, to: st.shard0.shardName})); +assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: 1}, to: st.shard1.shardName})); + +// Check that there are only 2 chunks before starting draining. +const chunksBeforeDrain = findChunksUtil.findChunksByNs(configDB, ns).toArray(); +assert.eq(2, chunksBeforeDrain.length); + +// Force checks to happen continuously to speedup the test. +const csrsPrimary = st.configRS.getPrimary(); +configureFailPoint(csrsPrimary, "forceBalancerWarningChecks"); + +let awaitRemoveShard = startParallelShell(funWithArgs(function(shardName) { + load("jstests/sharding/libs/remove_shard_util.js"); + removeShard(db, shardName); + }, st.shard1.shardName), st.s.port); + +// Test warning when the balancer is disabled. +// "Draining of removed shards cannot be completed because the balancer is disabled" +checkLog.containsJson(csrsPrimary, 6434000); + +sh.disableBalancing(coll); +st.startBalancer(); + +// Test warning when the balancer is enabled, but a specific collection is disabled. +// "Draining of removed shards cannot be completed because the balancer is disabled for a collection +// which has chunks in those shards". +checkLog.containsJson(csrsPrimary, 7977400); + +// Re-enable balancing to let test terminate. +sh.enableBalancing(coll); +awaitRemoveShard(); + +st.stop(); diff --git a/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js b/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js index 91af933d10d..0ea93b8ae2a 100644 --- a/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js +++ b/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js @@ -26,6 +26,7 @@ load("jstests/libs/fail_point_util.js"); load('jstests/libs/parallelTester.js'); load("jstests/sharding/libs/create_sharded_collection_util.js"); +load("jstests/libs/auto_retry_transaction_in_sharding.js"); // For withTxnAndAutoRetryOnMongos. const kNumWriteTickets = 10; const st = new ShardingTest({ @@ -82,10 +83,10 @@ const sessionCollection = session.getDatabase(dbName).getCollection(collName); // transactionThread won't need to persist a topology time. The scenario reported in SERVER-60685 // depended on the TransactionCoordinator being interrupted while persisting the participant list // which happens after waiting for the topology time to become durable. -session.startTransaction(); -assert.commandWorked(sessionCollection.insert({key: 400})); -assert.commandWorked(sessionCollection.insert({key: -400})); -assert.commandWorked(session.commitTransaction_forTesting()); +withTxnAndAutoRetryOnMongos(session, () => { + assert.commandWorked(sessionCollection.insert({key: 400})); + assert.commandWorked(sessionCollection.insert({key: -400})); +}); const hangWithLockDuringBatchRemoveFp = configureFailPoint(txnCoordinator, failpointName); diff --git a/jstests/sharding/change_stream_lookup_single_shard_cluster.js b/jstests/sharding/change_stream_lookup_single_shard_cluster.js index e9ae2492ba7..fc1ae516f8c 100644 --- a/jstests/sharding/change_stream_lookup_single_shard_cluster.js +++ b/jstests/sharding/change_stream_lookup_single_shard_cluster.js @@ -2,6 +2,8 @@ // cluster only has a single shard, and that it can therefore successfully look up a document in a // sharded collection. // @tags: [ +// # If a rollback is triggered during a stepdown, the change stream cursor can become invalid. +// does_not_support_stepdowns, // requires_majority_read_concern, // uses_change_streams, // bf25011, diff --git a/jstests/sharding/change_streams_shards_start_in_sync.js b/jstests/sharding/change_streams_shards_start_in_sync.js index 4cf64d5f54c..cb57974099c 100644 --- a/jstests/sharding/change_streams_shards_start_in_sync.js +++ b/jstests/sharding/change_streams_shards_start_in_sync.js @@ -7,6 +7,8 @@ // and 'B' will be seen in the changestream before 'C'. // @tags: [ // does_not_support_stepdowns, +// # This test is flaky when running on PPC64 variants. +// ppc64le_incompatible, // requires_majority_read_concern, // uses_change_streams, // ] diff --git a/jstests/sharding/cluster_time_across_add_shard.js b/jstests/sharding/cluster_time_across_add_shard.js index 57c611cf1d2..ff983be7fea 100644 --- a/jstests/sharding/cluster_time_across_add_shard.js +++ b/jstests/sharding/cluster_time_across_add_shard.js @@ -102,7 +102,16 @@ for (let session of sessions) { // Verify that the application is able to use a signed cluster time although the addShard // or transitionFromDedicatedConfigServer command has not been run. - assert.commandWorked(session.getDatabase("admin").runCommand("hello")); + assert.soon(() => { + const res = session.getDatabase("admin").runCommand("hello"); + // TODO (SERVER-78909): KeysCollectionManager::getKeysForValidation() should retry + // refreshing if the refresh failed with ReadConcernMajorityNotAvailableYet + if (res.code == ErrorCodes.KeyNotFound) { + return false; + } + assert.commandWorked(res); + return true; + }); assert.eq(session.getClusterTime().signature.keyId, lastClusterTime.signature.keyId); } diff --git a/jstests/sharding/collection_uuid_reshard_collection.js b/jstests/sharding/collection_uuid_reshard_collection.js index b500165b69e..49548d92fcf 100644 --- a/jstests/sharding/collection_uuid_reshard_collection.js +++ b/jstests/sharding/collection_uuid_reshard_collection.js @@ -39,8 +39,12 @@ const uuid = function() { resetColl(coll); // The command succeeds when provided with the correct collection UUID. -assert.commandWorked(mongos.adminCommand( - {reshardCollection: coll.getFullName(), key: newKeyDoc, collectionUUID: uuid()})); +assert.commandWorked(mongos.adminCommand({ + reshardCollection: coll.getFullName(), + key: newKeyDoc, + collectionUUID: uuid(), + numInitialChunks: 1 +})); // The command fails when provided with a UUID with no corresponding collection. resetColl(coll); @@ -49,6 +53,7 @@ let res = assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: coll.getFullName(), key: newKeyDoc, collectionUUID: nonexistentUUID, + numInitialChunks: 1, }), ErrorCodes.CollectionUUIDMismatch); assert.eq(res.db, db.getName()); @@ -63,6 +68,7 @@ res = assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: coll2.getFullName(), key: newKeyDoc, collectionUUID: uuid(), + numInitialChunks: 1, }), ErrorCodes.CollectionUUIDMismatch); assert.eq(res.db, db.getName()); @@ -79,6 +85,7 @@ res = assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: coll3.getFullName(), key: newKeyDoc, collectionUUID: uuid(), + numInitialChunks: 1, }), ErrorCodes.CollectionUUIDMismatch); assert.eq(res.db, otherDB.getName()); @@ -93,6 +100,7 @@ res = assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: coll2.getFullName(), key: newKeyDoc, collectionUUID: uuid(), + numInitialChunks: 1, }), ErrorCodes.CollectionUUIDMismatch); assert.eq(res.db, db.getName()); @@ -108,6 +116,7 @@ res = assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: view.getFullName(), key: newKeyDoc, collectionUUID: uuid(), + numInitialChunks: 1, }), ErrorCodes.CollectionUUIDMismatch); assert.eq(res.db, db.getName()); diff --git a/jstests/sharding/configsvr_remove_chunks.js b/jstests/sharding/configsvr_remove_chunks.js index 32bd4de8e6a..9eed04de342 100644 --- a/jstests/sharding/configsvr_remove_chunks.js +++ b/jstests/sharding/configsvr_remove_chunks.js @@ -48,7 +48,9 @@ function insertLeftoverChunks(configDB, uuid) { let st = new ShardingTest({mongos: 1, shards: 1}); -const configDB = st.s.getDB('config'); +// Use retriable writes when writing to the config server since these are not automatically retried +const mongosSession = st.s.startSession({retryWrites: true}); +const configDB = mongosSession.getDatabase("config"); const dbName = "test"; const collName = "foo"; diff --git a/jstests/sharding/documents_db_not_exist.js b/jstests/sharding/documents_db_not_exist.js index 1742d149516..55d49349125 100644 --- a/jstests/sharding/documents_db_not_exist.js +++ b/jstests/sharding/documents_db_not_exist.js @@ -1,6 +1,6 @@ /** * Tests that $documents stage continues even when the database does not exist - * @tags: [requires_fcv_62, multiversion_incompatible] + * @tags: [multiversion_incompatible] * */ diff --git a/jstests/sharding/drop_database_before_write_is_targeted.js b/jstests/sharding/drop_database_before_write_is_targeted.js index d758ca5e950..18414baa1b9 100644 --- a/jstests/sharding/drop_database_before_write_is_targeted.js +++ b/jstests/sharding/drop_database_before_write_is_targeted.js @@ -5,7 +5,11 @@ * @tags: [ * # TODO (SERVER-84043): Requires the mongos to define the fail point. Enable multiversion * # once 8.0 becomes last LTS. - * multiversion_incompatible + * multiversion_incompatible, + * # The createDatabase command inside of the write might fail with FailedToSatisfyReadPreference + * # if it does not find a primary. This is not a retriable error, and is correct, but + * # incompatible with this test. + * does_not_support_stepdowns, * ] */ diff --git a/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js b/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js index 5b4b582f164..66433f99037 100644 --- a/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js +++ b/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js @@ -1,7 +1,7 @@ /* * @tags: [multiversion_incompatible] */ -const PROGRESS_TIMEOUT_SECONDS = 5; +const PROGRESS_TIMEOUT_SECONDS = 3; (function() { 'use strict'; @@ -45,7 +45,7 @@ assert.commandWorked( st.s1.adminCommand({"configureFailPoint": 'hangTestHealthObserver', "mode": "alwaysOn"})); // Wait for the progress monitor timeout to elapse. -sleep(1.1 * PROGRESS_TIMEOUT_SECONDS * 1000); +sleep(2.1 * PROGRESS_TIMEOUT_SECONDS * 1000); // Servers should be still be alive. jsTestLog("Expect monogs processes to still be alive"); @@ -55,7 +55,7 @@ jsTestLog("Change observer to critical"); changeObserverIntensity(st.s1, 'test', 'critical'); // Wait for the progress monitor timeout to elapse. -sleep(1.1 * PROGRESS_TIMEOUT_SECONDS * 1000); +sleep((2.1 * PROGRESS_TIMEOUT_SECONDS * 1000) + 1000); jsTestLog("Done sleeping"); // Servers should be still be alive. diff --git a/jstests/sharding/hedged_reads.js b/jstests/sharding/hedged_reads.js index bd121680d0a..10b655d5608 100644 --- a/jstests/sharding/hedged_reads.js +++ b/jstests/sharding/hedged_reads.js @@ -1,5 +1,10 @@ /** * Tests hedging metrics in the serverStatus output. + * @tags: [ + * # This test is known to be racey due to implementation of hedged reads (SERVER-65329). + * # Disable windows testing as this feature is deprecated in v8.0. + * incompatible_with_windows_tls, + * ] */ (function() { "use strict"; @@ -95,7 +100,9 @@ try { setCommandDelay(sortedNodes[0], "count", kBlockCmdTimeMS, ns); // Make the hedged request block for a while to allow the operation to start on the other node. - setCommandDelay(sortedNodes[1], "count", 100, ns); + // The delay is intentionally large so we avoid the race where a killOp can arrive before the + // request. + setCommandDelay(sortedNodes[1], "count", 1000, ns); const comment = "test_kill_initial_request_" + ObjectId(); assert.commandWorked(testDB.runCommand({ @@ -126,7 +133,9 @@ try { setCommandDelay(sortedNodes[1], "count", kBlockCmdTimeMS, ns); // Make the initial request block for a while to allow the operation to start on the other node. - setCommandDelay(sortedNodes[0], "count", 100, ns); + // The delay is intentionally large so we avoid the race where a killOp can arrive before the + // request. + setCommandDelay(sortedNodes[0], "count", 1000, ns); const comment = "test_kill_additional_request_" + ObjectId(); assert.commandWorked(testDB.runCommand({ diff --git a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js index fa0511ab27f..d2fad117f10 100644 --- a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js +++ b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js @@ -13,6 +13,8 @@ load("jstests/libs/feature_flag_util.js"); // Test deliberately inserts orphans outside of migration. TestData.skipCheckOrphans = true; +// TODO (SERVER-91380): remove skipCheckingIndexesConsistentAcrossCluster flag. +TestData.skipCheckingIndexesConsistentAcrossCluster = true; /* * Runs moveChunk on the host to move the chunk to the given shard. @@ -192,8 +194,7 @@ if (FeatureFlagUtil.isEnabled(st.shard0.getDB('admin'), "ShardKeyIndexOptionalHa // Verify dropping the shard key index succeeds. ShardedIndexUtil.assertIndexDoesNotExistOnShard( st.shard0, dbName, collName, hashedShardKey); - ShardedIndexUtil.assertIndexDoesNotExistOnShard( - st.shard1, dbName, collName, hashedShardKey); + // TODO (SERVER-91380): assert the shard key is not present on recipient shard1 as well. }); } diff --git a/jstests/sharding/libs/mongos_api_params_util.js b/jstests/sharding/libs/mongos_api_params_util.js index 696ef0d1c1c..7de6a36dcda 100644 --- a/jstests/sharding/libs/mongos_api_params_util.js +++ b/jstests/sharding/libs/mongos_api_params_util.js @@ -1412,7 +1412,7 @@ let MongosAPIParametersUtil = (function() { }; }); - const st = new ShardingTest({mongos: 1, shards: 2, rs: {nodes: 1}}); + const st = new ShardingTest({mongos: 1, shards: 2, config: 1, rs: {nodes: 1}}); const listCommandsRes = st.s0.adminCommand({listCommands: 1}); assert.commandWorked(listCommandsRes); diff --git a/jstests/sharding/libs/remove_shard_util.js b/jstests/sharding/libs/remove_shard_util.js new file mode 100644 index 00000000000..c096e45b8d4 --- /dev/null +++ b/jstests/sharding/libs/remove_shard_util.js @@ -0,0 +1,46 @@ +// Make it easier to understand whether or not returns from the assert.soon are being retried. +const kRetry = false; +const kNoRetry = true; + +function removeShard(shardingTestOrConn, shardName, timeout) { + if (timeout == undefined) { + timeout = 10 * 60 * 1000; // 10 minutes + } + + var s; + if (shardingTestOrConn instanceof ShardingTest) { + s = shardingTestOrConn.s; + } else { + s = shardingTestOrConn; + } + + assert.soon(function() { + let res; + if (TestData.configShard && shardName == "config") { + // Need to use transitionToDedicatedConfigServer if trying + // to remove config server as a shard + res = s.adminCommand({transitionToDedicatedConfigServer: shardName}); + } else { + res = s.adminCommand({removeShard: shardName}); + } + if (!res.ok) { + if (res.code === ErrorCodes.ShardNotFound) { + // If the config server primary steps down right after removing the config.shards + // doc for the shard but before responding with "state": "completed", the mongos + // would retry the _configsvrRemoveShard command against the new config server + // primary, which would not find the removed shard in its ShardRegistry if it has + // done a ShardRegistry reload after the config.shards doc for the shard was + // removed. This would cause the command to fail with ShardNotFound. + return kNoRetry; + } + if (res.code === ErrorCodes.HostUnreachable && TestData.runningWithConfigStepdowns) { + // The mongos may exhaust its retries due to having consecutive config stepdowns. In + // this case, the mongos will return a HostUnreachable error. + // We should retry the operation when this happens. + return kRetry; + } + } + assert.commandWorked(res); + return res.state == 'completed'; + }, "failed to remove shard " + shardName + " within " + timeout + "ms", timeout); +} diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js index 66c2133c90a..890fe0533ea 100644 --- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js +++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js @@ -19,7 +19,15 @@ TestData.skipCheckOrphans = true; (function() { 'use strict'; -var st = new ShardingTest({shards: 3, mongos: 1, other: {rs: true, rsOptions: {nodes: 2}}}); +var st = new ShardingTest({ + shards: 3, + other: { + rs: true, + // Disables elections to avoid secondaries becoming primaries after stepdowns. The test + // relies on specific topology changes done explicitly. + rsOptions: {nodes: 2, settings: {electionTimeoutMillis: ReplSetTest.kForeverMillis}} + }, +}); var mongos = st.s0; var admin = mongos.getDB("admin"); @@ -130,7 +138,19 @@ jsTest.log("Testing active connection with second primary down..."); // Reads with read prefs mongosConnActive.setSecondaryOk(); assert.neq(null, mongosConnActive.getCollection(collSharded.toString()).findOne({_id: -1})); -assert.neq(null, mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1})); +// This special retry logic is to mimic connection pool timeout in v8.0 and later. In earlier +// versions, the connection pool may time out with NetworkInterfaceExceededTimeLimit, which +// is not a retryable error. This is necessary because the now downed primary may time out +// with NetworkInterfaceExceededTimeLimit instead of HostUnreachable. +var res = null; +try { + res = mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1}); +} catch (e) { + // RSM marks failed host. + assert.commandFailedWithCode(e, ErrorCodes.NetworkInterfaceExceededTimeLimit); + res = mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1}); +} +assert.neq(null, res); assert.neq(null, mongosConnActive.getCollection(collUnsharded.toString()).findOne({_id: 1})); mongosConnActive.setSecondaryOk(false); diff --git a/jstests/sharding/query/aggregation_currentop.js b/jstests/sharding/query/aggregation_currentop.js index 58d63128329..207d93b42f2 100644 --- a/jstests/sharding/query/aggregation_currentop.js +++ b/jstests/sharding/query/aggregation_currentop.js @@ -13,7 +13,7 @@ * applicable. * * This test requires replica set configuration and user credentials to persist across a restart. - * @tags: [requires_persistence, uses_transactions, uses_prepare_transaction] + * @tags: [requires_persistence, uses_transactions, uses_prepare_transaction, requires_fcv_60] */ // Restarts cause issues with authentication for awaiting replication. @@ -411,7 +411,8 @@ function runCommonTests(conn, curOpSpec) { explain: true })); - let expectedStages = [{$currentOp: {idleConnections: true}}, {$match: {desc: {$eq: "test"}}}]; + let expectedStages = + [{$currentOp: {idleConnections: true, allUsers: false}}, {$match: {desc: {$eq: "test"}}}]; if (isRemoteShardCurOp) { assert.docEq(explainPlan.splitPipeline.shardsPart, expectedStages); diff --git a/jstests/sharding/query/collation_lookup.js b/jstests/sharding/query/collation_lookup.js index 8c479d8e12a..0e9577b43ca 100644 --- a/jstests/sharding/query/collation_lookup.js +++ b/jstests/sharding/query/collation_lookup.js @@ -299,130 +299,6 @@ function runTests(withDefaultCollationColl, withoutDefaultCollationColl, collati assert(arrayEq(expected, res), "Expected " + tojson(expected) + " to equal " + tojson(res) + " up to ordering"); - // Test that an explicit collation on the $lookup stage is respected. - res = withoutDefaultCollationColl - .aggregate( - [ - {$match: {_id: "lowercase"}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - localField: "str", - foreignField: "str", - as: "matched", - _internalCollation: collation["collation"], - }, - }, - ]) - .toArray(); - assert.eq(1, res.length, tojson(res)); - - expected = [{_id: "lowercase", str: "abc"}, {_id: "uppercase", str: "ABC"}]; - assert( - arrayEq(expected, res[0].matched), - "Expected " + tojson(expected) + " to equal " + tojson(res[0].matched) + " up to ordering"); - - res = withoutDefaultCollationColl - .aggregate( - [ - {$match: {_id: "lowercase"}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - let : {str1: "$str"}, - pipeline: [ - {$match: {$expr: {$eq: ["$str", "$$str1"]}}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - let : {str2: "$str"}, - pipeline: [{$match: {$expr: {$eq: ["$str", "$$str1"]}}}], - as: "matched2" - } - } - ], - as: "matched1", - _internalCollation: collation["collation"], - }, - } - ]) - .toArray(); - assert.eq(1, res.length, tojson(res)); - - expected = [ - { - "_id": "lowercase", - "str": "abc", - "matched2": [{"_id": "lowercase", "str": "abc"}, {"_id": "uppercase", "str": "ABC"}] - }, - { - "_id": "uppercase", - "str": "ABC", - "matched2": [{"_id": "lowercase", "str": "abc"}, {"_id": "uppercase", "str": "ABC"}] - } - ]; - assert(arrayEq(expected, res[0].matched1), - "Expected " + tojson(expected) + " to equal " + tojson(res[0].matched1) + - " up to ordering"); - - // Test that an explicit collation on the $lookup stage takes precedence over a command - // collation. - res = withoutDefaultCollationColl - .aggregate( - [ - {$match: {_id: "lowercase"}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - localField: "str", - foreignField: "str", - as: "matched", - _internalCollation: {locale: "simple"}, - }, - }, - ], - collation) - .toArray(); - assert.eq(1, res.length, tojson(res)); - - expected = [{_id: "lowercase", str: "abc"}]; - assert( - arrayEq(expected, res[0].matched), - "Expected " + tojson(expected) + " to equal " + tojson(res[0].matched) + " up to ordering"); - - res = withoutDefaultCollationColl - .aggregate( - [ - {$match: {_id: "lowercase"}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - let : {str1: "$str"}, - pipeline: [ - {$match: {$expr: {$eq: ["$str", "$$str1"]}}}, - { - $lookup: { - from: withoutDefaultCollationColl.getName(), - let : {str2: "$str"}, - pipeline: [{$match: {$expr: {$eq: ["$str", "$$str1"]}}}], - as: "matched2" - } - } - ], - as: "matched1", - _internalCollation: {locale: "simple"}, - }, - } - ],collation) - .toArray(); - assert.eq(1, res.length, tojson(res)); - - expected = - [{"_id": "lowercase", "str": "abc", "matched2": [{"_id": "lowercase", "str": "abc"}]}]; - assert(arrayEq(expected, res[0].matched1), - "Expected " + tojson(expected) + " to equal " + tojson(res[0].matched1) + - " up to ordering"); - // Test that the $lookup stage uses the "simple" collation if a collation isn't set on the // collection or the aggregation operation, even if the foreign collection has a collation. res = withoutDefaultCollationColl diff --git a/jstests/sharding/query/merge_nondefault_read_concern.js b/jstests/sharding/query/merge_nondefault_read_concern.js index 70df9207bd8..f59e21ee876 100644 --- a/jstests/sharding/query/merge_nondefault_read_concern.js +++ b/jstests/sharding/query/merge_nondefault_read_concern.js @@ -1,7 +1,6 @@ /** * Tests that $merge doesn't fail when a non-default readConcern is * set on the session. - * @tags: [requires_fcv_71] */ (function() { "use strict"; diff --git a/jstests/sharding/range_deletions_has_index.js b/jstests/sharding/range_deletions_has_index.js index deef14a9550..9f00ca8970b 100644 --- a/jstests/sharding/range_deletions_has_index.js +++ b/jstests/sharding/range_deletions_has_index.js @@ -14,6 +14,7 @@ const st = new ShardingTest({shards: {rs0: {nodes: 2}}}); // Test index gets created with an empty range deletions collection let newPrimary = st.rs0.getSecondaries()[0]; +st.rs0.awaitReplication(); assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1})); st.rs0.waitForPrimary(); const rangeDeletionColl = newPrimary.getDB("config").getCollection("rangeDeletions"); diff --git a/jstests/sharding/range_deletions_setFCV.js b/jstests/sharding/range_deletions_setFCV.js index a664e20ab54..d5bb9658e5f 100644 --- a/jstests/sharding/range_deletions_setFCV.js +++ b/jstests/sharding/range_deletions_setFCV.js @@ -13,6 +13,8 @@ load("jstests/libs/fail_point_util.js"); load('jstests/libs/parallel_shell_helpers.js'); +load("jstests/sharding/libs/chunk_bounds_util.js"); +load("jstests/sharding/libs/find_chunks_util.js"); const rangeDeleterBatchSize = 128; @@ -23,75 +25,128 @@ const st = new ShardingTest({ } }); -// Setup database and collection for test const dbName = 'db'; +const numDocsInColl = 1000; const db = st.getDB(dbName); +const configDB = st.getDB('config'); +const primaryShard = st.shard0; assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); -const coll = db['test']; -const nss = coll.getFullName(); -assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: 1}})); + st.s.adminCommand({enableSharding: dbName, primaryShard: primaryShard.shardName})); -assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: lastContinuousFCV})); - -function assertOrphanCountIsCorrectOrMissing(conn, ns, numOrphans) { - let fcv = - assert.commandWorked(conn.adminCommand({getParameter: 1, featureCompatibilityVersion: 1})); +function checkNumOrphansOnRangeDeletionTask(conn, ns, numOrphans) { const rangeDeletionDoc = conn.getDB("config").getCollection("rangeDeletions").findOne({nss: ns}); - if (fcv.featureCompatibilityVersion.version === "6.0") { - assert.neq( - null, - rangeDeletionDoc, - "did not find document for namespace " + ns + - ", contents of config.rangeDeletions on " + conn + ": " + - tojson(conn.getDB("config").getCollection("rangeDeletions").find().toArray())); - assert.eq(numOrphans, - rangeDeletionDoc.numOrphanDocs, - "Incorrect count of orphaned documents in config.rangeDeletions on " + conn + - ": expected " + numOrphans + - " orphaned documents but found range deletion document " + - tojson(rangeDeletionDoc)); + assert.neq(null, + rangeDeletionDoc, + "did not find document for namespace " + ns + + ", contents of config.rangeDeletions on " + conn + ": " + + tojson(conn.getDB("config").getCollection("rangeDeletions").find().toArray())); + assert.eq(numOrphans, + rangeDeletionDoc.numOrphanDocs, + "Incorrect count of orphaned documents in config.rangeDeletions on " + conn + + ": expected " + numOrphans + + " orphaned documents but found range deletion document " + + tojson(rangeDeletionDoc)); +} + +function moveChunkAndCheckRangeDeletionTasksUponFCVUpgrade(nss, donorShard, moveChunkCmd) { + // Ensure that no outstanding range deletion task will actually be executed (so that their + // recovery docs may be inspected). + let beforeDeletionFailpoint = configureFailPoint(donorShard, "hangBeforeDoingDeletion"); + let afterDeletionFailpoint = configureFailPoint(donorShard, "hangAfterDoingDeletion"); + + // Downgrade the cluster + assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: lastContinuousFCV})); + + // Upgrade the cluster - pausing the process before the "drain outstanding migrations" step + let pauseBeforeDrainingMigrations = + configureFailPoint(donorShard, "hangBeforeDrainingMigrations"); + const joinFCVUpgrade = startParallelShell( + funWithArgs(function(fcv) { + assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: fcv})); + }, latestFCV), st.s.port); + pauseBeforeDrainingMigrations.wait(); + // Complete a migration and check the pending range deletion tasks in the donor + assert.commandWorked(db.adminCommand(moveChunkCmd)); + + pauseBeforeDrainingMigrations.off(); + joinFCVUpgrade(); + // Check the batches are deleted correctly + const numBatches = numDocsInColl / rangeDeleterBatchSize; + assert(numBatches > 0); + for (let i = 0; i < numBatches; i++) { + // Wait for failpoint and check num orphans + beforeDeletionFailpoint.wait(); + checkNumOrphansOnRangeDeletionTask( + donorShard, nss, numDocsInColl - rangeDeleterBatchSize * i); + // Unset and reset failpoint without allowing any batches deleted in the meantime + afterDeletionFailpoint = configureFailPoint(donorShard, "hangAfterDoingDeletion"); + beforeDeletionFailpoint.off(); + afterDeletionFailpoint.wait(); + beforeDeletionFailpoint = configureFailPoint(donorShard, "hangBeforeDoingDeletion"); + afterDeletionFailpoint.off(); } + beforeDeletionFailpoint.off(); + assert.commandWorked(donorShard.rs.getPrimary().adminCommand({cleanupOrphaned: nss})); } -// Insert some docs into the collection. -const numDocs = 1000; -let bulk = coll.initializeUnorderedBulkOp(); -for (let i = 0; i < numDocs; i++) { - bulk.insert({_id: i}); +function testAgainstCollectionWithRangeShardKey() { + // Fill up a sharded collection + const coll = db['collWithRangeShardKey']; + const nss = coll.getFullName(); + assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: 1}})); + + // Insert some docs into the collection. + let bulk = coll.initializeUnorderedBulkOp(); + for (let i = 0; i < numDocsInColl; i++) { + bulk.insert({_id: i}); + } + assert.commandWorked(bulk.execute()); + + const moveChunkCmd = {moveChunk: nss, find: {_id: 0}, to: st.shard1.shardName}; + + moveChunkAndCheckRangeDeletionTasksUponFCVUpgrade(nss, primaryShard, moveChunkCmd); } -assert.commandWorked(bulk.execute()); - -// Pause before first range deletion task -let beforeDeletionFailpoint = configureFailPoint(st.shard0, "hangBeforeDoingDeletion"); -let afterDeletionFailpoint = configureFailPoint(st.shard0, "hangAfterDoingDeletion"); - -// Upgrade FCV to 6.0 -let pauseBeforeDrainingMigrations = configureFailPoint(st.shard0, "hangBeforeDrainingMigrations"); -const FCVUpgrade = startParallelShell( - funWithArgs(function(fcv) { - assert.commandWorked(db.adminCommand({setFeatureCompatibilityVersion: fcv})); - }, latestFCV), st.s.port); -pauseBeforeDrainingMigrations.wait(); -assert.commandWorked(db.adminCommand({moveChunk: nss, find: {_id: 0}, to: st.shard1.shardName})); - -pauseBeforeDrainingMigrations.off(); -// Check the batches are deleted correctly -const numBatches = numDocs / rangeDeleterBatchSize; -for (let i = 0; i < numBatches; i++) { - // Wait for failpoint and check num orphans - beforeDeletionFailpoint.wait(); - assertOrphanCountIsCorrectOrMissing(st.shard0, nss, numDocs - rangeDeleterBatchSize * i); - // Unset and reset failpoint without allowing any batches deleted in the meantime - afterDeletionFailpoint = configureFailPoint(st.shard0, "hangAfterDoingDeletion"); - beforeDeletionFailpoint.off(); - afterDeletionFailpoint.wait(); - beforeDeletionFailpoint = configureFailPoint(st.shard0, "hangBeforeDoingDeletion"); - afterDeletionFailpoint.off(); + +function testAgainstCollectionWithHashedShardKey() { + const collName = 'collWithHashedShardKey'; + const hotKeyValue = 'hotKeyValue'; + const hashedKeyValue = convertShardKeyToHashed(hotKeyValue); + const docWithHotShardKey = {k: hotKeyValue}; + const coll = db[collName]; + const nss = coll.getFullName(); + assert.commandWorked( + st.s.adminCommand({shardCollection: nss, key: {k: 'hashed'}, numInitialChunks: 1})); + + // Insert some docs into the collection. + let bulk = coll.initializeUnorderedBulkOp(); + for (let i = 0; i < numDocsInColl; i++) { + bulk.insert(docWithHotShardKey); + } + assert.commandWorked(bulk.execute()); + + // All the documents are supposed to be stored within a single shard + const allCollChunks = findChunksUtil.findChunksByNs(configDB, nss).toArray(); + const chunksWithDoc = allCollChunks.filter((chunk) => { + return chunkBoundsUtil.containsKey({k: hashedKeyValue}, chunk.min, chunk.max); + }); + assert.eq(1, chunksWithDoc.length); + const shardHoldingData = chunksWithDoc[0].shard === st.shard0.shardName ? st.shard0 : st.shard1; + const shardWithoutData = + shardHoldingData.shardName === st.shard0.shardName ? st.shard1 : st.shard0; + + const moveChunkCmd = { + moveChunk: nss, + bounds: [chunksWithDoc[0].min, chunksWithDoc[0].max], + to: shardWithoutData.shardName + }; + + moveChunkAndCheckRangeDeletionTasksUponFCVUpgrade(nss, shardHoldingData, moveChunkCmd); } -beforeDeletionFailpoint.off(); -FCVUpgrade(); +// Test Cases +testAgainstCollectionWithRangeShardKey(); +testAgainstCollectionWithHashedShardKey(); + st.stop(); })(); diff --git a/jstests/sharding/refresh_sessions.js b/jstests/sharding/refresh_sessions.js index c6d229707ca..dec07890b38 100644 --- a/jstests/sharding/refresh_sessions.js +++ b/jstests/sharding/refresh_sessions.js @@ -8,8 +8,15 @@ var sessionsDb = "config"; var refresh = {refreshLogicalSessionCacheNow: 1}; var startSession = {startSession: 1}; -// Create a cluster with 1 shard. -var cluster = new ShardingTest({shards: 2}); +var cluster = new ShardingTest({ + mongos: [{setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}}], + shards: 2, + rs: {setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}}, + other: { + configOptions: + {setParameter: {sessionWriteConcernTimeoutSystemMillis: 0, sessionMaxBatchSize: 500}} + } +}); // Test that we can refresh without any sessions, as a sanity check. { diff --git a/jstests/sharding/reshard_collection_basic.js b/jstests/sharding/reshard_collection_basic.js index 4b527757ecd..2dee0d4e5e7 100644 --- a/jstests/sharding/reshard_collection_basic.js +++ b/jstests/sharding/reshard_collection_basic.js @@ -212,29 +212,37 @@ let presetReshardedChunks = */ jsTest.log('Fail if sharding is disabled.'); -assert.commandFailedWithCode(mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}}), - ErrorCodes.NamespaceNotFound); +assert.commandFailedWithCode( + mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}, numInitialChunks: 1}), + ErrorCodes.NamespaceNotFound); assert.commandWorked(mongos.adminCommand({enableSharding: kDbName})); jsTest.log("Fail if collection is unsharded."); -assert.commandFailedWithCode(mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}}), - ErrorCodes.NamespaceNotSharded); +assert.commandFailedWithCode( + mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}, numInitialChunks: 1}), + ErrorCodes.NamespaceNotSharded); assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {oldKey: 1}})); jsTest.log("Fail if missing required key."); -assert.commandFailedWithCode(mongos.adminCommand({reshardCollection: ns}), 40414); +assert.commandFailedWithCode(mongos.adminCommand({reshardCollection: ns, numInitialChunks: 1}), + 40414); jsTest.log("Fail if unique is specified and is true."); assert.commandFailedWithCode( - mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}, unique: true}), + mongos.adminCommand( + {reshardCollection: ns, key: {newKey: 1}, unique: true, numInitialChunks: 1}), ErrorCodes.BadValue); jsTest.log("Fail if collation is specified and is not {locale: 'simple'}."); -assert.commandFailedWithCode( - mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}, collation: {locale: 'en_US'}}), - ErrorCodes.BadValue); +assert.commandFailedWithCode(mongos.adminCommand({ + reshardCollection: ns, + key: {newKey: 1}, + collation: {locale: 'en_US'}, + numInitialChunks: 1 +}), + ErrorCodes.BadValue); jsTest.log("Fail if both numInitialChunks and _presetReshardedChunks are provided."); assert.commandFailedWithCode(mongos.adminCommand({ @@ -263,6 +271,7 @@ jsTest.log("Fail if zone provided is invalid for storage."); assert.commandFailedWithCode(mongos.adminCommand({ reshardCollection: ns, key: {"_id": "hashed"}, + numInitialChunks: 1, zones: [{min: {"_id": {"$minKey": 1}}, max: {"_id": {"$maxKey": 1}}, zone: "Namezone"}] }), ErrorCodes.BadValue); @@ -301,13 +310,16 @@ assert.commandFailedWithCode(mongos.getDB('test').system.resharding.mycoll.inser mongos.getDB(kDbName)[collName].drop(); jsTest.log("Succeed when correct locale is provided."); -assertReshardCollOk({reshardCollection: ns, key: {newKey: 1}, collation: {locale: 'simple'}}, 1); +assertReshardCollOk( + {reshardCollection: ns, key: {newKey: 1}, collation: {locale: 'simple'}, numInitialChunks: 1}, + 1); jsTest.log("Succeed base case."); -assertReshardCollOk({reshardCollection: ns, key: {newKey: 1}}, 1); +assertReshardCollOk({reshardCollection: ns, key: {newKey: 1}, numInitialChunks: 1}, 1); jsTest.log("Succeed if unique is specified and is false."); -assertReshardCollOk({reshardCollection: ns, key: {newKey: 1}, unique: false}, 1); +assertReshardCollOk({reshardCollection: ns, key: {newKey: 1}, unique: false, numInitialChunks: 1}, + 1); jsTest.log( "Succeed if _presetReshardedChunks is provided and test commands are enabled (default)."); @@ -344,6 +356,7 @@ assertReshardCollOk({ key: {newKey: 1}, unique: false, collation: {locale: 'simple'}, + numInitialChunks: 1, zones: [{zone: newZoneName, min: {newKey: 5}, max: {newKey: 10}}] }, 3); @@ -353,8 +366,8 @@ assertReshardCollOk({ reshardCollection: ns, key: {newKey: 1}, unique: false, - numInitialChunks: 1, collation: {locale: 'simple'}, + numInitialChunks: 1, zones: [{zone: newZoneName, min: {newKey: MinKey}, max: {newKey: MaxKey}}] }, 1); @@ -380,6 +393,7 @@ assertReshardCollOk({ key: {oldKey: 1, newKey: 1}, unique: false, collation: {locale: 'simple'}, + numInitialChunks: 1, zones: [{zone: existingZoneName, min: {oldKey: 0}, max: {oldKey: 5}}] }, 3); diff --git a/jstests/sharding/reshard_collection_joins_existing_operation.js b/jstests/sharding/reshard_collection_joins_existing_operation.js index cd47186a5bd..72f5a19630e 100644 --- a/jstests/sharding/reshard_collection_joins_existing_operation.js +++ b/jstests/sharding/reshard_collection_joins_existing_operation.js @@ -22,8 +22,12 @@ load("jstests/sharding/libs/resharding_test_fixture.js"); const makeConfigsvrReshardCollectionThread = (configsvrConnString, ns) => { return new Thread((configsvrConnString, ns) => { const configsvr = new Mongo(configsvrConnString); - assert.commandWorked(configsvr.adminCommand( - {_configsvrReshardCollection: ns, key: {newKey: 1}, writeConcern: {w: "majority"}})); + assert.commandWorked(configsvr.adminCommand({ + _configsvrReshardCollection: ns, + key: {newKey: 1}, + numInitialChunks: 1, + writeConcern: {w: "majority"} + })); }, configsvrConnString, ns); }; diff --git a/jstests/sharding/resharding_change_stream_namespace_filtering.js b/jstests/sharding/resharding_change_stream_namespace_filtering.js index 85b5fd8a33d..d286b7f928c 100644 --- a/jstests/sharding/resharding_change_stream_namespace_filtering.js +++ b/jstests/sharding/resharding_change_stream_namespace_filtering.js @@ -48,8 +48,8 @@ for (let i = 0; i < 100; ++i) { } // Reshard the 'coll_reshard' collection on {b: 1}. -assert.commandWorked( - mongosDB.adminCommand({reshardCollection: mongosReshardColl.getFullName(), key: {b: 1}})); +assert.commandWorked(mongosDB.adminCommand( + {reshardCollection: mongosReshardColl.getFullName(), key: {b: 1}, numInitialChunks: 1})); // Confirm that the change stream we opened on 'coll_other' only sees the sentinel 'insert' but does // not see the earlier 'reshardBegin' or 'reshardDoneCatchUp' events on the 'coll_reshard' diff --git a/jstests/sharding/resharding_coordinator_recovers_abort_decision.js b/jstests/sharding/resharding_coordinator_recovers_abort_decision.js index d7b13c2b068..623f1adda62 100644 --- a/jstests/sharding/resharding_coordinator_recovers_abort_decision.js +++ b/jstests/sharding/resharding_coordinator_recovers_abort_decision.js @@ -93,7 +93,12 @@ reshardingTest.withReshardingInBackground( assert.commandWorked(mongos.getDB("admin").killOp(ops[0].opid)); - reshardingTest.stepUpNewPrimaryOnShard(reshardingTest.configShardName); + // Step down the config shard's primary. + let configRS = reshardingTest.getReplSetForShard(reshardingTest.configShardName); + let primary = configRS.getPrimary(); + assert.commandWorked( + primary.getDB("admin").runCommand({replSetStepDown: 60, force: true})); + configRS.waitForPrimary(); // After a stepdown, the _configsvrReshardCollection command will be retried by the // primary shard. We use the reshardCollectionJoinedExistingOperation failpoint to @@ -112,7 +117,6 @@ reshardingTest.withReshardingInBackground( // Wait for secondaries to recover and catchup with primary before turning off the // failpoints as a replication roll back can disconnect the test client. - const configRS = reshardingTest.getReplSetForShard(reshardingTest.configShardName); configRS.awaitSecondaryNodes(); configRS.awaitReplication(); reshardCollectionJoinedFailPointsList.forEach( diff --git a/jstests/sharding/resharding_disallow_drop.js b/jstests/sharding/resharding_disallow_drop.js index 144abd55b16..089c8f5a87f 100644 --- a/jstests/sharding/resharding_disallow_drop.js +++ b/jstests/sharding/resharding_disallow_drop.js @@ -31,7 +31,8 @@ const reshardingPauseBeforeInsertCoordinatorDocFailpoint = configureFailPoint(st.configRS.getPrimary(), "pauseBeforeInsertCoordinatorDoc"); assert.commandFailedWithCode( - db.adminCommand({reshardCollection: ns, key: {newKey: 1}, maxTimeMS: 1000}), + db.adminCommand( + {reshardCollection: ns, key: {newKey: 1}, maxTimeMS: 1000, numInitialChunks: 1}), ErrorCodes.MaxTimeMSExpired); // Wait for resharding to start running on the configsvr @@ -53,7 +54,8 @@ assert.commandFailedWithCode(db.runCommand({drop: collName, maxTimeMS: 5000}), // Finish resharding reshardingPauseBeforeInsertCoordinatorDocFailpoint.off(); -assert.commandWorked(db.adminCommand({reshardCollection: ns, key: {newKey: 1}})); +assert.commandWorked( + db.adminCommand({reshardCollection: ns, key: {newKey: 1}, numInitialChunks: 1})); // Now the drop can complete assert.commandWorked(db.runCommand({drop: collName})); diff --git a/jstests/sharding/retryable_write_error_labels.js b/jstests/sharding/retryable_write_error_labels.js index 23b0ff3c831..f4cf2db1a8e 100644 --- a/jstests/sharding/retryable_write_error_labels.js +++ b/jstests/sharding/retryable_write_error_labels.js @@ -137,6 +137,10 @@ function testMongodError(errorCode, isWCError) { function testMongosError() { const shard0Primary = st.rs0.getPrimary(); + // Insert initial documents used by the test. + const docs = [{k: 0, x: 0}, {k: 1, x: 1}]; + assert.commandWorked(shard0Primary.getDB(dbName)[collName].insert(docs)); + // Test retryable writes. jsTestLog("Retryable write should return mongos shutdown error with RetryableWriteError label"); @@ -211,7 +215,7 @@ function testMongosError() { const sessionDb = session.getDatabase(dbName); const sessionColl = sessionDb.getCollection(collName); session.startTransaction(); - assert.commandWorked(sessionColl.update({}, {$inc: {x: 1}})); + assert.commandWorked(sessionColl.update({k: 0}, {$inc: {x: 1}})); return sessionDb.adminCommand({ commitTransaction: 1, txnNumber: NumberLong(session.getTxnNumber_forTesting()), @@ -248,7 +252,7 @@ function testMongosError() { const sessionDb = session.getDatabase(dbName); const sessionColl = sessionDb.getCollection(collName); session.startTransaction(); - assert.commandWorked(sessionColl.update({}, {$inc: {x: 1}})); + assert.commandWorked(sessionColl.update({k: 1}, {$inc: {x: 1}})); return sessionDb.adminCommand({ abortTransaction: 1, txnNumber: NumberLong(session.getTxnNumber_forTesting()), diff --git a/jstests/sharding/run_restore.js b/jstests/sharding/run_restore.js index 3dfbd0b5b38..b5dd22e10f9 100644 --- a/jstests/sharding/run_restore.js +++ b/jstests/sharding/run_restore.js @@ -12,6 +12,12 @@ load("jstests/libs/feature_flag_util.js"); +// Because we restart nodes in standalone mode, it's possible for fast count, which doesn't +// discriminate between majority committed data and locally committed data, and the true count, +// which only includes majority committed data on standalones, to diverge. Therefore skip +// validating fast count. +TestData.skipEnforceFastCountOnValidate = true; + const s = new ShardingTest({name: "runRestore", shards: 2, mongos: 1, config: 1, other: {chunkSize: 1}}); diff --git a/jstests/sharding/run_restore_unsharded.js b/jstests/sharding/run_restore_unsharded.js index 038e79f1dd9..30591a0bbb5 100644 --- a/jstests/sharding/run_restore_unsharded.js +++ b/jstests/sharding/run_restore_unsharded.js @@ -11,6 +11,12 @@ load("jstests/libs/feature_flag_util.js"); +// Because we restart nodes in standalone mode, it's possible for fast count, which doesn't +// discriminate between majority committed data and locally committed data, and the true count, +// which only includes majority committed data on standalones, to diverge. Therefore skip +// validating fast count. +TestData.skipEnforceFastCountOnValidate = true; + const s = new ShardingTest({name: "runRestore", shards: 2, mongos: 1, config: 1, other: {chunkSize: 1}}); diff --git a/jstests/sharding/secondaries_clear_filtering_metadata.js b/jstests/sharding/secondaries_clear_filtering_metadata.js new file mode 100644 index 00000000000..a42ce83a8e7 --- /dev/null +++ b/jstests/sharding/secondaries_clear_filtering_metadata.js @@ -0,0 +1,73 @@ +/** + * Verifies that secondaries clear their filtering metadata whenever a collection is created. + * + * TODO (SERVER-91143): Move and adapt this test to the `core_sharding_passthrough` suite once the + * infrastructure is ready to support at least two routers. + */ + +const st = new ShardingTest({mongos: 2, shards: 2, rs: {nodes: 2}}); + +const s0 = st.s0; +const s1 = st.s1; + +const dbName = "MyDb"; +const collName = "MyColl"; +const collNs = dbName + "." + collName; + +assert.commandWorked(s0.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); +assert.commandWorked(s0.adminCommand({shardCollection: collNs, key: {x: 1}})); +assert.commandWorked(s0.adminCommand({split: collNs, middle: {x: 0}})); +assert.commandWorked(s0.adminCommand({moveChunk: collNs, find: {x: 0}, to: st.shard1.shardName})); + +// router0: SV1 +// router1: Empty +// shard0: primary=SV1, secondary=UNKNOWN +// shard1: primary=SV2, secondary=UNKNOWN + +assert.commandWorked(s0.getDB(dbName).MyColl.insert({x: -100})); +assert.commandWorked(s0.getDB(dbName).MyColl.insert({x: 100})); +assert.commandWorked(s1.getDB(dbName).MyColl.insert({x: -1000})); +assert.commandWorked(s1.getDB(dbName).MyColl.insert({x: 1000})); + +// router0: SV1 +// router1: SV1 +// shard0: primary=SV1, secondary=UNKNOWN +// shard1: primary=SV2, secondary=UNKNOWN + +s0.getDB(dbName).MyColl.drop(); + +// router0: Empty +// router1: SV1 (not aware of drop) +// shard0: primary=UNKNOWN, secondary=UNKNOWN +// shard1: primary=UNKNOWN, secondary=UNKNOWN + +// router1 will target shard1, then shard0. +s1.getDB(dbName).MyColl.find({x: 0}).readPref('secondary').toArray(); + +// router0: Empty +// router1: Empty +// shard0: primary=UNSHARDED, secondary=UNSHARDED +// shard1: primary=UNSHARDED, secondary=UNSHARDED + +assert.commandWorked(s0.adminCommand({shardCollection: collNs, key: {y: 1}})); +assert.commandWorked(s0.getDB(dbName).MyColl.insert({y: 42})); + +// router0: SV1 +// router1: Empty +// shard0: primary=SV1, secondary=UNKNOWN +// shard1: primary=UNSHARDED, secondary=UNSHARDED + +// This should reset shard1's shard version to be UNKNOWN on all nodes. +assert.commandWorked(s0.adminCommand({movePrimary: dbName, to: st.shard1.shardName})); + +// router0: SV1 +// router1: Empty +// shard0: primary=SV1, secondary=UNKNOWN +// shard1: primary=UNKNOWN, secondary=UNKNOWN + +// If this were to equal 0 it would mean that s1 sent an UNSHARDED version to a stale secondary on +// the new dbPrimary shard1. Being 1 means we're doing the correct thing here. +assert.eq(s1.getDB(dbName).MyColl.find({}).readPref('secondary').itcount(), 1); +assert.eq(s0.getDB(dbName).MyColl.find({}).itcount(), 1); + +st.stop(); diff --git a/jstests/sharding/shard_encrypted_collection.js b/jstests/sharding/shard_encrypted_collection.js index d4c4de4f642..9c467a2b86b 100644 --- a/jstests/sharding/shard_encrypted_collection.js +++ b/jstests/sharding/shard_encrypted_collection.js @@ -40,6 +40,10 @@ function testShardingCommand(command) { let commandObj = {}; commandObj[command] = kDbName + '.basic'; + if (command === "reshardCollection") { + commandObj['numInitialChunks'] = 1; + } + jsTestLog('Fail ' + command + ' if shard key is an encrypted field'); commandObj['key'] = {firstName: 1}; res = mongos.adminCommand(commandObj); diff --git a/jstests/sharding/shard_keys_with_dollar_sign.js b/jstests/sharding/shard_keys_with_dollar_sign.js index c7fbbc73be5..5748a99cd3e 100644 --- a/jstests/sharding/shard_keys_with_dollar_sign.js +++ b/jstests/sharding/shard_keys_with_dollar_sign.js @@ -42,7 +42,8 @@ function testValidation(key, {isValidIndexKey, isValidShardKey}) { } assert.commandWorked(st.s.adminCommand({shardCollection: ns2, key: {_id: 1}})); - const reshardCollectionRes = st.s.adminCommand({reshardCollection: ns2, key}); + const reshardCollectionRes = + st.s.adminCommand({reshardCollection: ns2, key, numInitialChunks: 1}); if (isValidShardKey) { assert.commandWorked(reshardCollectionRes); } else { diff --git a/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js b/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js new file mode 100644 index 00000000000..b4dc7636ed4 --- /dev/null +++ b/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js @@ -0,0 +1,58 @@ +// Tests that a deleteMany operation on a cluster doesn't return a UUID mismatch in case it targets +// a shard with no chunks. +// +(function() { +"use strict"; +const st = new ShardingTest({shards: 3}); + +const db = st.s.getDB(jsTestName()); +assert.commandWorked( + st.s.adminCommand({enableSharding: db.getName(), primaryShard: st.shard0.shardName})); + +const shardedColl1 = db.sharded_1; + +assert.commandWorked(shardedColl1.insert({_id: 0})); +assert.commandWorked(shardedColl1.insert({_id: 10})); +assert.commandWorked(shardedColl1.insert({_id: 50})); +assert.commandWorked(shardedColl1.insert({_id: 100})); + +const splitChunk = function(coll, splitPointKeyValue) { + assert.commandWorked( + st.s.adminCommand({split: coll.getFullName(), middle: {_id: splitPointKeyValue}})); +}; + +const uuid = function(coll) { + return assert.commandWorked(db.runCommand({listCollections: 1})) + .cursor.firstBatch.find(c => c.name === coll.getName()) + .info.uuid; +}; + +// shard0: [inf, 50), shard1: [50, inf), shard2 has no chunks +assert.commandWorked( + st.s.adminCommand({shardCollection: shardedColl1.getFullName(), key: {_id: 1}})); +splitChunk(shardedColl1, 50); +assert.commandWorked(st.s.adminCommand( + {moveChunk: shardedColl1.getFullName(), find: {_id: 50}, to: st.shard1.shardName})); + +const cmdObj = { + delete: shardedColl1.getName(), + collectionUUID: uuid(shardedColl1), + deletes: [{ + q: { + $and: + [{$expr: {$gte: ["$_id", {$literal: 0}]}}, {$expr: {$lt: ["$_id", {$literal: 99}]}}] + }, + limit: 0, + hint: {_id: 1} + }], +}; + +// Run a multi-write which should only target shards 0 & 1. This could target shard 2 but we +// shouldn't see any UUID mismatch errors since shard2 has no chunks and a delete is correctly +// defined in this case. +let res = db.runCommand(cmdObj); +assert.commandWorked(res); +// Only 3 documents fulfill the criteria so verify we've deleted them. +assert.eq(3, res.n); +st.stop(); +})(); diff --git a/jstests/sharding/timeseries_multiple_mongos.js b/jstests/sharding/timeseries_multiple_mongos.js index dbf88f6fa56..6b6aee4987c 100644 --- a/jstests/sharding/timeseries_multiple_mongos.js +++ b/jstests/sharding/timeseries_multiple_mongos.js @@ -298,19 +298,6 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { update: collName, updates: [{ - q: {}, - u: {$inc: {[metaField + ".b"]: 1}}, - multi: true, - }] - }, - numProfilerEntries: {sharded: 2, unsharded: 1}, - }); - - runTest({ - shardKey: {[metaField + ".a"]: 1}, - cmdObj: { - update: collName, - updates: [{ q: {[metaField + ".a"]: 1}, u: {$inc: {[metaField + ".b"]: -1}}, multi: true, @@ -324,19 +311,6 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { update: bucketsCollName, updates: [{ - q: {}, - u: {$inc: {["meta.b"]: 1}}, - multi: true, - }] - }, - numProfilerEntries: {sharded: 2, unsharded: 1}, - }); - - runTest({ - shardKey: {[metaField + ".a"]: 1}, - cmdObj: { - update: bucketsCollName, - updates: [{ q: {["meta.a"]: 1}, u: {$inc: {["meta.b"]: -1}}, multi: true, @@ -351,18 +325,6 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { delete: collName, deletes: [{ - q: {}, - limit: 0, - }], - }, - numProfilerEntries: {sharded: 2, unsharded: 1}, - }); - - runTest({ - shardKey: {[metaField]: 1}, - cmdObj: { - delete: collName, - deletes: [{ q: {[metaField]: 0}, limit: 0, }], @@ -375,18 +337,6 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { delete: bucketsCollName, deletes: [{ - q: {}, - limit: 0, - }], - }, - numProfilerEntries: {sharded: 2, unsharded: 1}, - }); - - runTest({ - shardKey: {[metaField]: 1}, - cmdObj: { - delete: bucketsCollName, - deletes: [{ q: {meta: 0}, limit: 0, }], diff --git a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js index 10ec9c91e31..7590dce0a73 100644 --- a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js +++ b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js @@ -21,14 +21,12 @@ const testDB = st.s.getDB('test'); const coll = testDB[jsTest.name()]; const collName = coll.getFullName(); +// Shard a collection on _id:1 so that the initial chunk will reside on the primary shard (shard0) assert.commandWorked( st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); assert.commandWorked(st.s.adminCommand({shardCollection: collName, key: {_id: 1}})); -// Initialize TTL index: delete documents with field `a: <current date>` after 20 seconds -assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 20})); - -// Insert documents that are going to be deleted in 20 seconds +// Insert documents that are going to be deleted by the TTL index created later on const currTime = new Date(); var bulk = coll.initializeUnorderedBulkOp(); const nDocs = 100; @@ -37,16 +35,20 @@ for (let i = 0; i < nDocs; i++) { } assert.commandWorked(bulk.execute()); -// Move all documents on other shards +// Move all documents to the other shard (shard1) but keep a chunk on shard0 to create the TTL index +assert.commandWorked(st.s.adminCommand({split: collName, middle: {_id: -1}})); assert.commandWorked( st.s.adminCommand({moveChunk: collName, find: {_id: 0}, to: st.shard1.shardName})); -// Verify that TTL index worked properly on owned documents +// Initialize TTL index: delete documents with field `a: <current date>` older than 1 second +assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 1})); + +// Verify that TTL index worked properly on owned documents on shard1 assert.soon(function() { return coll.countDocuments({}) == 0; }, "Failed to move all documents", 60000 /* 60 seconds */, 5000 /* 5 seconds */); -// Verify that TTL index did not delete orphaned documents +// Verify that TTL index did not delete orphaned documents on shard0 assert.eq(nDocs, st.rs0.getPrimary().getCollection(collName).countDocuments({})); st.stop(); diff --git a/jstests/sharding/txn_recover_decision_using_recovery_router.js b/jstests/sharding/txn_recover_decision_using_recovery_router.js index 09afd9a8a04..62818ab0bb2 100644 --- a/jstests/sharding/txn_recover_decision_using_recovery_router.js +++ b/jstests/sharding/txn_recover_decision_using_recovery_router.js @@ -185,8 +185,19 @@ const waitForCommitTransactionToComplete = function(coordinatorRs, lsid, txnNumb }); }; -let st = - new ShardingTest({shards: 2, rs: {nodes: 2}, mongos: 2, other: {mongosOptions: {verbose: 3}}}); +let st = new ShardingTest({ + shards: 2, + rs: {nodes: 2}, + rsOptions: { + setParameter: { + // Set this to match the value of transactionLifetimeLimitSeconds to avoid transition + // failure due to not being able to acquire the lock quickly enough. + maxTransactionLockRequestTimeoutMillis: TestData.transactionLifetimeLimitSeconds * 1000 + } + }, + mongos: 2, + other: {mongosOptions: {verbose: 3}} +}); // The default WC is majority and this test can't satisfy majority writes. assert.commandWorked(st.s.adminCommand( diff --git a/jstests/sharding/verify_sessions_expiration_sharded.js b/jstests/sharding/verify_sessions_expiration_sharded.js index bc3cf254755..ea225b96d04 100644 --- a/jstests/sharding/verify_sessions_expiration_sharded.js +++ b/jstests/sharding/verify_sessions_expiration_sharded.js @@ -123,6 +123,10 @@ for (let i = 0; i < 3; i++) { lastUseValues[j] = sessionsCollectionArray[j].lastUse; } } + + // Date_t has the granularity of milliseconds, so we have to make sure we don't run this loop + // faster than that. + sleep(10); } // 3. Verify that letting sessions expire (simulated by manual deletion) will kill their |
