diff options
Diffstat (limited to 'jstests/sharding')
170 files changed, 1534 insertions, 6587 deletions
diff --git a/jstests/sharding/agg_out_drop_database.js b/jstests/sharding/agg_out_drop_database.js index 7cceb60e848..a911278cc84 100644 --- a/jstests/sharding/agg_out_drop_database.js +++ b/jstests/sharding/agg_out_drop_database.js @@ -3,7 +3,7 @@ * * @tags: [ * requires_fcv_51, - * does_not_support_stepdowns, # DropDatabaseCoordinator drops the input collection on step-up + * does_not_support_stepdowns, // DropDatabaseCoordinator drops the input collection on step-up * ] */ diff --git a/jstests/sharding/all_collection_stats.js b/jstests/sharding/all_collection_stats.js deleted file mode 100644 index 1635d6f0573..00000000000 --- a/jstests/sharding/all_collection_stats.js +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Test to validate the $_internalAllCollectionStats stage for storageStats. - * - * @tags: [ - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; - -function checkResults(aggregationPipeline, checksToDo) { - assert.soon(() => { - const results = adminDb.aggregate(aggregationPipeline).toArray(); - assert.lte(numCollections, results.length); - - for (let i = 0; i < numCollections; i++) { - try { - const coll = "coll" + i; - - // To check that the data retrieve from $_internalAllCollectionStats is correct we - // will call $collStats for each namespace to retrieve its storage stats and compare - // the two outputs. - const expectedResults = testDb.getCollection(coll) - .aggregate([{$collStats: {storageStats: {}}}]) - .toArray(); - assert.neq(null, expectedResults); - assert.eq(1, expectedResults.length); - - let exists = false; - for (const data of results) { - const ns = data.ns; - if (dbName + "." + coll === ns) { - checksToDo(data, expectedResults); - exists = true; - break; - } - } - assert(exists, - "Expected to have $_internalAllCollectionStats results for coll" + i); - } catch (e) { - // As we perform two logical executions of $collStats they might return different - // storageSizes since WT may have rewritten the file during a checkpoint or - // background compaction. We retry the operation as it is a transient error. - jsTest.log(e); - return false; - } - } - return true; - }); -} - -// Configure initial sharding cluster -const st = new ShardingTest({shards: 2}); -const mongos = st.s; - -const dbName = "test"; -const testDb = mongos.getDB(dbName); -const adminDb = mongos.getDB("admin"); -const numCollections = 4; - -// Insert sharded collections to validate the aggregation stage -for (let i = 0; i < (numCollections / 2); i++) { - const coll = "coll" + i; - assert(st.adminCommand({shardcollection: dbName + "." + coll, key: {skey: 1}})); - assert.commandWorked(testDb.getCollection(coll).insert({skey: i})); -} - -// Insert some unsharded collections to validate the aggregation stage -for (let i = numCollections / 2; i < numCollections; i++) { - const coll = "coll" + i; - assert.commandWorked(testDb.getCollection(coll).insert({skey: i})); -} - -// Testing for comparing each collection returned from $_internalAllCollectionStats to $collStats -(function testInternalAllCollectionStats() { - const aggregationPipeline = [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}]; - - const checksToDo = (left, right) => { - const msg = "Expected same output from $_internalAllCollectionStats and $collStats " + - "for same namespace"; - assert.eq(left.host, right[0].host, msg); - assert.eq(left.shard, right[0].shard, msg); - assert.eq(left.storageStats.size, right[0].storageStats.size, msg); - assert.eq(left.storageStats.count, right[0].storageStats.count, msg); - assert.eq(left.storageStats.avgObjSize, right[0].storageStats.avgObjSize, msg); - assert.eq(left.storageStats.storageSize, right[0].storageStats.storageSize, msg); - assert.eq(left.storageStats.freeStorageSize, right[0].storageStats.freeStorageSize, msg); - assert.eq(left.storageStats.nindexes, right[0].storageStats.nindexes, msg); - assert.eq(left.storageStats.totalIndexSize, right[0].storageStats.totalIndexSize, msg); - assert.eq(left.storageStats.totalSize, right[0].storageStats.totalSize, msg); - }; - checkResults(aggregationPipeline, checksToDo); -})(); - -// Test valid query with empty specification -assert.commandWorked( - adminDb.runCommand({aggregate: 1, pipeline: [{$_internalAllCollectionStats: {}}], cursor: {}})); - -// Test invalid queries/values. -assert.commandFailedWithCode( - adminDb.runCommand({aggregate: 1, pipeline: [{$_internalAllCollectionStats: 3}], cursor: {}}), - 6789103); - -const response = assert.commandFailedWithCode(testDb.runCommand({ - aggregate: "foo", - pipeline: [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}], - cursor: {} -}), - 6789104); -assert.neq(-1, response.errmsg.indexOf("$_internalAllCollectionStats"), response.errmsg); -assert.neq(-1, response.errmsg.indexOf("admin database"), response.errmsg); - -st.stop(); -})(); diff --git a/jstests/sharding/all_collection_stats_auth.js b/jstests/sharding/all_collection_stats_auth.js deleted file mode 100644 index 022bc88ab4a..00000000000 --- a/jstests/sharding/all_collection_stats_auth.js +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Test to validate the privileges of using $_internalAllCollectionStats stage. - * - * @tags: [ - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; - -if (!TestData.auth) { - jsTestLog("Skipping testing authorization since auth is not enabled"); - return; -} - -// Test privileges -function testPrivileges() { - // Create new role with the exact privileges to execute $allCollectionStats - assert.commandWorked(adminDb.runCommand({ - createRole: "role_ok_priv", - roles: [], - privileges: [{resource: {cluster: true}, actions: ["allCollectionStats"]}] - })); - - // Creates users with privileges and no privileges - assert.commandWorked(adminDb.runCommand({createUser: "user_no_priv", pwd: "pwd", roles: []})); - - assert.commandWorked(adminDb.runCommand( - {createUser: "user_priv1", pwd: "pwd", roles: [{role: "role_ok_priv", db: 'admin'}]})); - - assert.commandWorked(adminDb.runCommand( - {createUser: "user_priv2", pwd: "pwd", roles: [{role: "clusterMonitor", db: 'admin'}]})); - - assert(adminDb.logout()); - - // User is in a role with privileges to execute the stage - assert(adminDb.auth("user_priv1", "pwd")); - assert.commandWorked(adminDb.runCommand({ - aggregate: 1, - pipeline: [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}], - cursor: {} - })); - assert(adminDb.logout()); - - // User is in a role with privileges to execute the stage - assert(adminDb.auth("user_priv2", "pwd")); - assert.commandWorked(adminDb.runCommand({ - aggregate: 1, - pipeline: [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}], - cursor: {} - })); - assert(adminDb.logout()); - - // User has no privileges to execute the stage - assert(adminDb.auth("user_no_priv", "pwd")); - assert.commandFailedWithCode( - adminDb.runCommand({ - aggregate: 1, - pipeline: [{$_internalAllCollectionStats: {stats: {storageStats: {}}}}], - cursor: {} - }), - ErrorCodes.Unauthorized, - "user should no longer have privileges to execute $_internalAllCollectionStats stage."); - assert(adminDb.logout()); -} - -// Configure initial sharding cluster -const st = new ShardingTest({shards: 1}); -const mongos = st.s; - -const ns1 = "test.foo"; -const adminDb = mongos.getDB("admin"); -const testDb = mongos.getDB("test"); - -// Create a super user with __system role. -assert.commandWorked(adminDb.runCommand({createUser: "super", pwd: "super", roles: ["__system"]})); -assert(adminDb.logout()); -assert(adminDb.auth("super", "super")); - -st.adminCommand({shardcollection: ns1, key: {skey: 1}}); - -// Insert data to validate the aggregation stage -for (let i = 0; i < 6; i++) { - assert.commandWorked(testDb.getCollection("foo").insert({skey: i})); -} - -testPrivileges(); - -st.stop(); -})(); diff --git a/jstests/sharding/append_oplog_note_mongos.js b/jstests/sharding/append_oplog_note_mongos.js index 1369ecb06be..cc19980c5b1 100644 --- a/jstests/sharding/append_oplog_note_mongos.js +++ b/jstests/sharding/append_oplog_note_mongos.js @@ -1,6 +1,8 @@ /** * Tests that the 'appendOplogNote' command on mongos correctly performs a no-op write on each * shard and advances the $clusterTime. + * + * @tags: [requires_fcv_60] */ (function() { diff --git a/jstests/sharding/auth.js b/jstests/sharding/auth.js index 3bf15b56716..48351d0a59b 100644 --- a/jstests/sharding/auth.js +++ b/jstests/sharding/auth.js @@ -10,7 +10,6 @@ 'use strict'; load("jstests/replsets/rslib.js"); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // Replica set nodes started with --shardsvr do not enable key generation until they are added // to a sharded cluster and reject commands with gossiped clusterTime from users without the @@ -175,32 +174,25 @@ awaitRSClientHosts(s.s, d2.nodes, {ok: true}); s.getDB("test").foo.remove({}); -var num = 10; +var num = 10000; assert.commandWorked(s.s.adminCommand({split: "test.foo", middle: {x: num / 2}})); -const bigString = 'X'.repeat(1024 * 1024); // 1MB var bulk = s.getDB("test").foo.initializeUnorderedBulkOp(); for (i = 0; i < num; i++) { - bulk.insert({_id: i, x: i, abc: "defg", date: new Date(), str: bigString}); + bulk.insert({_id: i, x: i, abc: "defg", date: new Date(), str: "all the talk on the market"}); } assert.commandWorked(bulk.execute()); s.startBalancer(60000); -const balanceAccordingToDataSize = - FeatureFlagUtil.isEnabled(s.getDB('admin'), "BalanceAccordingToDataSize"); -if (!balanceAccordingToDataSize) { - assert.soon(function() { - var d1Chunks = - findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo', {shard: "d1"}); - var d2Chunks = - findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo', {shard: "d2"}); - var totalChunks = findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo'); +assert.soon(function() { + var d1Chunks = findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo', {shard: "d1"}); + var d2Chunks = findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo', {shard: "d2"}); + var totalChunks = findChunksUtil.countChunksForNs(s.getDB("config"), 'test.foo'); - print("chunks: " + d1Chunks + " " + d2Chunks + " " + totalChunks); + print("chunks: " + d1Chunks + " " + d2Chunks + " " + totalChunks); - return d1Chunks > 0 && d2Chunks > 0 && (d1Chunks + d2Chunks == totalChunks); - }, "Chunks failed to balance", 60000, 5000); -} + return d1Chunks > 0 && d2Chunks > 0 && (d1Chunks + d2Chunks == totalChunks); +}, "Chunks failed to balance", 60000, 5000); // SERVER-33753: count() without predicate can be wrong on sharded collections. // assert.eq(s.getDB("test").foo.count(), num+1); @@ -242,7 +234,7 @@ if (numDocs != num) { // This call also waits for any ongoing balancing to stop s.stopBalancer(60000); -var cursor = s.getDB("test").foo.find({x: {$lt: 5}}); +var cursor = s.getDB("test").foo.find({x: {$lt: 500}}); var count = 0; while (cursor.hasNext()) { @@ -250,7 +242,7 @@ while (cursor.hasNext()) { count++; } -assert.eq(count, 5); +assert.eq(count, 500); logout(adminUser); diff --git a/jstests/sharding/authCommands.js b/jstests/sharding/authCommands.js index d1059db00d1..7d78366b99e 100644 --- a/jstests/sharding/authCommands.js +++ b/jstests/sharding/authCommands.js @@ -88,8 +88,12 @@ st.startBalancer(); // Make sure we've done at least some splitting, so the balancer will work assert.gt(findChunksUtil.findChunksByNs(configDB, 'test.foo').count(), 2); -// Make sure we eventually balance the 'test.foo' collection -st.awaitBalance('foo', 'test', 60 * 5 * 1000); +// Make sure we eventually balance all the chunks we've created +assert.soon(function() { + var x = st.chunkDiff("foo", "test"); + print("chunk diff: " + x); + return x < 2 && configDB.locks.findOne({_id: 'test.foo'}).state == 0; +}, "no balance happened", 15 * 60 * 1000); var map = function() { emit(this.i, this.j); diff --git a/jstests/sharding/auto_rebalance_parallel.js b/jstests/sharding/auto_rebalance_parallel.js index d9f2f00715e..1e55e8cbc11 100644 --- a/jstests/sharding/auto_rebalance_parallel.js +++ b/jstests/sharding/auto_rebalance_parallel.js @@ -1,8 +1,5 @@ /** * Tests that the cluster is balanced in parallel in one balancer round (standalone). - * @tags: [ - * requires_fcv_60, - * ] */ (function() { @@ -10,7 +7,7 @@ load("jstests/sharding/libs/find_chunks_util.js"); -const st = new ShardingTest({shards: 4, other: {chunkSize: 1, enableAutoSplitter: false}}); +var st = new ShardingTest({shards: 4}); var config = st.s0.getDB('config'); assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'})); @@ -21,34 +18,35 @@ function prepareCollectionForBalance(collName) { var coll = st.s0.getCollection(collName); - const bigString = 'X'.repeat(1024 * 1024); // 1MB - // Create 6 chunks initially and ensure they get balanced within 1 balancer round - assert.commandWorked(coll.insert({Key: 1, Value: 'Test value 1', s: bigString})); - assert.commandWorked(coll.insert({Key: 10, Value: 'Test value 10', s: bigString})); - assert.commandWorked(coll.insert({Key: 20, Value: 'Test value 20', s: bigString})); - assert.commandWorked(coll.insert({Key: 30, Value: 'Test value 30', s: bigString})); - assert.commandWorked(coll.insert({Key: 40, Value: 'Test value 40', s: bigString})); - assert.commandWorked(coll.insert({Key: 50, Value: 'Test value 50', s: bigString})); + // Create 4 chunks initially and ensure they get balanced within 1 balancer round + assert.commandWorked(coll.insert({Key: 1, Value: 'Test value 1'})); + assert.commandWorked(coll.insert({Key: 10, Value: 'Test value 10'})); + assert.commandWorked(coll.insert({Key: 20, Value: 'Test value 20'})); + assert.commandWorked(coll.insert({Key: 30, Value: 'Test value 30'})); assert.commandWorked(st.splitAt(collName, {Key: 10})); assert.commandWorked(st.splitAt(collName, {Key: 20})); assert.commandWorked(st.splitAt(collName, {Key: 30})); - assert.commandWorked(st.splitAt(collName, {Key: 40})); - assert.commandWorked(st.splitAt(collName, {Key: 50})); - // Move 3 of the chunks to st.shard1.shardName so we have option to do parallel balancing + // Move two of the chunks to st.shard1.shardName so we have option to do parallel balancing + assert.commandWorked(st.moveChunk(collName, {Key: 20}, st.shard1.shardName)); assert.commandWorked(st.moveChunk(collName, {Key: 30}, st.shard1.shardName)); - assert.commandWorked(st.moveChunk(collName, {Key: 40}, st.shard1.shardName)); - assert.commandWorked(st.moveChunk(collName, {Key: 50}, st.shard1.shardName)); assert.eq( - 3, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard0.shardName}).itcount()); + 2, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard0.shardName}).itcount()); assert.eq( - 3, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard1.shardName}).itcount()); + 2, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard1.shardName}).itcount()); } function checkCollectionBalanced(collName) { - st.verifyCollectionIsBalanced(st.s.getCollection(collName)); + assert.eq( + 1, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard0.shardName}).itcount()); + assert.eq( + 1, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard1.shardName}).itcount()); + assert.eq( + 1, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard2.shardName}).itcount()); + assert.eq( + 1, findChunksUtil.findChunksByNs(config, collName, {shard: st.shard3.shardName}).itcount()); } function countMoves(collName) { @@ -64,8 +62,8 @@ const testColl1InitialMoves = countMoves('TestDB.TestColl1'); const testColl2InitialMoves = countMoves('TestDB.TestColl2'); st.startBalancer(); -st.awaitBalance("TestColl1", "TestDB"); -st.awaitBalance("TestColl2", "TestDB"); +st.waitForBalancer(true, 60000); +st.waitForBalancer(true, 60000); st.stopBalancer(); checkCollectionBalanced('TestDB.TestColl1'); diff --git a/jstests/sharding/autosplit.js b/jstests/sharding/autosplit.js index e2242b45bcb..af67d7820cb 100644 --- a/jstests/sharding/autosplit.js +++ b/jstests/sharding/autosplit.js @@ -5,7 +5,6 @@ 'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled var s = new ShardingTest({ name: "auto1", @@ -14,15 +13,6 @@ var s = new ShardingTest({ other: {enableAutoSplit: true, chunkSize: 10}, }); -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(s.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - s.stop(); - return; -} - assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', s.shard1.shardName); assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {num: 1}})); diff --git a/jstests/sharding/autosplit_configure_collection.js b/jstests/sharding/autosplit_configure_collection.js index 83acde55afd..c1658add12f 100644 --- a/jstests/sharding/autosplit_configure_collection.js +++ b/jstests/sharding/autosplit_configure_collection.js @@ -11,7 +11,6 @@ 'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled let st = new ShardingTest({ name: "auto1", @@ -20,15 +19,6 @@ let st = new ShardingTest({ other: {enableAutoSplit: true}, }); -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - st.stop(); - return; -} - const fullNS = "test.foo"; const bigString = "X".repeat(1024 * 1024 / 16); // 65 KB diff --git a/jstests/sharding/autosplit_low_cardinality.js b/jstests/sharding/autosplit_low_cardinality.js index d6abe0d807b..1f944fc2140 100644 --- a/jstests/sharding/autosplit_low_cardinality.js +++ b/jstests/sharding/autosplit_low_cardinality.js @@ -8,22 +8,12 @@ 'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled var st = new ShardingTest({ name: "low_cardinality", other: {enableAutoSplit: true, chunkSize: 1}, }); -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - st.stop(); - return; -} - assert.commandWorked(st.s.adminCommand({enablesharding: "test"})); assert.commandWorked(st.s.adminCommand({shardcollection: "test.foo", key: {sk: 1}})); diff --git a/jstests/sharding/awaitable_hello_primary_failures.js b/jstests/sharding/awaitable_hello_primary_failures.js index 1522f755347..b6a387920c3 100644 --- a/jstests/sharding/awaitable_hello_primary_failures.js +++ b/jstests/sharding/awaitable_hello_primary_failures.js @@ -22,9 +22,9 @@ let rsPrimary = st.rs0.getPrimary(); // Make sure mongos knows who the primary is awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: true, ismaster: true}); -// Turn on the waitInHello failpoint. This will cause the primary node to cease sending "hello" +// Turn on the waitInHello failpoint. This will cause the primary node to cease sending isMaster // responses and the RSM should mark the node as down -jsTestLog("Turning on waitInHello failpoint. Node should stop sending hello responses."); +jsTestLog("Turning on waitInHello failpoint. Node should stop sending isMaster responses."); const helloFailpoint = configureFailPoint(rsPrimary, "waitInHello"); awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: false, ismaster: false}); helloFailpoint.off(); @@ -32,26 +32,25 @@ helloFailpoint.off(); // Wait for mongos to find out the node is still primary awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: true, ismaster: true}); -// Force the primary node to fail all "hello" requests. The RSM should mark the node as down. -jsTestLog("Turning on failCommand failpoint. Node should fail all hello/isMaster responses."); -const failCmdFailpoint = configureFailPoint(rsPrimary, "failCommand", { - errorCode: ErrorCodes.CommandFailed, - failCommands: ["hello", "isMaster"], - failInternalCommands: true -}); +// Force the primary node to fail all isMaster requests. The RSM should mark the node as down. +jsTestLog("Turning on failCommand failpoint. Node should fail all isMaster responses."); +const failCmdFailpoint = configureFailPoint( + rsPrimary, + "failCommand", + {errorCode: ErrorCodes.CommandFailed, failCommands: ["isMaster"], failInternalCommands: true}); awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: false, ismaster: false}); failCmdFailpoint.off(); // Wait for mongos to find out the node is still primary awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: true, ismaster: true}); -// Force the primary node to end the "hello" stream by not setting the 'moreToCome' bit on the +// Force the primary node to end the isMaster stream by not setting the 'moreToCome' bit on the // resposne. The RSM should not mark the server as down or unknown and should continue monitoring // the node. jsTestLog( - "Turning on doNotSetMoreToCome failpoint. Node should return successful hello responses."); + "Turning on doNotSetMoreToCome failpoint. Node should return successful isMaster responses."); const moreToComeFailpoint = configureFailPoint(rsPrimary, "doNotSetMoreToCome"); -// Wait for maxAwaitTimeMS to guarantee that mongos has received at least one "hello" response from +// Wait for maxAwaitTimeMS to guarantee that mongos has received at least one isMaster response from // the primary without the moreToCome bit set. sleep(10000); awaitRSClientHosts(mongos, {host: rsPrimary.name}, {ok: true, ismaster: true}); diff --git a/jstests/sharding/balance_random_data_distribution.js b/jstests/sharding/balance_random_data_distribution.js deleted file mode 100644 index 36c58f47ab9..00000000000 --- a/jstests/sharding/balance_random_data_distribution.js +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Test that the balancer redistributes data from multiple tracked collections across the - * cluster and it is able to converge within a limited amount of time. - * (Data amount & distribution, as well as per-collection maxChunkSize, are randomly chosen). - * - * @tags: [ - * requires_fcv_60, - * does_not_support_stepdowns, # TODO SERVER-89797 remove this tag. - * ] - * */ - -(function() { -"use strict"; - -load("jstests/libs/parallel_shell_helpers.js"); - -const numShards = 2; -Random.setRandomSeed(); - -const st = new ShardingTest({shards: numShards}); - -const clusterMaxChunkSizeMB = 8; -const collectionBalancedTimeoutMS = 10 * 60 * 1000 /* 10min */; - -const numDatabases = numShards; -const numCollInDB = 3; -const dbNamePrefix = 'test_db_'; -const collNamePrefix = 'coll_'; - -// 1. Setup an initial set of collections. -for (let i = 0; i < numDatabases; ++i) { - const dbName = dbNamePrefix + `${i}`; - const primaryShardId = st[`shard${i}`].shardName; - // Avoid assigning the same primary shard for every collection. - assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: primaryShardId})); - for (let j = 0; j < numCollInDB; ++j) { - let collName = collNamePrefix + `${j}`; - const coll = st.s.getDB(dbName)[collName]; - const ns = coll.getFullName(); - // Use {_id: 1} as shard key to allow room for imbalance as documents get later inserted. - st.s.adminCommand({shardCollection: ns, key: {_id: 1}}); - const collMaxChunkSizeMB = Random.randInt(clusterMaxChunkSizeMB - 1) + 1; - assert.commandWorked(st.s.adminCommand({ - configureCollectionBalancing: ns, - chunkSize: collMaxChunkSizeMB, - })); - } -} - -// 2. Launch the balancer and start multiple workers inserting random data into the existing -// collections. -st.startBalancer(); - -function doBatchInserts( - numDatabases, dbNamePrefix, numCollInDB, collNamePrefix, clusterMaxChunkSizeMB) { - Random.setRandomSeed(); - const numOfBatchInserts = 8; - const bigString = - 'X'.repeat(1024 * 1024 - 30); // Almost 1MB, to create documents of exactly 1MB - - for (let i = 0; i < numOfBatchInserts; ++i) { - const dbName = dbNamePrefix + `${Random.randInt(numDatabases)}`; - let randomDB = db.getSiblingDB(dbName); - const collName = collNamePrefix + `${Random.randInt(numCollInDB)}`; - const coll = randomDB[collName]; - - const numDocs = Random.randInt(clusterMaxChunkSizeMB - 1) + 1; - let insertBulkOp = coll.initializeUnorderedBulkOp(); - for (let i = 0; i < numDocs; ++i) { - insertBulkOp.insert({s: bigString}); - } - - assert.commandWorked(insertBulkOp.execute()); - } -} - -const numBackgroundBatchInserters = 5; -let backgroundBatchInserters = []; -for (let i = 0; i < numBackgroundBatchInserters; ++i) { - backgroundBatchInserters.push(startParallelShell(funWithArgs(doBatchInserts, - numDatabases, - dbNamePrefix, - numCollInDB, - collNamePrefix, - clusterMaxChunkSizeMB), - st.s.port)); -} - -// 3. Once the insertion workers are done, verify that the balancer may bring each tracked -// collection to a "balanced" state within the deadline. -for (let joinInserter of backgroundBatchInserters) { - joinInserter(); -} - -let testedAtLeastOneCollection = false; -for (let i = 0; i < numDatabases; i++) { - const dbName = dbNamePrefix + `${i}`; - for (let j = 0; j < numCollInDB; j++) { - const ns = dbName + '.' + collNamePrefix + `${j}`; - - const coll = st.s.getCollection(ns); - if (coll.countDocuments({}) === 0) { - // Skip empty collections - continue; - } - testedAtLeastOneCollection = true; - - // Wait for collection to be considered balanced - sh.awaitCollectionBalance(coll, collectionBalancedTimeoutMS, 1000 /* 1s interval */); - sh.verifyCollectionIsBalanced(coll); - } - - assert(testedAtLeastOneCollection); -} - -st.stop(); -}()); diff --git a/jstests/sharding/balancer_collection_status.js b/jstests/sharding/balancer_collection_status.js index e2bb502dc04..25622820605 100644 --- a/jstests/sharding/balancer_collection_status.js +++ b/jstests/sharding/balancer_collection_status.js @@ -6,6 +6,7 @@ 'use strict'; const chunkSizeMB = 1; + let st = new ShardingTest({ shards: 3, other: { @@ -54,10 +55,6 @@ assert.eq(result.balancerCompliant, true); // get shardIds const shards = st.s0.getDB('config').shards.find().toArray(); -const bigString = 'X'.repeat(1024 * 1024); // 1MB -for (var i = 0; i < 30; i += 10) { - assert.commandWorked(st.s0.getDB('db').getCollection('col').insert({key: i, s: bigString})); -} // manually split and place the 3 chunks on the same shard assert.commandWorked(st.s0.adminCommand({split: 'db.col', middle: {key: 10}})); assert.commandWorked(st.s0.adminCommand({split: 'db.col', middle: {key: 20}})); diff --git a/jstests/sharding/balancer_warns_draining_shards_blocked.js b/jstests/sharding/balancer_warns_draining_shards_blocked.js deleted file mode 100644 index 18af7a8cf59..00000000000 --- a/jstests/sharding/balancer_warns_draining_shards_blocked.js +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Verifies the balancer emits a warning when a removed shard cannot be drained due to balancing - * being disabled. There are two cases: - * - Balancer is disabled. - * - Balancing is disabled for a collection, which has chunks on draining shards. - */ - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallel_shell_helpers.js"); -load('jstests/sharding/libs/find_chunks_util.js'); - -const st = new ShardingTest({ - shards: 2, - config: 1, - rs: {nodes: 1}, - other: { - enableBalancer: false, - } -}); - -const mongos = st.s0; -const configDB = st.getDB('config'); - -const dbName = 'test'; -const collName = 'collToDrain'; -const ns = dbName + '.' + collName; -const testDB = st.getDB(dbName); -const coll = testDB.getCollection(collName); - -// Shard collection with shard0 as db primary. -assert.commandWorked( - mongos.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); -assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {x: 1}})); - -assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); -assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: -1}, to: st.shard0.shardName})); -assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: 1}, to: st.shard1.shardName})); - -// Check that there are only 2 chunks before starting draining. -const chunksBeforeDrain = findChunksUtil.findChunksByNs(configDB, ns).toArray(); -assert.eq(2, chunksBeforeDrain.length); - -// Force checks to happen continuously to speedup the test. -const csrsPrimary = st.configRS.getPrimary(); -configureFailPoint(csrsPrimary, "forceBalancerWarningChecks"); - -let awaitRemoveShard = startParallelShell(funWithArgs(function(shardName) { - load("jstests/sharding/libs/remove_shard_util.js"); - removeShard(db, shardName); - }, st.shard1.shardName), st.s.port); - -// Test warning when the balancer is disabled. -// "Draining of removed shards cannot be completed because the balancer is disabled" -checkLog.containsJson(csrsPrimary, 6434000); - -sh.disableBalancing(coll); -st.startBalancer(); - -// Test warning when the balancer is enabled, but a specific collection is disabled. -// "Draining of removed shards cannot be completed because the balancer is disabled for a collection -// which has chunks in those shards". -checkLog.containsJson(csrsPrimary, 7977400); - -// Re-enable balancing to let test terminate. -sh.enableBalancing(coll); -awaitRemoveShard(); - -st.stop(); diff --git a/jstests/sharding/balancer_window.js b/jstests/sharding/balancer_window.js index 0e93d967363..ee48db64844 100644 --- a/jstests/sharding/balancer_window.js +++ b/jstests/sharding/balancer_window.js @@ -45,23 +45,17 @@ var HourAndMinute = function(hour, minutes) { }; }; -const st = new ShardingTest({shards: 2, other: {chunkSize: 1, enableAutoSplit: false}}); -const dbName = 'test'; -const collName = 'user'; -const ns = dbName + '.' + collName; -const configDB = st.s.getDB('config'); -assert.commandWorked(configDB.adminCommand({enableSharding: dbName})); -assert.commandWorked(configDB.adminCommand({shardCollection: ns, key: {_id: 1}})); +var st = new ShardingTest({shards: 2}); +var configDB = st.s.getDB('config'); +assert.commandWorked(configDB.adminCommand({enableSharding: 'test'})); +assert.commandWorked(configDB.adminCommand({shardCollection: 'test.user', key: {_id: 1}})); -const bigString = 'X'.repeat(1024 * 1024); // 1MB -const coll = st.s.getDB(dbName).getCollection(collName); for (var x = 0; x < 150; x += 10) { - coll.insert({_id: x, s: bigString}); - configDB.adminCommand({split: ns, middle: {_id: x}}); + configDB.adminCommand({split: 'test.user', middle: {_id: x}}); } var shard0Chunks = - findChunksUtil.findChunksByNs(configDB, ns, {shard: st.shard0.shardName}).count(); + findChunksUtil.findChunksByNs(configDB, 'test.user', {shard: st.shard0.shardName}).count(); var startDate = new Date(); var hourMinStart = new HourAndMinute(startDate.getHours(), startDate.getMinutes()); @@ -78,10 +72,10 @@ assert.commandWorked( true)); st.startBalancer(); -st.awaitBalancerRound(); +st.waitForBalancer(true, 60000); var shard0ChunksAfter = - findChunksUtil.findChunksByNs(configDB, ns, {shard: st.shard0.shardName}).count(); + findChunksUtil.findChunksByNs(configDB, 'test.user', {shard: st.shard0.shardName}).count(); assert.eq(shard0Chunks, shard0ChunksAfter); assert.commandWorked(configDB.settings.update( @@ -93,10 +87,10 @@ assert.commandWorked(configDB.settings.update( }, true)); -st.awaitBalancerRound(); +st.waitForBalancer(true, 60000); shard0ChunksAfter = - findChunksUtil.findChunksByNs(configDB, ns, {shard: st.shard0.shardName}).count(); + findChunksUtil.findChunksByNs(configDB, 'test.user', {shard: st.shard0.shardName}).count(); assert.neq(shard0Chunks, shard0ChunksAfter); st.stop(); diff --git a/jstests/sharding/balancing_based_on_size.js b/jstests/sharding/balancing_based_on_size.js deleted file mode 100644 index 60e5dec3b7f..00000000000 --- a/jstests/sharding/balancing_based_on_size.js +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Test that the balancer is redistributing data based on the actual amount of data - * for a collection on each node, converging when the size difference becomes small. - * - * @tags: [ - * featureFlagBalanceAccordingToDataSize, - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; - -load("jstests/sharding/libs/find_chunks_util.js"); - -const maxChunkSizeMB = 1; -const st = new ShardingTest( - {shards: 2, mongos: 1, other: {chunkSize: maxChunkSizeMB, enableBalancer: false}}); -const dbName = 'test'; -const coll = st.getDB(dbName).getCollection('foo'); -const ns = coll.getFullName(); -const mongos = st.s; -const shard0 = st.shard0.shardName; -const shard1 = st.shard1.shardName; - -// Shard collection with one chunk on shard0 [MinKey, 0) and one chunk on shard1 [0, MinKey) -assert.commandWorked(mongos.adminCommand({enablesharding: dbName, primaryShard: shard0})); -assert.commandWorked(mongos.adminCommand({shardcollection: ns, key: {_id: 1}})); -assert.commandWorked(mongos.adminCommand({split: ns, middle: {_id: 0}})); -assert.commandWorked(mongos.adminCommand({moveChunk: ns, find: {_id: 0}, to: shard1})); - -const bigString = 'X'.repeat(1024 * 1024); // 1MB - -// Insert 10MB of documents in range [MinKey, 0) on shard0 -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = -1; i > -11; i--) { - bulk.insert({_id: i, s: bigString}); -} -assert.commandWorked(bulk.execute()); - -// Insert 3MB of documents in range [0, MaxKey) on shard1 -bulk = coll.initializeUnorderedBulkOp(); -for (var i = 0; i < 3; i++) { - bulk.insert({_id: i, s: bigString}); -} -assert.commandWorked(bulk.execute()); - -// Create 3 more chunks on shard0 -assert.commandWorked(mongos.adminCommand({split: ns, middle: {_id: 1}})); -assert.commandWorked(mongos.adminCommand({split: ns, middle: {_id: 2}})); -assert.commandWorked(mongos.adminCommand({split: ns, middle: {_id: 3}})); - -// At this point, the distribution of chunks for the testing collection is the following: -// - On shard0 (10MB): -// { "_id" : { "$minKey" : 1 } } -->> { "_id" : 0 } -// - On shard1 (3MB): -// { "_id" : 0 } -->> { "_id" : 1 } -// { "_id" : 1 } -->> { "_id" : 2 } -// { "_id" : 2 } -->> { "_id" : 3 } -// { "_id" : 3 } -->> { "_id" : { "$maxKey" : 1 } } -jsTestLog("Printing sharding status before starting balancer"); -st.printShardingStatus(); -st.startBalancer(); - -st.awaitCollectionBalance(coll, 60000 /* timeout */, 1000 /* interval */); -const chunksBeforeNoopRound = findChunksUtil.findChunksByNs(st.config, ns).toArray(); - -// Check that the collection is balanced -st.verifyCollectionIsBalanced(coll); - -// Wait for some more rounds and then check the balancer is not wrongly moving around data -st.forEachConfigServer((conn) => { - conn.adminCommand({ - configureFailPoint: 'overrideBalanceRoundInterval', - mode: 'alwaysOn', - data: {intervalMs: 100} - }); -}); - -st.awaitBalancerRound(); -st.awaitBalancerRound(); -st.awaitBalancerRound(); - -st.stopBalancer(); -jsTestLog("Printing sharding status after stopping balancer"); -st.printShardingStatus(); - -// Check that no move has been performed during the noop rounds (if the routing table did not -// change, it means data are still balanced) -const chunksAfterNoopRound = findChunksUtil.findChunksByNs(st.config, ns).toArray(); -assert.eq(chunksBeforeNoopRound, chunksAfterNoopRound); - -st.stop(); -})(); diff --git a/jstests/sharding/balancing_sessions_collection_legacy.js b/jstests/sharding/balancing_sessions_collection.js index 230235a7a47..231f633dead 100644 --- a/jstests/sharding/balancing_sessions_collection_legacy.js +++ b/jstests/sharding/balancing_sessions_collection.js @@ -6,7 +6,6 @@ (function() { "use strict"; -load("jstests/libs/feature_flag_util.js"); load("jstests/sharding/libs/find_chunks_util.js"); // TODO SERVER-50144 Remove this and allow orphan checking. @@ -113,14 +112,6 @@ const st = new ShardingTest({ shards: numShards, other: {configOptions: {setParameter: {minNumChunksForSessionsCollection: kMinNumChunks}}} }); - -// TODO SERVER-66782 delete this file -if (FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), - "BalanceAccordingToDataSize")) { - jsTestLog("Skipping as featureFlagBalanceAccordingToDataSize is enabled"); - st.stop(); - return; -} const kSessionsNs = "config.system.sessions"; const configDB = st.s.getDB("config"); diff --git a/jstests/sharding/basic_split.js b/jstests/sharding/basic_split.js index f401b8d0047..c4a819ae026 100644 --- a/jstests/sharding/basic_split.js +++ b/jstests/sharding/basic_split.js @@ -91,10 +91,6 @@ assert.neq(null, findChunksUtil.findOneChunkByNs(configDB, 'test.compound', {min // cannot split on existing chunk boundary. assert.commandFailed(configDB.adminCommand({split: 'test.compound', middle: {x: 0, y: 0}})); -assert.commandFailed( - configDB.adminCommand({split: 'test.compound', middle: {x: MinKey, y: MinKey}})); -assert.commandFailed( - configDB.adminCommand({split: 'test.compound', middle: {x: MaxKey, y: MaxKey}})); bulk = testDB.compound.initializeUnorderedBulkOp(); for (x = -1200; x < 1200; x++) { diff --git a/jstests/sharding/block_chunk_migrations_without_hashed_shard_key_index.js b/jstests/sharding/block_chunk_migrations_without_hashed_shard_key_index.js deleted file mode 100644 index 7073538492b..00000000000 --- a/jstests/sharding/block_chunk_migrations_without_hashed_shard_key_index.js +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Tests that chunk migrations are blocked when there is no index on a hashed shard key. - * - * @tags: [ - * requires_fcv_50, - * ] - */ - -(function() { -"use strict"; - -load("jstests/sharding/libs/find_chunks_util.js"); - -const st = new ShardingTest({}); - -const dbName = "testDb"; -const collName = "testColl"; -const nss = dbName + "." + collName; -const configDB = st.s.getDB('config'); - -const coll = st.getDB(dbName).getCollection(collName); -const kDataString = "a".repeat(1024 * 1024); -let docs = Array.from({length: 1000}, (_, i) => ({_id: i, field: kDataString})); - -assert.commandWorked( - st.s.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); -assert.commandWorked(coll.createIndex({"_id": "hashed"})); -assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: "hashed"}})); - -// Move all chunks to a single shard so the balancer is triggered due to data imbalance. -let chunks = findChunksUtil.findChunksByNs(configDB, nss).toArray(); -chunks.forEach(chunk => { - st.s.adminCommand({moveChunk: nss, bounds: [chunk.min, chunk.max], to: st.shard0.shardName}); -}); - -assert.eq(0, findChunksUtil.findChunksByNs(configDB, nss, {shard: st.shard1.shardName}).itcount()); - -assert.commandWorked(coll.insert(docs)); -assert.commandWorked(coll.dropIndex({"_id": "hashed"})); - -st.startBalancer(); -st.awaitBalancerRound(); - -// During balancing, the balancer should catch the IndexNotFound and turn off the balancer for the -// collection by setting {noBalance : true}. -assert.soon(() => { - return configDB.getCollection('collections').findOne({_id: nss}).noBalance === true; -}); - -// Confirm all chunks remain on shard0. -assert.eq(0, findChunksUtil.findChunksByNs(configDB, nss, {shard: st.shard1.shardName}).itcount()); - -// Commands that trigger chunk migrations should fail with IndexNotFound. -assert.commandFailedWithCode( - st.s.adminCommand( - {moveChunk: nss, bounds: [chunks[0].min, chunks[0].max], to: st.shard1.shardName}), - ErrorCodes.IndexNotFound); - -// moveRange is not present in 5.0 so skip this assertion if FCV is lesser than 6.0 in multi-version -// test suite. -if (st.rs0.getPrimary().getDB('admin').system.version.findOne( - {_id: 'featureCompatibilityVersion'}) == getFCVConstants().latest && - st.rs1.getPrimary().getDB('admin').system.version.findOne( - {_id: 'featureCompatibilityVersion'}) == getFCVConstants().latest) { - assert.commandFailedWithCode( - st.s.adminCommand( - {moveRange: nss, toShard: st.shard1.shardName, min: chunks[0].min, max: chunks[0].max}), - ErrorCodes.IndexNotFound); -} - -// Recreate the index and verify that we can re-enable balancing. -assert.commandWorked(coll.createIndex({"_id": "hashed"})); -st.enableBalancing(nss); - -assert.eq(false, configDB.getCollection('collections').findOne({_id: nss}).noBalance); -st.awaitBalancerRound(); -assert.soon(() => { - return findChunksUtil.findChunksByNs(configDB, nss, {shard: st.shard1.shardName}).itcount() > 0; -}); - -st.stop(); -})(); diff --git a/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js b/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js index 0ea93b8ae2a..91af933d10d 100644 --- a/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js +++ b/jstests/sharding/cancel_coordinate_txn_commit_with_tickets_exhausted.js @@ -26,7 +26,6 @@ load("jstests/libs/fail_point_util.js"); load('jstests/libs/parallelTester.js'); load("jstests/sharding/libs/create_sharded_collection_util.js"); -load("jstests/libs/auto_retry_transaction_in_sharding.js"); // For withTxnAndAutoRetryOnMongos. const kNumWriteTickets = 10; const st = new ShardingTest({ @@ -83,10 +82,10 @@ const sessionCollection = session.getDatabase(dbName).getCollection(collName); // transactionThread won't need to persist a topology time. The scenario reported in SERVER-60685 // depended on the TransactionCoordinator being interrupted while persisting the participant list // which happens after waiting for the topology time to become durable. -withTxnAndAutoRetryOnMongos(session, () => { - assert.commandWorked(sessionCollection.insert({key: 400})); - assert.commandWorked(sessionCollection.insert({key: -400})); -}); +session.startTransaction(); +assert.commandWorked(sessionCollection.insert({key: 400})); +assert.commandWorked(sessionCollection.insert({key: -400})); +assert.commandWorked(session.commitTransaction_forTesting()); const hangWithLockDuringBatchRemoveFp = configureFailPoint(txnCoordinator, failpointName); diff --git a/jstests/sharding/catalog_cache_refresh_counters.js b/jstests/sharding/catalog_cache_refresh_counters.js new file mode 100644 index 00000000000..028ef2f2c59 --- /dev/null +++ b/jstests/sharding/catalog_cache_refresh_counters.js @@ -0,0 +1,114 @@ +/** + * Test that operations that get blocked by catalog cache refreshes get logged to + * shardingStatistics. + */ + +(function() { +'use strict'; + +load('jstests/sharding/libs/sharded_transactions_helpers.js'); +load("jstests/sharding/libs/chunk_bounds_util.js"); +load("jstests/sharding/libs/find_chunks_util.js"); + +let st = new ShardingTest({mongos: 2, shards: 2}); +const configDB = st.s.getDB('config'); +const dbName = "test"; +const collName = "foo"; +const ns = dbName + "." + collName; +const mongos0DB = st.s0.getDB(dbName); +const mongos1DB = st.s1.getDB(dbName); +const mongos0Coll = mongos0DB[collName]; +const mongos1Coll = mongos1DB[collName]; + +let setUp = () => { + /** + * Sets up a test by moving chunks to such that one chunk is on each + * shard, with the following distribution: + * shard0: [-inf, 0) + * shard1: [0, inf) + */ + assert.commandWorked(st.s0.adminCommand({enableSharding: dbName})); + assert.commandWorked(st.s0.adminCommand({movePrimary: dbName, to: st.shard0.shardName})); + assert.commandWorked(st.s0.adminCommand({shardCollection: ns, key: {x: 1}})); + assert.commandWorked(st.s0.adminCommand({split: ns, middle: {x: 0}})); + + flushRoutersAndRefreshShardMetadata(st, {ns}); +}; + +let getOpsBlockedByRefresh = () => { + return assert.commandWorked(st.s1.adminCommand({serverStatus: 1})) + .shardingStatistics.catalogCache.operationsBlockedByRefresh; +}; + +let verifyBlockedOperationsChange = (oldOperationsCount, increasedOps) => { + let newOperationsCount = getOpsBlockedByRefresh(); + increasedOps.forEach(op => { + assert.eq(newOperationsCount[op.opType], oldOperationsCount[op.opType] + op.increase); + }); +}; + +let getShardToTargetForMoveChunk = () => { + const chunkDocs = findChunksUtil.findChunksByNs(configDB, ns).toArray(); + const shardChunkBounds = chunkBoundsUtil.findShardChunkBounds(chunkDocs); + const shardThatOwnsChunk = chunkBoundsUtil.findShardForShardKey(st, shardChunkBounds, {x: 100}); + return st.getOther(shardThatOwnsChunk).shardName; +}; + +let runTest = (operationToRunFn, expectedOpIncreases) => { + let opsBlockedByRefresh = getOpsBlockedByRefresh(); + + // Move chunk to ensure stale shard version for the next operation. + assert.commandWorked( + st.s0.adminCommand({moveChunk: ns, find: {x: 100}, to: getShardToTargetForMoveChunk()})); + + operationToRunFn(); + verifyBlockedOperationsChange(opsBlockedByRefresh, expectedOpIncreases); +}; + +setUp(); + +/** + * Verify that insert operations get logged when blocked by a refresh. + */ +runTest(() => assert.commandWorked(mongos1Coll.insert({x: 250})), [ + {opType: 'countAllOperations', increase: 2}, + {opType: 'countInserts', increase: 1}, + {opType: 'countCommands', increase: 1} +]); + +/** + * Verify that queries get logged when blocked by a refresh. + */ +runTest(() => mongos1Coll.findOne({x: 250}), + [{opType: 'countAllOperations', increase: 1}, {opType: 'countQueries', increase: 1}]); + +/** + * Verify that updates get logged when blocked by a refresh. + */ +runTest(() => assert.commandWorked(mongos1Coll.update({x: 250}, {$set: {a: 1}})), [ + {opType: 'countAllOperations', increase: 2}, + {opType: 'countUpdates', increase: 1}, + {opType: 'countCommands', increase: 1} +]); + +/** + * Verify that deletes get logged when blocked by a refresh. + */ +runTest(() => assert.commandWorked(mongos1Coll.remove({x: 250})), [ + {opType: 'countAllOperations', increase: 2}, + {opType: 'countDeletes', increase: 1}, + {opType: 'countCommands', increase: 1} +]); + +/** + * Verify that non-CRUD commands get logged when blocked by a refresh. + */ +runTest(() => assert.commandWorked(mongos1DB.runCommand( + {createIndexes: collName, indexes: [{key: {a: 1}, name: 'index'}]})), + [ + {opType: 'countAllOperations', increase: 1}, + {opType: 'countCommands', increase: 1}, + ]); + +st.stop(); +})(); diff --git a/jstests/sharding/change_stream_lookup_single_shard_cluster.js b/jstests/sharding/change_stream_lookup_single_shard_cluster.js index fc1ae516f8c..e9ae2492ba7 100644 --- a/jstests/sharding/change_stream_lookup_single_shard_cluster.js +++ b/jstests/sharding/change_stream_lookup_single_shard_cluster.js @@ -2,8 +2,6 @@ // cluster only has a single shard, and that it can therefore successfully look up a document in a // sharded collection. // @tags: [ -// # If a rollback is triggered during a stepdown, the change stream cursor can become invalid. -// does_not_support_stepdowns, // requires_majority_read_concern, // uses_change_streams, // bf25011, diff --git a/jstests/sharding/change_stream_no_drop.js b/jstests/sharding/change_stream_no_drop.js deleted file mode 100644 index d378421cd18..00000000000 --- a/jstests/sharding/change_stream_no_drop.js +++ /dev/null @@ -1,62 +0,0 @@ -/** - * DDL coordinator are responsible for dropping temporary collections, especially after failures. - * However, the change stream should not be aware of those events. - * @tags: [ - * # Requires all nodes to be running the latest binary. - * multiversion_incompatible, - * ] - */ -function assertNoDrop(changeStream) { - while (changeStream.hasNext()) { - assert.neq(changeStream.next().operationType, 'drop'); - } -} - -function emptyChangeStream(changeStream) { - while (changeStream.hasNext()) { - changeStream.next(); - } -} - -(function() { - -const dbName = 'db'; - -load('jstests/libs/fail_point_util.js'); // For configureFailPoint - -// Enable explicitly the periodic no-op writer to allow the router to process change stream events -// coming from all shards. This is enabled for production clusters by default. -const st = new ShardingTest({ - mongos: 1, - config: 1, - shards: 2, - rs: {nodes: 1, setParameter: {writePeriodicNoops: true, periodicNoopIntervalSecs: 1}}, - other: {enableBalancer: true} -}); - -// create a database and a change stream on it -jsTest.log('Creating a change stream on ' + dbName); -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); -let changeStream = st.s.getDB('db').watch(); - -// setFeatureCompatibilityVersion might cause dropping of deprecated collections -emptyChangeStream(changeStream); - -jsTest.log( - 'The shard_collection_coordinator at second attempt (after failure) should not report drop events for orphaned'); -{ - configureFailPoint(st.shard0, - 'failAtCommitCreateCollectionCoordinator', - data = {}, - failPointMode = {times: 1}); - - collectionName = dbName + '.coll'; - assert.commandWorked(st.s.adminCommand( - {shardCollection: collectionName, key: {_id: "hashed"}, numInitialChunks: 10})); - - assertNoDrop(changeStream); -} - -st.stop(); -}()); diff --git a/jstests/sharding/change_streams_shards_start_in_sync.js b/jstests/sharding/change_streams_shards_start_in_sync.js index cb57974099c..4cf64d5f54c 100644 --- a/jstests/sharding/change_streams_shards_start_in_sync.js +++ b/jstests/sharding/change_streams_shards_start_in_sync.js @@ -7,8 +7,6 @@ // and 'B' will be seen in the changestream before 'C'. // @tags: [ // does_not_support_stepdowns, -// # This test is flaky when running on PPC64 variants. -// ppc64le_incompatible, // requires_majority_read_concern, // uses_change_streams, // ] diff --git a/jstests/sharding/clone_catalog_data.js b/jstests/sharding/clone_catalog_data.js index bd0651b0e78..62596057477 100644 --- a/jstests/sharding/clone_catalog_data.js +++ b/jstests/sharding/clone_catalog_data.js @@ -106,18 +106,14 @@ checkOptions(c2, Object.keys(coll2Options)); checkUUID(c2, coll2uuid); - function checkIndexes(collName, expectedIndexes, shardedColl) { + function checkIndexes(collName, expectedIndexes) { var res = toShard.getDB('test').runCommand({listIndexes: collName}); assert.commandWorked(res, 'Failed to get indexes for collection ' + collName); var indexes = res.cursor.firstBatch; indexes.sort(sortByName); - // TODO SERVER-74252: once 7.0 becomes LastLTS we can assume that the movePrimary will never - // copy indexes of sharded collections. - if (shardedColl) - assert(indexes.length === 1 || indexes.length === 3); - else - assert(indexes.length === 3); + // There should be 3 indexes on each collection - the _id one, and the 2 we created. + assert.eq(indexes.length, 3); indexes.forEach((index, i) => { var expected; @@ -131,8 +127,8 @@ }); } - checkIndexes('coll1', coll1Indexes, /*shardedColl*/ false); - checkIndexes('coll2', coll2Indexes, /*shardedColl*/ true); + checkIndexes('coll1', coll1Indexes); + checkIndexes('coll2', coll2Indexes); // Verify that the data from the unsharded collections resides on the new primary shard, and was // copied as part of the clone. diff --git a/jstests/sharding/cluster_time_across_add_shard.js b/jstests/sharding/cluster_time_across_add_shard.js deleted file mode 100644 index ff983be7fea..00000000000 --- a/jstests/sharding/cluster_time_across_add_shard.js +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Test that a shardsvr replica set that has not initialized its shard identity via an - * addShard command can validate and sign cluster times, and that after its shard identity has - * been initialized, it is still able to validate cluster times that were signed when it was not a - * shard. - */ - -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); -load("jstests/multiVersion/libs/multi_rs.js"); -load('jstests/replsets/rslib.js'); - -function createUser(rst) { - rst.getPrimary().getDB("admin").createUser({user: "root", pwd: "root", roles: ["root"]}, - {w: rst.nodes.length}); -} - -function authUser(node) { - assert(node.getDB("admin").auth("root", "root")); -} - -function createSession(node) { - const conn = new Mongo(node.host); - authUser(conn); - return conn.startSession({causalConsistency: false, retryWrites: false}); -} - -function withTemporaryTestData(callback, mods = {}) { - const originalTestData = TestData; - try { - TestData = Object.assign({}, TestData, mods); - callback(); - } finally { - TestData = originalTestData; - } -} - -// Start a replica set with keyfile authentication enabled so it will return signed cluster times -// its responses. -const numNodes = 3; -const keyFile = "jstests/libs/key1"; -const rstOpts = { - nodes: numNodes, - keyFile -}; -const rst = new ReplSetTest(rstOpts); - -rst.startSet(); -rst.initiate(); -const primary = rst.getPrimary(); - -// Create a user for running commands later on in the test. Make the user not have the -// advanceClusterTime privilege. This ensures that the server will not return cluster times -// signed with a dummy key. -createUser(rst); - -let sessions = []; -let sessionOnPrimary; -rst.nodes.forEach(node => { - const session = createSession(node); - if (node == primary) { - sessionOnPrimary = session; - } - sessions.push(session); -}); - -const dbName = "testDb"; -const collName = "testColl"; -assert.commandWorked(sessionOnPrimary.getDatabase(dbName).getCollection(collName).insert({})); -const lastClusterTime = sessionOnPrimary.getClusterTime(); - -for (let session of sessions) { - session.advanceClusterTime(lastClusterTime); - assert.commandWorked(session.getDatabase("admin").runCommand("hello")); -} - -// Restart the replica set as a shardsvr. Use TestData with -// authentication settings so Mongo.prototype.getDB() takes care of re-authenticating after the -// network connection is re-established during ReplSetTest.prototype.upgradeSet(). -const tmpTestData = { - auth: true, - keyFile, - authUser: "__system", - keyFileData: "foopdedoop", - authenticationDatabase: "local" -}; -const upgradeOpts = { - appendOptions: true -}; -// Restart the replica set as a shardsvr. -withTemporaryTestData(() => { - rst.upgradeSet(Object.assign({"shardsvr": ""}, upgradeOpts)); -}, tmpTestData); - -for (let session of sessions) { - // Reconnect and re-authenticate after the network connection was closed due to restart. - const error = assert.throws(() => session.getDatabase("admin").runCommand("hello")); - assert(isNetworkError(error), error); - authUser(session.getClient()); - - // Verify that the application is able to use a signed cluster time although the addShard - // or transitionFromDedicatedConfigServer command has not been run. - assert.soon(() => { - const res = session.getDatabase("admin").runCommand("hello"); - // TODO (SERVER-78909): KeysCollectionManager::getKeysForValidation() should retry - // refreshing if the refresh failed with ReadConcernMajorityNotAvailableYet - if (res.code == ErrorCodes.KeyNotFound) { - return false; - } - assert.commandWorked(res); - return true; - }); - assert.eq(session.getClusterTime().signature.keyId, lastClusterTime.signature.keyId); -} - -// Start a sharded cluster and add the shardsvr replica set to it. -const st = new ShardingTest({ - mongos: 1, - config: 1, - shards: 1, - other: {keyFile}, - configOptions: { - // Additionally test TTL deletion of key documents. To speed up the test, make the - // documents expire right away. To prevent the documents from being deleted before all - // cluster time validation testing is completed, make the TTL monitor have a large - // sleep interval at first and then lower it at the end of the test when verifying that - // the documents do get deleted by the TTL monitor. - setParameter: {newShardExistingClusterTimeKeysExpirationSecs: 1, ttlMonitorSleepSecs: 3600} - } -}); -assert.commandWorked(st.s.adminCommand({addShard: rst.getURL()})); -createUser(st.configRS); - -rst.awaitReplication(); - -for (let session of sessions) { - // As a performance optimization, LogicalTimeValidator::validate() skips validating $clusterTime - // values which have a $clusterTime.clusterTime value smaller than the currently known signed - // $clusterTime value. It is possible (but not strictly guaranteed) for internal communication - // to have already happened between cluster members such that they all know about a signed - // $clusterTime value. This signed $clusterTime value would come from the new signing key - // generated by the config server primary. Here we use the alwaysValidateClientsClusterTime - // fail point to simulate the behavior of case when the internal communication with a signed - // $clusterTime value has not happened yet. - const fp = (() => { - const fpConn = new Mongo(session.getClient().host); - authUser(fpConn); - return configureFailPoint(fpConn, "alwaysValidateClientsClusterTime"); - })(); - - // Verify that after the addShard or transitionFromDedicatedConfigServer command has been run, - // the application is still able to use the cluster time signed when the replica set was not a - // shard. - assert.commandWorked(session.getDatabase("admin").runCommand("hello")); - - // Verify that the new cluster time was signed with the sharded cluster's key (generated by - // the config server) instead of the shardsvr replica set's key. - assert.neq(session.getClusterTime().signature.keyId, lastClusterTime.signature.keyId); - - // Verify that the old cluster time can also be used against the mongos, config server, and - // other shard. - assert.commandWorked(st.s.getDB("admin").runCommand({hello: 1, $clusterTime: lastClusterTime})); - assert.commandWorked(st.configRS.getPrimary().getDB("admin").runCommand( - {hello: 1, $clusterTime: lastClusterTime})); - assert.commandWorked( - st.rs0.getPrimary().getDB("admin").runCommand({hello: 1, $clusterTime: lastClusterTime})); - - fp.off(); -} - -st.stop(); -rst.stopSet(); -})(); diff --git a/jstests/sharding/clustered_coll_scan.js b/jstests/sharding/clustered_coll_scan.js deleted file mode 100644 index b96e50b065d..00000000000 --- a/jstests/sharding/clustered_coll_scan.js +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Testing if mongos can deal with clustered collections and the related clustered IDX scan bounds - * (SERVER-83119) - */ -(function() { -load("jstests/libs/analyze_plan.js"); -load("jstests/libs/collection_drop_recreate.js"); - -const st = new ShardingTest({ - shards: 2, - mongos: 1, -}); - -st.s.adminCommand({enableSharding: "test"}); - -const db = st.getDB("test"); -// Create the collection as a clustered collection. -const coll = assertDropAndRecreateCollection( - db, jsTestName(), {clusteredIndex: {key: {_id: 1}, unique: true}}); -st.shardColl(coll, {a: 1}); -// First of all check that we can execute the query. -assert.commandWorked(coll.insertMany([...Array(10).keys()].map(i => { - return {_id: i, a: i}; -}))); - -{ - var explain = coll.find({_id: 2}).explain(); - // Make sure that we have a clusteredIDXScan in the plan. - - assert(isClusteredIxscan(db, explain)); - assert.commandWorked( - st.getPrimaryShard("test").adminCommand({setParameter: 1, notablescan: 1})); - // Do the same thing only with notablescan enabled. - explain = coll.find({_id: 2}).explain(); - assert(isClusteredIxscan(db, explain)); - // Sanity count check. - assert.eq(1, coll.find({_id: 2}).itcount()); -} -// Test the same with aggregate. -{ - var explain = coll.explain().aggregate([{$match: {_id: 22}}]); - assert(isClusteredIxscan(db, explain)); -} - -st.stop(); -})(); diff --git a/jstests/sharding/collection_uuid_shard_capped_collection.js b/jstests/sharding/collection_uuid_shard_capped_collection.js deleted file mode 100644 index 3b26f26715b..00000000000 --- a/jstests/sharding/collection_uuid_shard_capped_collection.js +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Tests the collectionUUID parameter of the shardCollection command against capped collections. - * - * @tags: [ - * requires_fcv_60, - * ] - */ -(function() { -'use strict'; - -const st = new ShardingTest({shards: 1}); -const mongos = st.s0; - -const db = mongos.getDB(jsTestName()); -const coll = db['cappedColl']; -const collName = coll.getName(); - -// Create a capped collection. -assert.commandWorked(db.createCollection(collName, {capped: true, size: 1024})); -assert.commandWorked(mongos.adminCommand({enableSharding: db.getName()})); - -// Ensure that we fail the shardCollection command with 'CollectionUUIDMismatch' when the UUID does -// not correspond to the collection. -const nonexistentUUID = UUID(); -let res = assert.commandFailedWithCode( - mongos.adminCommand( - {shardCollection: coll.getFullName(), key: {_id: 1}, collectionUUID: nonexistentUUID}), - ErrorCodes.CollectionUUIDMismatch); -assert.eq(res.db, db.getName()); -assert.eq(res.collectionUUID, nonexistentUUID); -assert.eq(res.expectedCollection, collName); -assert.eq(res.actualCollection, null); - -const uuid = assert.commandWorked(db.runCommand({listCollections: 1})) - .cursor.firstBatch.find(c => c.name === collName) - .info.uuid; - -// Ensure that we fail the shard command with 'InvalidOptions' when the UUID corresponds to the -// capped collection. -assert.commandFailedWithCode( - mongos.adminCommand({shardCollection: coll.getFullName(), collectionUUID: uuid, key: {_id: 1}}), - ErrorCodes.InvalidOptions); - -st.stop(); -})(); diff --git a/jstests/sharding/compound_hashed_shard_key_targeting.js b/jstests/sharding/compound_hashed_shard_key_targeting.js index 59de60b9508..9e4858ee462 100644 --- a/jstests/sharding/compound_hashed_shard_key_targeting.js +++ b/jstests/sharding/compound_hashed_shard_key_targeting.js @@ -280,5 +280,10 @@ profileFilter = { }; verifyProfilerEntryOnCorrectShard(1, profileFilter); +// Test to verify that delete with limit:1, without full shard key in query fails. +assert.commandFailedWithCode( + coll.runCommand({delete: coll.getName(), deletes: [{q: {a: 1}, limit: 1}], ordered: false}), + ErrorCodes.ShardKeyNotFound); + st.stop(); })(); diff --git a/jstests/sharding/configsvr_remove_chunks.js b/jstests/sharding/configsvr_remove_chunks.js index 9eed04de342..32bd4de8e6a 100644 --- a/jstests/sharding/configsvr_remove_chunks.js +++ b/jstests/sharding/configsvr_remove_chunks.js @@ -48,9 +48,7 @@ function insertLeftoverChunks(configDB, uuid) { let st = new ShardingTest({mongos: 1, shards: 1}); -// Use retriable writes when writing to the config server since these are not automatically retried -const mongosSession = st.s.startSession({retryWrites: true}); -const configDB = mongosSession.getDatabase("config"); +const configDB = st.s.getDB('config'); const dbName = "test"; const collName = "foo"; diff --git a/jstests/sharding/coordinate_txn_recover_on_stepup_with_tickets_exhausted.js b/jstests/sharding/coordinate_txn_recover_on_stepup_with_tickets_exhausted.js deleted file mode 100644 index f25d8b02a73..00000000000 --- a/jstests/sharding/coordinate_txn_recover_on_stepup_with_tickets_exhausted.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Validate that the TransactionCoordinator for a prepared transaction can be recovered on step-up - * and commit the transaction when there are no storage tickets available. See SERVER-82883 and - * SERVER-60682. - * - * @tags: [ - * uses_transactions, - * uses_multi_shard_transaction, - * uses_prepare_transaction, - * ] - */ - -load("jstests/libs/fail_point_util.js"); -load('jstests/libs/parallelTester.js'); -load("jstests/sharding/libs/create_sharded_collection_util.js"); - -const st = new ShardingTest({ - mongos: 1, - config: 1, - shards: 2, - rs: {nodes: 3}, - rsOptions: { - setParameter: { - maxTransactionLockRequestTimeoutMillis: 24 * 60 * 60 * 1000, - transactionLifetimeLimitSeconds: 24 * 60 * 60, - } - } -}); - -const sourceCollection = st.s.getCollection("test.mycoll"); -CreateShardedCollectionUtil.shardCollectionWithChunks(sourceCollection, {key: 1}, [ - {min: {key: MinKey}, max: {key: 0}, shard: st.shard0.shardName}, - {min: {key: 0}, max: {key: MaxKey}, shard: st.shard1.shardName}, -]); - -// Insert a document into each shard. -assert.commandWorked(sourceCollection.insert([{key: 200}, {key: -200}])); - -// Create a thread which leaves the TransactionCoordinator in a state where -// prepareTransaction has been run on both participant shards and it is about to write -// the commit decision locally to the config.transaction_coordinators collection. -const preparedTxnThread = new Thread(function runTwoPhaseCommitTxn(host, dbName, collName) { - const conn = new Mongo(host); - const session = conn.startSession({causalConsistency: false}); - const sessionCollection = session.getDatabase(dbName).getCollection(collName); - - session.startTransaction(); - assert.commandWorked(sessionCollection.update({key: 200}, {$inc: {counter: 1}})); - assert.commandWorked(sessionCollection.update({key: -200}, {$inc: {counter: 1}})); - assert.commandWorked(session.commitTransaction_forTesting()); -}, st.s.host, sourceCollection.getDB().getName(), sourceCollection.getName()); -const txnCoordinator = st.rs1.getPrimary(); -const hangBeforeWritingDecisionFp = configureFailPoint(txnCoordinator, "hangBeforeWritingDecision"); - -preparedTxnThread.start(); -hangBeforeWritingDecisionFp.wait(); - -// Step-up the secondary and make it hang before doing the work to recover -// the TransactionCoordinator for the prepared transaction on step-up. -const secondary = st.rs1.getSecondary(); -const hangBeforeTxnCoordinatorOnStepUpWorkFp = - configureFailPoint(secondary, "hangBeforeTxnCoordinatorOnStepUpWork"); -st.rs1.stepUp(secondary, {awaitWritablePrimary: false, awaitReplicationBeforeStepUp: false}); -hangBeforeTxnCoordinatorOnStepUpWorkFp.wait(); - -const hangBeforeDeletingCoordinatorDocFp = - configureFailPoint(secondary, "hangBeforeDeletingCoordinatorDoc"); - -// Set the read and write tickets to 0 before executing the code to recover the -// TransactionCoordinator. -assert.commandWorked(secondary.getDB("admin").adminCommand( - {setParameter: 1, wiredTigerConcurrentReadTransactions: NumberInt(0)})); -assert.commandWorked(secondary.getDB("admin").adminCommand( - {setParameter: 1, wiredTigerConcurrentWriteTransactions: NumberInt(0)})); -hangBeforeTxnCoordinatorOnStepUpWorkFp.off(); - -// The TransactionCoordinator has successfully been recovered and the prepared transaction has been -// committed once this failpoint has been reached. -hangBeforeDeletingCoordinatorDocFp.wait(); -hangBeforeDeletingCoordinatorDocFp.off(); - -// Reset the read and write tickets to a non-zero value to allow the test to finish. -assert.commandWorked(secondary.getDB("admin").adminCommand( - {setParameter: 1, wiredTigerConcurrentReadTransactions: NumberInt(128)})); -assert.commandWorked(secondary.getDB("admin").adminCommand( - {setParameter: 1, wiredTigerConcurrentWriteTransactions: NumberInt(128)})); - -preparedTxnThread.join(); -st.stop(); diff --git a/jstests/sharding/create_indexes_on_stale_router.js b/jstests/sharding/create_indexes_on_stale_router.js deleted file mode 100644 index 8d7bb7c4cad..00000000000 --- a/jstests/sharding/create_indexes_on_stale_router.js +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Tests createIndexes in a stale shard and stale db router - * @tags: [ - * multiversion_incompatible, - * requires_fcv_60, - * ] - */ -(function() { -'use strict'; - -const dbName = 'testDB'; -const collName = 'testColl'; -const bucketsCollName = 'system.buckets.' + collName; -const timeField = 'time'; -const metaField = 'hostid'; -const shardKey = { - [metaField]: 1 -}; - -const createIndexOnStaleRouter = (staleRouter) => { - let cmdObj = { - createIndexes: collName, - indexes: [{key: {[timeField]: 1}, name: "index_on_time"}] - }; - assert.commandWorked(staleRouter.runCommand(cmdObj)); -}; - -const st = new ShardingTest({mongos: 2, shards: 2, rs: {nodes: 2}}); -const mongos0 = st.s0.getDB(dbName); -const mongos1 = st.s1.getDB(dbName); -const shard0DB = st.shard0.getDB(dbName); -const shard1DB = st.shard1.getDB(dbName); - -// Insert some dummy data using 'mongos1' as the router, so that the cache is -// initialized on the mongos while the collection is unsharded. -assert.commandWorked( - mongos1.adminCommand({shardCollection: `${dbName}.${collName}`, key: {_id: 1}})); -assert.commandWorked(mongos1.getCollection(collName).insert({_id: "aaa"})); - -// Drop and shard the collection with 'mongos0' as the router. -assert(mongos0.getCollection(collName).drop()); -assert.commandWorked(mongos0.adminCommand({ - shardCollection: `${dbName}.${collName}`, - key: shardKey, - timeseries: {timeField: timeField, metaField: metaField} -})); - -createIndexOnStaleRouter(mongos1); - -// Drop the database and create a new one. -assert.commandWorked(st.s0.getDB(dbName).dropDatabase()); -assert.commandWorked( - mongos0.adminCommand({shardCollection: `${dbName}.${collName}`, key: shardKey})); - -createIndexOnStaleRouter(mongos1); -st.stop(); -})(); diff --git a/jstests/sharding/data_size_aware_balancing_sessions_collection.js b/jstests/sharding/data_size_aware_balancing_sessions_collection.js deleted file mode 100644 index a9d4acb4b68..00000000000 --- a/jstests/sharding/data_size_aware_balancing_sessions_collection.js +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Tests that the balancer splits the sessions collection and uniformly distributes the chunks - * across shards in the cluster. - * @tags: [ - * featureFlagBalanceAccordingToDataSize, - * requires_fcv_60, - * resource_intensive, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/feature_flag_util.js"); -load("jstests/sharding/libs/find_chunks_util.js"); - -// TODO SERVER-50144 Remove this and allow orphan checking. -// This test calls removeShard which can leave docs in config.rangeDeletions in state "pending", -// therefore preventing orphans from being cleaned up. -TestData.skipCheckOrphans = true; - -/* - * Returns the number of chunks for the sessions collection. - */ -function getNumTotalChunks() { - return findChunksUtil.countChunksForNs(configDB, kSessionsNs); -} - -/* - * Returns the number of chunks for the sessions collection that are the given shard. - */ -function getNumChunksOnShard(shardName) { - return findChunksUtil.countChunksForNs(configDB, kSessionsNs, {shard: shardName}); -} - -/* - * Returns the number of docs in the sessions collection on the given host. - */ -function getNumSessionDocs(conn) { - return conn.getCollection(kSessionsNs).find().itcount(); -} - -/* - * Starts a replica-set shard, adds the shard to the cluster, and increments numShards. - * Returns the ReplSetTest object for the shard. - */ -function addShardsToCluster(shardsToAdd) { - let addedReplicaSets = []; - for (let i = 0; i < shardsToAdd; ++i) { - const shardName = clusterName + "-rs" + numShards; - const replTest = new ReplSetTest({name: shardName, nodes: 1}); - replTest.startSet({shardsvr: ""}); - replTest.initiate(); - - assert.commandWorked(st.s.adminCommand({addShard: replTest.getURL(), name: shardName})); - numShards++; - addedReplicaSets.push(replTest); - } - return addedReplicaSets; -} - -/* - * Removes the given shard from the cluster, waits util the state is completed, and - * decrements numShards. - */ -function removeShardFromCluster(shardName) { - assert.commandWorked(st.s.adminCommand({removeShard: shardName})); - assert.soon(function() { - const res = st.s.adminCommand({removeShard: shardName}); - if (!res.ok && res.code === ErrorCodes.ShardNotFound) { - // If the config server primary steps down right after removing the config.shards doc - // for the shard but before responding with "state": "completed", the mongos would retry - // the _configsvrRemoveShard command against the new config server primary, which would - // not find the removed shard in its ShardRegistry if it has done a ShardRegistry reload - // after the config.shards doc for the shard was removed. This would cause the command - // to fail with ShardNotFound. - return true; - } - assert.commandWorked(res); - return ("completed" == res.state); - }, "failed to remove shard " + shardName, kBalancerTimeoutMS); - numShards--; -} - -/* - * Returns the estimated size (in bytes) of the sessions collection chunks hosted by the shard. - */ -function getSessionsCollSizeInShard(shardStats) { - const orphansSize = - shardStats['storageStats']['numOrphanDocs'] * shardStats['storageStats']['avgObjSize']; - return shardStats['storageStats']['size'] - orphansSize; -} - -function printSessionsCollectionDistribution(shards) { - const numDocsOnShards = shards.map(shard => getNumSessionDocs(shard)); - const collStatsPipeline = [ - {'$collStats': {'storageStats': {}}}, - { - '$project': { - 'shard': true, - 'storageStats': - {'count': true, 'size': true, 'avgObjSize': true, 'numOrphanDocs': true} - } - }, - {'$sort': {'shard': 1}} - ]; - const collectionStorageStats = - st.s.getCollection(kSessionsNs).aggregate(collStatsPipeline).toArray(); - const collSizeDistribution = - collectionStorageStats.map(shardStats => getSessionsCollSizeInShard(shardStats)); - const numChunksOnShard = shards.map(shard => getNumChunksOnShard(shard.shardName)); - const kMaxChunkSizeBytes = st.config.collections.findOne({_id: kSessionsNs}).maxChunkSizeBytes; - - jsTest.log(`Sessions distribution across shards ${tojson(shards)}: #docs = ${ - tojson(numDocsOnShards)}, #chunks = ${tojson(numChunksOnShard)}, size = ${ - tojson(collSizeDistribution)}, #maxChunkSize: ${tojson(kMaxChunkSizeBytes)}`); -} - -function waitUntilBalancedAndVerify(shards) { - const coll = st.s.getCollection(kSessionsNs); - st.awaitBalance( - kSessionsCollName, kConfigDbName, 9 * 60000 /* 9min timeout */, 1000 /* 1s interval */); - printSessionsCollectionDistribution(shards); - st.verifyCollectionIsBalanced(coll); -} - -const kMinNumChunks = 100; -const kExpectedNumChunks = 128; // the balancer rounds kMinNumChunks to the next power of 2. -const kNumSessions = 2000; -const kBalancerTimeoutMS = 5 * 60 * 1000; - -let numShards = 2; -const clusterName = jsTest.name(); -const st = new ShardingTest({ - name: clusterName, - shards: numShards, - other: {configOptions: {setParameter: {minNumChunksForSessionsCollection: kMinNumChunks}}} -}); - -const kConfigDbName = "config"; -const kSessionsCollName = "system.sessions"; -const kSessionsNs = `${kConfigDbName}.${kSessionsCollName}`; -const configDB = st.s.getDB(kConfigDbName); - -// There is only one chunk initially. -assert.eq(1, getNumTotalChunks()); - -st.startBalancer(); - -jsTest.log( - `Verify that the balancer generates the expected initial set of chunks for ${kSessionsNs}`); - -assert.soon(() => getNumTotalChunks() == kExpectedNumChunks, - "balancer did not split the initial chunk for the sessions collection"); - -jsTest.log(`Verify that no chunks are moved from the primary shard of ${ - kSessionsNs} if the are no open sessions`); -{ - st.awaitBalance(kSessionsCollName, kConfigDbName); - const numChunksInShard0 = getNumChunksOnShard(st.shard0.shardName); - const numChunksInShard1 = getNumChunksOnShard(st.shard1.shardName); - assert(numChunksInShard0 === kExpectedNumChunks && numChunksInShard1 === 0 || - numChunksInShard1 === kExpectedNumChunks && numChunksInShard0 === 0); -} - -jsTest.log(`Creating ${kNumSessions} sessions`); -for (let i = 0; i < kNumSessions; i++) { - assert.commandWorked(st.s.adminCommand({startSession: 1})); -} -assert.commandWorked(st.s.adminCommand({refreshLogicalSessionCacheNow: 1})); -assert.lte(kNumSessions, getNumSessionDocs(st.s)); -let shards = [st.shard0, st.shard1]; -jsTest.log(`Verify that the chunks of ${kSessionsNs} get distributed across the original cluster`); -waitUntilBalancedAndVerify(shards); - -jsTest.log( - "Verify that the balancer redistributes chunks when more shards are added to the cluster"); -const addedReplicaSets = addShardsToCluster(3); -shards = shards.concat(addedReplicaSets.map(rs => { - const primaryNode = rs.getPrimary(); - primaryNode.shardName = rs.name; - return primaryNode; -})); -waitUntilBalancedAndVerify(shards); - -jsTest.log( - "Verify that the balancer redistributes chunks when shards are removed from the cluster"); -removeShardFromCluster(shards[2].shardName); -shards.splice(2, 1); -waitUntilBalancedAndVerify(shards); - -st.stopBalancer(); - -st.stop(); -addedReplicaSets.forEach(rs => { - rs.stopSet(); -}); -}()); diff --git a/jstests/sharding/database_versioning_all_commands.js b/jstests/sharding/database_versioning_all_commands.js index 94413d4e9ae..fe7c409f524 100644 --- a/jstests/sharding/database_versioning_all_commands.js +++ b/jstests/sharding/database_versioning_all_commands.js @@ -247,8 +247,6 @@ let testCases = { _isSelf: {skip: "executes locally on mongos (not sent to any remote node)"}, _killOperations: {skip: "executes locally on mongos (not sent to any remote node)"}, _mergeAuthzCollections: {skip: "always targets the config server"}, - _mongotConnPoolStats: {skip: "not on a user database", conditional: true}, - _dropConnectionsToMongot: {skip: "not on a user database", conditional: true}, abortReshardCollection: {skip: "always targets the config server"}, abortTransaction: {skip: "unversioned and uses special targetting rules"}, addShard: {skip: "not on a user database"}, @@ -349,7 +347,6 @@ let testCases = { } }, createRole: {skip: "always targets the config server"}, - createSearchIndexes: {skip: "executes locally on mongos"}, createUser: {skip: "always targets the config server"}, currentOp: {skip: "not on a user database"}, dataSize: { @@ -413,7 +410,6 @@ let testCases = { } }, dropRole: {skip: "always targets the config server"}, - dropSearchIndex: {skip: "executes locally on mongos"}, dropUser: {skip: "always targets the config server"}, echo: {skip: "does not forward command to primary shard"}, enableSharding: {skip: "does not forward command to primary shard"}, @@ -459,7 +455,6 @@ let testCases = { }, flushRouterConfig: {skip: "executes locally on mongos (not sent to any remote node)"}, fsync: {skip: "broadcast to all shards"}, - fsyncUnlock: {skip: "broadcast to all shards"}, getAuditConfig: {skip: "not on a user database", conditional: true}, getChangeStreamOptions: { skip: "executes locally on mongos (not sent to any remote node)", @@ -516,7 +511,6 @@ let testCases = { }, } }, - listSearchIndexes: {skip: "executes locally on mongos"}, listShards: {skip: "does not forward command to primary shard"}, logApplicationMessage: {skip: "not on a user database", conditional: true}, logMessage: {skip: "not on a user database"}, @@ -655,7 +649,8 @@ let testCases = { } }, setFeatureCompatibilityVersion: {skip: "not on a user database"}, - setProfilingFilterGlobally: {skip: "executes locally on mongos (not sent to any remote node)"}, + setFreeMonitoring: + {skip: "explicitly fails for mongos, primary mongod only", conditional: true}, setParameter: {skip: "executes locally on mongos (not sent to any remote node)"}, setClusterParameter: {skip: "always targets the config server"}, setUserWriteBlockMode: {skip: "executes locally on mongos (not sent to any remote node)"}, @@ -695,7 +690,6 @@ let testCases = { } }, updateRole: {skip: "always targets the config server"}, - updateSearchIndex: {skip: "executes locally on mongos"}, updateUser: {skip: "always targets the config server"}, updateZoneKeyRange: {skip: "not on a user database"}, usersInfo: {skip: "always targets the config server"}, diff --git a/jstests/sharding/delete_range_deletion_tasks_on_dropped_hashed_shard_key_index.js b/jstests/sharding/delete_range_deletion_tasks_on_dropped_hashed_shard_key_index.js deleted file mode 100644 index fed08fc05ed..00000000000 --- a/jstests/sharding/delete_range_deletion_tasks_on_dropped_hashed_shard_key_index.js +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Tests that deleting the index on a hashed shard key blocks its orphan documents from being - * deleted and allows other range deletion processes to continue. - * - * @tags: [ - * requires_fcv_50, - * ] - */ -(function() { -'use strict'; - -load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/fail_point_util.js"); - -const rangeDeleterBatchSize = 50; - -const st = new ShardingTest({ - other: { - enableBalancer: false, - shardOptions: {setParameter: {rangeDeleterBatchSize: rangeDeleterBatchSize}}, - } -}); - -// Setup database and collection for test -const dbName = 'db'; -const db = st.getDB(dbName); -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); -const coll = db['test']; -const collWithIndex = db['collWithIndex']; - -function setUpCollection(collectionName, nss) { - // This creates an index on the hashed shard key. - assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: 'hashed'}})); - - // Insert some documents into the collection. - const numDocs = 1000; - let bulk = collectionName.initializeUnorderedBulkOp(); - for (let i = 0; i < numDocs; i++) { - bulk.insert({_id: i}); - } - assert.commandWorked(bulk.execute()); - - // Move a chunk to create orphan documents. - const chunk = - findChunksUtil.findOneChunkByNs(st.s.getDB('config'), nss, {shard: st.shard0.shardName}); - assert.commandWorked( - db.adminCommand({moveChunk: nss, bounds: [chunk.min, chunk.max], to: st.shard1.shardName})); -} - -// Pause range deletion on shard0. -let suspendRangeDeletionFailpoint = configureFailPoint(st.shard0, "suspendRangeDeletion"); -setUpCollection(coll, coll.getFullName()); -setUpCollection(collWithIndex, collWithIndex.getFullName()); -assert.commandWorked(coll.dropIndex({"_id": "hashed"})); -suspendRangeDeletionFailpoint.off(); - -// Verify that the range deletion document for db.test persists while the document for -// db.collWithIndex is successfully deleted. -assert.eq(1, st.shard0.getDB("config").getCollection("rangeDeletions").countDocuments({ - nss: coll.getFullName() -})); - -assert.soon(() => { - return st.shard0.getDB("config").getCollection("rangeDeletions").countDocuments({ - nss: collWithIndex.getFullName() - }) === 0; -}); - -assert.eq(1, st.shard0.getDB("config").getCollection("rangeDeletions").countDocuments({ - nss: coll.getFullName() -})); - -// Rebuild the hashed shard key index for db.test and ensure the pending range deletion completes. -assert.commandWorked(coll.createIndex({"_id": "hashed"})); -assert.soon(() => { - return st.shard0.getDB("config").getCollection("rangeDeletions").countDocuments({ - nss: coll.getFullName() - }) === 0; -}); - -st.stop(); -})(); diff --git a/jstests/sharding/documents_db_not_exist.js b/jstests/sharding/documents_db_not_exist.js deleted file mode 100644 index 1742d149516..00000000000 --- a/jstests/sharding/documents_db_not_exist.js +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Tests that $documents stage continues even when the database does not exist - * @tags: [requires_fcv_62, multiversion_incompatible] - * - */ - -(function() { -"use strict"; - -let st = new ShardingTest({shards: 3}); - -function listDatabases(options) { - return assert.commandWorked(st.s.adminCommand(Object.assign({listDatabases: 1}, options))) - .databases; -} - -function createAndDropDatabase(dbName) { - // Create the database. - let db = st.s.getDB(dbName); - assert.commandWorked(db.foo.insert({})); - // Confirms the database exists. - assert.eq(1, listDatabases({nameOnly: true, filter: {name: dbName}}).length); - // Drop the database - assert.commandWorked(db.dropDatabase()); - // Confirm the database is dropped. - assert.eq(0, listDatabases({nameOnly: true, filter: {name: dbName}}).length); - return db; -} - -// $documents stage evaluates to an array of objects. -let db = createAndDropDatabase("test"); -let documents = []; -for (let i = 0; i < 50; i++) { - documents.push({_id: i}); -} -let result = db.aggregate([{$documents: documents}]); -assert(result.toArray().length == 50); - -//$documents stage evaluates to an array of objects in a pipeline -db = createAndDropDatabase("test2"); -result = db.aggregate([ - {$documents: [{_id: 1, size: "medium"}, {_id: 2, size: "large"}]}, - {$match: {size: "medium"}} -]); -assert(result.toArray().length == 1); - -st.stop(); -})(); diff --git a/jstests/sharding/documents_sharded.js b/jstests/sharding/documents_sharded.js deleted file mode 100644 index 85b20cd29c3..00000000000 --- a/jstests/sharding/documents_sharded.js +++ /dev/null @@ -1,160 +0,0 @@ -/** - * This is the test for $documents stage in aggregation pipeline on a sharded collection. - * @tags: [ do_not_wrap_aggregations_in_facets, requires_fcv_51 ] - * - */ - -(function() { -"use strict"; - -load("jstests/aggregation/extras/utils.js"); // For resultsEq. - -let st = new ShardingTest({shards: 2}); -const db = st.s.getDB(jsTestName()); -const dbName = db.getName(); -assert.commandWorked(db.adminCommand({enableSharding: dbName})); - -// Create sharded collections. -const coll = db['shardedColl']; -st.shardColl(coll, {x: 1}, {x: 1}, {x: 1}, dbName); - -const lookup_coll = db['lookupColl']; -st.shardColl(lookup_coll, {id_name: 1}, {id_name: 1}, {id_name: 1}, dbName); -for (let i = 0; i < 10; i++) { - assert.commandWorked(lookup_coll.insert({id_name: i, name: "name_" + i})); -} - -// $documents given an array of objects. -const docs = db.aggregate([{$documents: [{a1: 1}, {a1: 2}]}]).toArray(); - -assert.eq(2, docs.length); -assert.eq(docs[0], {a1: 1}); -assert.eq(docs[1], {a1: 2}); - -// $documents evaluates to an array of objects. -const docs1 = - db.aggregate([{$documents: {$map: {input: {$range: [0, 100]}, in : {x: "$$this"}}}}]).toArray(); - -assert.eq(100, docs1.length); -for (let i = 0; i < 100; i++) { - assert.eq(docs1[i], {x: i}); -} - -// $documents evaluates to an array of objects. -const docsUnionWith = - coll.aggregate([ - { - $unionWith: { - pipeline: [{$documents: {$map: {input: {$range: [0, 5]}, in : {x: "$$this"}}}}] - } - }, - {$group: {_id: "$x", x: {$first: "$x"}}}, - {$project: {_id: 0}}, - ]) - .toArray(); -assert(resultsEq([{x: 0}, {x: 1}, {x: 2}, {x: 3}, {x: 4}], docsUnionWith)); - -{ // $documents with const objects inside $unionWith. - const res = coll.aggregate([ - {$unionWith: {pipeline: [{$documents: [{x: 1}, {x: 2}]}]}}, - {$group: {_id: "$x", x: {$first: "$x"}}}, - {$project: {_id: 0}} - ]) - .toArray(); - assert(resultsEq([{x: 1}, {x: 2}], res)); -} - -{ // $documents with const objects inside $lookup (no "coll", explicit $match). - const res = lookup_coll.aggregate([ - { - $lookup: { - let: {"id_lookup": "$id_name"}, - pipeline: [ - {$documents: [{xx: 1}, {xx: 2}, {xx : 3}]}, - { - $match: - { - $expr: - { - $eq: - ["$$id_lookup", "$xx"] - } - } - } - ], - as: "names" - } - }, - {$match: {"names": {"$ne": []}}}, - {$project: {_id: 0}} - ] - ) - .toArray(); - assert(resultsEq( - [ - {id_name: 1, name: "name_1", names: [{"xx": 1}]}, - {id_name: 2, name: "name_2", names: [{"xx": 2}]}, - {id_name: 3, name: "name_3", names: [{"xx": 3}]} - ], - res)); -} -{ // $documents with const objects inside $lookup (no "coll", + localField/foreignField). - const res = lookup_coll.aggregate([ - { - $lookup: { - localField: "id_name", - foreignField: "xx", - pipeline: [ - {$documents: [{xx: 1}, {xx: 2}, {xx: 3}]} - ], - as: "names" - } - }, - {$match: {"names": {"$ne": []}}}, - {$project: {_id: 0}} - ]) - .toArray(); - assert(resultsEq( - [ - {id_name: 1, name: "name_1", names: [{"xx": 1}]}, - {id_name: 2, name: "name_2", names: [{"xx": 2}]}, - {id_name: 3, name: "name_3", names: [{"xx": 3}]} - ], - res)); -} - -// Must fail when $document appears in the top level collection pipeline. -assert.throwsWithCode(() => { - coll.aggregate([{$documents: {$map: {input: {$range: [0, 100]}, in : {x: "$$this"}}}}]); -}, ErrorCodes.InvalidNamespace); - -// Must fail due to misplaced $document. -assert.throwsWithCode(() => { - coll.aggregate([{$project: {x: [{xx: 1}, {xx: 2}]}}, {$documents: [{x: 1}]}]); -}, 40602); - -// Test that $documents fails due to producing array of non-objects. -assert.throwsWithCode(() => { - db.aggregate([{$documents: [1, 2, 3]}]); -}, 40228); - -// Now with one object and one scalar. -assert.throwsWithCode(() => { - db.aggregate([{$documents: [{x: 1}, 2]}]); -}, 40228); - -// Test that $documents fails due when provided a non-array. -assert.throwsWithCode(() => { - db.aggregate([{$documents: "string"}]); -}, 5858203); - -// Test that $documents succeeds when given a singleton object. -assert.eq(db.aggregate([{$documents: [{x: [1, 2, 3]}]}]).toArray(), [{x: [1, 2, 3]}]); - -// Must fail when $document appears in the top level collection pipeline. -assert.throwsWithCode(() => { - coll.aggregate([{$documents: {$map: {input: {$range: [0, 100]}, in : {x: "$$this"}}}}]); -}, ErrorCodes.InvalidNamespace); - -st.stop(); -})();
\ No newline at end of file diff --git a/jstests/sharding/drop_collection.js b/jstests/sharding/drop_collection.js index 0947b56124a..f07ced08ae5 100644 --- a/jstests/sharding/drop_collection.js +++ b/jstests/sharding/drop_collection.js @@ -32,29 +32,17 @@ function assertCollectionDropped(ns, uuid = null) { configDB.tags.countDocuments({ns: ns}), "Found unexpected tag for a collection after drop."); - // Sharding metadata checks - - // No more coll entry - assert.eq(null, st.s.getCollection(ns).exists()); - assert.eq(0, - configDB.collections.countDocuments({_id: ns}), - "Found collection entry in 'config.collection' after drop."); - // No more chunks + const errMsg = "Found collection entry in 'config.collection' after drop."; + // Before 5.0 chunks were indexed by ns, now by uuid + assert.eq(0, configDB.chunks.countDocuments({ns: ns}), errMsg); if (uuid != null) { - assert.eq(0, - configDB.chunks.countDocuments({uuid: uuid}), - "Found references to collection uuid in 'config.chunks' after drop."); + assert.eq(0, configDB.chunks.countDocuments({uuid: uuid}), errMsg); } - // Verify that persisted cached metadata was removed as part of the dropCollection - const chunksCollName = 'cache.chunks.' + ns; - for (let configDb of [st.shard0.getDB('config'), st.shard1.getDB('config')]) { - assert.eq(configDb['cache.collections'].countDocuments({_id: ns}), - 0, - "Found collection entry in 'config.cache.collections' after drop."); - assert(!configDb[chunksCollName].exists()); - } + // No more coll entry + assert.eq(null, st.s.getCollection(ns).exists()); + assert.eq(0, configDB.collections.countDocuments({_id: ns})); } jsTest.log("Drop unsharded collection."); @@ -86,11 +74,10 @@ jsTest.log("Drop unsharded collection also remove tags."); assert.commandWorked(db.runCommand({drop: coll.getName()})); assertCollectionDropped(coll.getFullName()); } - jsTest.log("Drop sharded collection repeated."); { const db = getNewDb(); - const coll = db['shardedColl0']; + const coll = db['unshardedColl0']; // Create the database assert.commandWorked(st.s.adminCommand({enableSharding: db.getName()})); for (var i = 0; i < 3; i++) { @@ -98,11 +85,9 @@ jsTest.log("Drop sharded collection repeated."); assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {x: 1}})); assert.commandWorked(coll.insert({x: 123})); assert.eq(1, coll.countDocuments({x: 123})); - // Drop the collection - var uuid = getCollectionUUID(coll.getFullName()); assert.commandWorked(db.runCommand({drop: coll.getName()})); - assertCollectionDropped(coll.getFullName(), uuid); + assertCollectionDropped(coll.getFullName()); } } @@ -260,9 +245,7 @@ jsTest.log("Move primary with drop and recreate - new primary own chunks."); configDB, coll.getFullName(), {shard: st.shard1.shardName})); jsTest.log("Drop sharded collection"); - var uuid = getCollectionUUID(coll.getFullName()); coll.drop(); - assertCollectionDropped(coll.getFullName(), uuid); jsTest.log("Re-Create sharded collection with one chunk on shard 0"); st.shardColl(coll, {skey: 1}, false, false); @@ -274,12 +257,11 @@ jsTest.log("Move primary with drop and recreate - new primary own chunks."); st.ensurePrimaryShard(db.getName(), st.shard1.shardName); jsTest.log("Drop sharded collection"); - uuid = getCollectionUUID(coll.getFullName()); coll.drop(); - assertCollectionDropped(coll.getFullName(), uuid); } -jsTest.log("Test that dropping an unsharded collection, relevant events are logged on CSRS."); +jsTest.log( + "Test that dropping a non-sharded collection, relevant events are properly logged on CSRS"); { // Create a non-sharded collection const db = getNewDb(); @@ -288,7 +270,6 @@ jsTest.log("Test that dropping an unsharded collection, relevant events are logg // Drop the collection assert.commandWorked(db.runCommand({drop: coll.getName()})); - assertCollectionDropped(coll.getFullName()); // Verify that the drop collection start event has been logged const startLogCount = @@ -301,7 +282,7 @@ jsTest.log("Test that dropping an unsharded collection, relevant events are logg assert.gte(1, endLogCount, "dropCollection end event not found in changelog"); } -jsTest.log("Test that dropping a sharded collection, relevant events are logged on CSRS."); +jsTest.log("Test that dropping a sharded collection, relevant events are properly logged on CSRS"); { // Create a sharded collection const db = getNewDb(); @@ -318,9 +299,7 @@ jsTest.log("Test that dropping a sharded collection, relevant events are logged assert.commandWorked(coll.insert({_id: -10})); // Drop the collection - const uuid = getCollectionUUID(coll.getFullName()); assert.commandWorked(db.runCommand({drop: coll.getName()})); - assertCollectionDropped(coll.getFullName(), uuid); // Verify that the drop collection start event has been logged const startLogCount = @@ -333,7 +312,7 @@ jsTest.log("Test that dropping a sharded collection, relevant events are logged assert.gte(1, endLogCount, "dropCollection end event not found in changelog"); } -jsTest.log("Test that dropping a sharded collection, the cached metadata on shards is cleaned up."); +jsTest.log("Test that dropping a sharded collection, the cached metadata on shards is cleaned up"); { // Create a sharded collection const db = getNewDb(); @@ -341,21 +320,24 @@ jsTest.log("Test that dropping a sharded collection, the cached metadata on shar assert.commandWorked( st.s.adminCommand({enableSharding: db.getName(), primaryShard: st.shard0.shardName})); assert.commandWorked(st.s.adminCommand({shardCollection: coll.getFullName(), key: {_id: 1}})); + + // Distribute the chunks among the shards + assert.commandWorked(st.s.adminCommand({split: coll.getFullName(), middle: {_id: 0}})); assert.commandWorked(coll.insert({_id: 10})); + assert.commandWorked(coll.insert({_id: -10})); - // At this point only one shard has valid filtering information (i.e. the one that has data). - // Below we are forcing to all shards to have their valid filtering information. For the shard - // that doesn't own any data, that means having a filtering information stating that no chunks - // are owned by that shard. - assert.commandWorked( - st.shard0.adminCommand({_flushRoutingTableCacheUpdates: coll.getFullName()})); - assert.commandWorked( - st.shard1.adminCommand({_flushRoutingTableCacheUpdates: coll.getFullName()})); + // Get the chunks cache collection name + const configCollDoc = st.s0.getDB('config').collections.findOne({_id: coll.getFullName()}); + const chunksCollName = 'cache.chunks.' + coll.getFullName(); // Drop the collection - const uuid = getCollectionUUID(coll.getFullName()); assert.commandWorked(db.runCommand({drop: coll.getName()})); - assertCollectionDropped(coll.getFullName(), uuid); + + // Verify that the cached metadata on shards is cleaned up + for (let configDb of [st.shard0.getDB('config'), st.shard1.getDB('config')]) { + assert.eq(configDb['cache.collections'].countDocuments({_id: coll.getFullName()}), 0); + assert(!configDb[chunksCollName].exists()); + } } st.stop(); diff --git a/jstests/sharding/drop_collection_if_uuid_not_matching.js b/jstests/sharding/drop_collection_if_uuid_not_matching.js index 81a0dacd2f7..59dde47a364 100644 --- a/jstests/sharding/drop_collection_if_uuid_not_matching.js +++ b/jstests/sharding/drop_collection_if_uuid_not_matching.js @@ -5,8 +5,8 @@ * - Keep the collection if the uuid is exactly the expected one. * * @tags: [ - * requires_fcv_51, # The command is not present in v5.0 - * does_not_support_stepdowns, # The command is not resilient to stepdowns + * requires_fcv_51, // The command is not present in v5.0 + * does_not_support_stepdowns, // The command is not resilient to stepdowns * ] */ diff --git a/jstests/sharding/drop_database_before_write_is_targeted.js b/jstests/sharding/drop_database_before_write_is_targeted.js deleted file mode 100644 index 18414baa1b9..00000000000 --- a/jstests/sharding/drop_database_before_write_is_targeted.js +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Verify that the write operation succeeds despite the database being dropped after the implicit - * creation and before the operation is targeted by the router. - * - * @tags: [ - * # TODO (SERVER-84043): Requires the mongos to define the fail point. Enable multiversion - * # once 8.0 becomes last LTS. - * multiversion_incompatible, - * # The createDatabase command inside of the write might fail with FailedToSatisfyReadPreference - * # if it does not find a primary. This is not a retriable error, and is correct, but - * # incompatible with this test. - * does_not_support_stepdowns, - * ] - */ - -load('jstests/libs/fail_point_util.js'); -load('jstests/libs/parallelTester.js'); - -const dbName = 'test'; -const collNS = dbName + '.foo'; - -const st = new ShardingTest({mongos: 1, shards: 1, config: 1}); - -// Pause the write operation after creating the database but before the operation is actually -// targeted by the router. -let failPoint = configureFailPoint(st.s, 'waitForDatabaseToBeDropped'); - -let insertThread = new Thread((mongosConnString, collNS) => { - let mongos = new Mongo(mongosConnString); - assert.commandWorked(mongos.getCollection(collNS).insert({})); -}, st.s0.host, collNS); - -// Perform a write operation, the database is implicitly created then the operation is paused. -insertThread.start(); -failPoint.wait(); - -// Before the router targets the write operation, the database is dropped. -assert.commandWorked(st.s0.getDB(dbName).runCommand({dropDatabase: 1})); -failPoint.off(); - -// The first targeting fails, then the database is recreated and the new targeting succeeds. -insertThread.join(); - -st.stop(); diff --git a/jstests/sharding/enforce_zone_policy.js b/jstests/sharding/enforce_zone_policy.js index 2ded590cb09..11a43d2572d 100644 --- a/jstests/sharding/enforce_zone_policy.js +++ b/jstests/sharding/enforce_zone_policy.js @@ -5,29 +5,25 @@ load("jstests/sharding/libs/find_chunks_util.js"); -const st = new ShardingTest({shards: 3, mongos: 1, other: {chunkSize: 1, enableAutoSplit: false}}); -const dbName = 'test'; -const collName = 'foo'; -const ns = dbName + '.' + collName; +var st = new ShardingTest({shards: 3, mongos: 1}); -assert.commandWorked( - st.s0.adminCommand({enablesharding: dbName, primaryShard: st.shard1.shardName})); +assert.commandWorked(st.s0.adminCommand({enablesharding: 'test'})); +st.ensurePrimaryShard('test', st.shard1.shardName); -var testDB = st.s0.getDB(dbName); +var testDB = st.s0.getDB('test'); var configDB = st.s0.getDB('config'); -assert.commandWorked(st.s0.adminCommand({shardCollection: ns, key: {_id: 1}})); - -const bigString = 'X'.repeat(1024 * 1024); // 1MB var bulk = testDB.foo.initializeUnorderedBulkOp(); for (var i = 0; i < 9; i++) { - bulk.insert({_id: i, x: bigString}); + bulk.insert({_id: i, x: i}); } assert.commandWorked(bulk.execute()); +assert.commandWorked(st.s0.adminCommand({shardCollection: 'test.foo', key: {_id: 1}})); + // Produce 9 chunks with min value at the documents just inserted for (var i = 0; i < 8; i++) { - assert.commandWorked(st.s0.adminCommand({split: ns, middle: {_id: i}})); + assert.commandWorked(st.s0.adminCommand({split: 'test.foo', middle: {_id: i}})); } /** @@ -39,7 +35,7 @@ function assertBalanceCompleteAndStable(checkFunc, stepName) { assert.soon(checkFunc, 'Balance at step ' + stepName + ' did not happen', 3 * 60 * 1000, 2000); - st.awaitBalancerRound(); + st.waitForBalancer(true, 60000); st.printShardingStatus(true); assert(checkFunc()); @@ -51,8 +47,8 @@ function assertBalanceCompleteAndStable(checkFunc, stepName) { * cluster is evenly balanced. */ function checkClusterEvenlyBalanced() { - assert.commandWorked(st.s.getDB('admin').runCommand({balancerStatus: 1})); - return true; + var maxChunkDiff = st.chunkDiff('foo', 'test'); + return maxChunkDiff <= 1; } st.startBalancer(); @@ -63,16 +59,17 @@ assertBalanceCompleteAndStable(checkClusterEvenlyBalanced, 'initial'); // Spread chunks correctly across zones st.addShardTag(st.shard0.shardName, 'a'); st.addShardTag(st.shard1.shardName, 'a'); -st.addTagRange(ns, {_id: -100}, {_id: 100}, 'a'); +st.addTagRange('test.foo', {_id: -100}, {_id: 100}, 'a'); st.addShardTag(st.shard2.shardName, 'b'); -st.addTagRange(ns, {_id: MinKey}, {_id: -100}, 'b'); -st.addTagRange(ns, {_id: 100}, {_id: MaxKey}, 'b'); +st.addTagRange('test.foo', {_id: MinKey}, {_id: -100}, 'b'); +st.addTagRange('test.foo', {_id: 100}, {_id: MaxKey}, 'b'); assertBalanceCompleteAndStable(function() { - var chunksOnShard2 = findChunksUtil.findChunksByNs(configDB, ns, {shard: st.shard2.shardName}) - .sort({min: 1}) - .toArray(); + var chunksOnShard2 = + findChunksUtil.findChunksByNs(configDB, 'test.foo', {shard: st.shard2.shardName}) + .sort({min: 1}) + .toArray(); jsTestLog('Chunks on shard2: ' + tojson(chunksOnShard2)); @@ -85,25 +82,24 @@ assertBalanceCompleteAndStable(function() { }, 'chunks to zones a and b'); // Tag the entire collection to shard0 and wait for everything to move to that shard -st.removeTagRange(ns, {_id: -100}, {_id: 100}, 'a'); -st.removeTagRange(ns, {_id: MinKey}, {_id: -100}, 'b'); -st.removeTagRange(ns, {_id: 100}, {_id: MaxKey}, 'b'); +st.removeTagRange('test.foo', {_id: -100}, {_id: 100}, 'a'); +st.removeTagRange('test.foo', {_id: MinKey}, {_id: -100}, 'b'); +st.removeTagRange('test.foo', {_id: 100}, {_id: MaxKey}, 'b'); st.removeShardTag(st.shard1.shardName, 'a'); st.removeShardTag(st.shard2.shardName, 'b'); -st.addTagRange(ns, {_id: MinKey}, {_id: MaxKey}, 'a'); +st.addTagRange('test.foo', {_id: MinKey}, {_id: MaxKey}, 'a'); assertBalanceCompleteAndStable(function() { - var counts = st.chunkCounts(collName); + var counts = st.chunkCounts('foo'); printjson(counts); - // All chunks must have been moved to shard 0, none left on shard 1 and 2 - return counts[st.shard0.shardName] > 0 && counts[st.shard1.shardName] == 0 && + return counts[st.shard0.shardName] == 11 && counts[st.shard1.shardName] == 0 && counts[st.shard2.shardName] == 0; }, 'all chunks to zone a'); // Remove all zones and ensure collection is correctly redistributed st.removeShardTag(st.shard0.shardName, 'a'); -st.removeTagRange(ns, {_id: MinKey}, {_id: MaxKey}, 'a'); +st.removeTagRange('test.foo', {_id: MinKey}, {_id: MaxKey}, 'a'); assertBalanceCompleteAndStable(checkClusterEvenlyBalanced, 'final'); diff --git a/jstests/sharding/error_propagation.js b/jstests/sharding/error_propagation.js index 1a74270f745..b4bb0b72331 100644 --- a/jstests/sharding/error_propagation.js +++ b/jstests/sharding/error_propagation.js @@ -20,6 +20,6 @@ assert.commandWorked(db.foo.insert({a: [1, 2]}, {writeConcern: {w: 3}})); var res = db.runCommand( {aggregate: 'foo', pipeline: [{$project: {total: {'$add': ['$a', 1]}}}], cursor: {}}); -assert.commandFailedWithCode(res, [16554, ErrorCodes.TypeMismatch]); +assert.commandFailedWithCode(res, 16554); st.stop(); }()); diff --git a/jstests/sharding/exhaust_hello_topology_changes.js b/jstests/sharding/exhaust_hello_topology_changes.js index b0173c9bc40..51b381838a5 100644 --- a/jstests/sharding/exhaust_hello_topology_changes.js +++ b/jstests/sharding/exhaust_hello_topology_changes.js @@ -1,8 +1,8 @@ /** - * Test to check that the RSM receives a hello reply "immediately" (or "quickly") after a RS + * Test to check that the RSM receives an isMaster reply "immediately" (or "quickly") after a RS * topology change when using the exhaust protocol. In order to test this, we'll set the * maxAwaitTimeMS to much higher than the default (5 mins). This will allow us to assert that the - * RSM receives the hello replies because of a topology change rather than maxAwaitTimeMS being + * RSM receives the isMaster replies because of a topology change rather than maxAwaitTimeMS being * hit. A replica set node should send a response to the mongos as soon as it processes a topology * change, so "immediately"/"quickly" can vary - we specify 5 seconds in this test ('timeoutMS'). * diff --git a/jstests/sharding/extract_shard_key_values.js b/jstests/sharding/extract_shard_key_values.js index 205cbc7f7b1..6969ad2b432 100644 --- a/jstests/sharding/extract_shard_key_values.js +++ b/jstests/sharding/extract_shard_key_values.js @@ -170,5 +170,13 @@ assert.writeErrorWithCode( mongos.getCollection(kNsName).update({b: 1}, {$set: {c: 2}}, {upsert: true}), ErrorCodes.ShardKeyNotFound); +// Find and modify will not treat missing shard key values as null and require the full shard key to +// be specified. +assert.commandWorked(sessionColl.insert({_id: "findAndModify", a: 1})); +assert.commandFailedWithCode( + sessionDB.runCommand( + {findAndModify: kCollName, query: {a: 1}, update: {$set: {updated: true}}}), + ErrorCodes.ShardKeyNotFound); + st.stop(); })(); diff --git a/jstests/sharding/features3.js b/jstests/sharding/features3.js index 9cd178b753b..9019c15ad19 100644 --- a/jstests/sharding/features3.js +++ b/jstests/sharding/features3.js @@ -8,7 +8,6 @@ // @tags: [ // expects_explicit_underscore_id_index, // ] - (function() { 'use strict'; @@ -151,5 +150,9 @@ assert(x.code == 13, "fsync on non-admin succeeded, but should have failed: " + x = dbForTest._adminCommand("fsync"); assert(x.ok == 1, "fsync failed: " + tojson(x)); +// test fsync+lock on admin db +x = dbForTest._adminCommand({"fsync": 1, lock: true}); +assert(!x.ok, "lock should fail: " + tojson(x)); + s.stop(); })(); diff --git a/jstests/sharding/findandmodify_autosplit.js b/jstests/sharding/findandmodify_autosplit.js index 9860df6ed83..f73a1cfa029 100644 --- a/jstests/sharding/findandmodify_autosplit.js +++ b/jstests/sharding/findandmodify_autosplit.js @@ -5,19 +5,9 @@ 'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled var st = new ShardingTest({shards: 1, mongos: 1, other: {chunkSize: 1, enableAutoSplit: true}}); -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - st.stop(); - return; -} - /* Return total number of chunks for a specific collection */ function getNumChunksForColl(coll) { return findChunksUtil.countChunksForNs(st.getDB('config'), coll.getFullName()); diff --git a/jstests/sharding/flushRoutingTableCacheUpdates_enforced_on_collections.js b/jstests/sharding/flushRoutingTableCacheUpdates_enforced_on_collections.js deleted file mode 100644 index 48b9094295a..00000000000 --- a/jstests/sharding/flushRoutingTableCacheUpdates_enforced_on_collections.js +++ /dev/null @@ -1,31 +0,0 @@ -// Ensure that a call to _flushRoutingTableCacheUpdates in a sharded cluster will return error if -// attempted on a database instead of a collection. -let st = new ShardingTest({}); -const testDBName = jsTestName(); -const collName = 'coll'; -const testDB = st.s.getDB(testDBName); - -assert.commandWorked(testDB.adminCommand({enableSharding: testDBName})); -assert.commandWorked( - testDB.adminCommand({shardCollection: testDB[collName].getFullName(), key: {x: 1}})); - -// On a collection, the command works -assert.commandWorked( - st.shard0.adminCommand({_flushRoutingTableCacheUpdates: testDB[collName].getFullName()})); - -// But on a database, the command fails with error "IllegalOperation" -assert.commandFailedWithCode(st.shard0.adminCommand({_flushRoutingTableCacheUpdates: testDBName}), - ErrorCodes.IllegalOperation); - -// Test also variant _flushRoutingTableCacheUpdatesWithWriteConcern -assert.commandWorked(st.shard0.adminCommand({ - _flushRoutingTableCacheUpdatesWithWriteConcern: testDB[collName].getFullName(), - writeConcern: {w: "majority"} -})); -assert.commandFailedWithCode(st.shard0.adminCommand({ - _flushRoutingTableCacheUpdatesWithWriteConcern: testDBName, - writeConcern: {w: "majority"} -}), - ErrorCodes.IllegalOperation); - -st.stop(); diff --git a/jstests/sharding/fsync_deadlock.js b/jstests/sharding/fsync_deadlock.js deleted file mode 100644 index 8a8a705c170..00000000000 --- a/jstests/sharding/fsync_deadlock.js +++ /dev/null @@ -1,118 +0,0 @@ -/* -This test runs a cross-shard transaction where the transaction holds onto the collection locks, then -runs an fsyncLock which should fail and timeout as the global S lock cannot be taken. - * @tags: [ - * requires_sharding, - * requires_fsync, - * ] - */ - -load('jstests/libs/fail_point_util.js'); // For configureFailPoint -load('jstests/libs/parallelTester.js'); - -const st = new ShardingTest({ - shards: 2, - mongos: 1, - config: 1, -}); -const shard0Primary = st.rs0.getPrimary(); -const shard1Primary = st.rs1.getPrimary(); - -// Set up a sharded collection with two chunks -const dbName = "testDb"; -const collName = "testColl"; -const ns = dbName + "." + collName; -assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); -assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}})); -assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); -assert.commandWorked( - st.s.adminCommand({moveChunk: ns, find: {x: MinKey}, to: st.shard0.shardName})); -assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: 1}, to: st.shard1.shardName})); - -const isMultiversion = - jsTest.options().shardMixedBinVersions || jsTest.options().useRandomBinVersionsWithinReplicaSet; - -if (isMultiversion) { - // Force the recipient shard to refresh its routing table for the collection since the recipient - // refresh during chunk migration (in 5.0 and previous versions) is only best-effort, and mongos - // would only retry a transaction on a StaleConfig error if the error is thrown by the first - // participant shard. - assert.commandWorked(shard1Primary.adminCommand({_flushRoutingTableCacheUpdates: ns})); -} - -function waitForFsyncLockToWaitForLock(st, numThreads) { - assert.soon(() => { - let ops = st.s.getDB('admin') - .aggregate([ - {$currentOp: {allUsers: true, idleConnections: true}}, - {$match: {desc: "fsyncLockWorker", waitingForLock: true}}, - ]) - .toArray(); - if (ops.length != numThreads) { - jsTest.log("Num operations: " + ops.length + ", expected: " + numThreads); - jsTest.log(ops); - return false; - } - return true; - }); -} - -function runTxn(mongosHost, dbName, collName) { - const mongosConn = new Mongo(mongosHost); - jsTest.log("Starting a cross-shard transaction with shard0 and shard1 as the participants " + - "and shard0 as the coordinator shard"); - const lsid = {id: UUID()}; - const txnNumber = NumberLong(35); - assert.commandWorked(mongosConn.getDB(dbName).runCommand({ - insert: collName, - documents: [{x: -1}], - lsid, - txnNumber, - startTransaction: true, - autocommit: false, - })); - assert.commandWorked(mongosConn.getDB(dbName).runCommand({ - insert: collName, - documents: [{x: 1}], - lsid, - txnNumber, - autocommit: false, - })); - jsTest.log("Committing the cross-shard transaction"); - assert.commandWorked( - mongosConn.adminCommand({commitTransaction: 1, lsid, txnNumber, autocommit: false})); - jsTest.log("Committed the cross-shard transaction"); -} - -function runFsyncLock(primaryHost) { - let primaryConn = new Mongo(primaryHost); - let ret = assert.commandFailed( - primaryConn.adminCommand({fsync: 1, lock: true, fsyncLockAcquisitionTimeoutMillis: 5000})); - let errmsg = "Fsync lock timed out"; - assert.eq(ret.errmsg.includes(errmsg), true); -} - -// Run a cross-shard transaction that has shard0 as the coordinator. Make the TransactionCoordinator -// thread hang right before the commit decision is written (i.e. after the transaction has entered -// the "prepared" state). -// This way the txn thread holds onto the collection locks -let writeDecisionFp = configureFailPoint(shard0Primary, "hangBeforeWritingDecision"); -let txnThread = new Thread(runTxn, st.s.host, dbName, collName); -txnThread.start(); -writeDecisionFp.wait(); - -let fsyncLockThread = new Thread(runFsyncLock, st.s.host); -fsyncLockThread.start(); - -// Wait for fsyncLockWorker threads on the shard primaries to wait for the global S lock (enqueued -// in the conflict queue). -waitForFsyncLockToWaitForLock(st, 2 /*blocked fsyncLockWorker threads*/); - -// Unpause the TransactionCoordinator. -// The transaction thread can now acquire the IX locks since the blocking fsyncLock request (with an -// incompatible Global S lock) has been removed from the conflict queue -writeDecisionFp.off(); - -fsyncLockThread.join(); -txnThread.join(); -st.stop(); diff --git a/jstests/sharding/fsync_lock_ddl_lock.js b/jstests/sharding/fsync_lock_ddl_lock.js deleted file mode 100644 index 54225b898c5..00000000000 --- a/jstests/sharding/fsync_lock_ddl_lock.js +++ /dev/null @@ -1,68 +0,0 @@ -/** - * This test script - * - Runs a DDL operation and waits before the DDL op takes a lock, calls fsync with lock: true, and - * verifies that the DDL op can take the lock after fsyncUnlock call. - * - * @tags: [ - * requires_fsync, - * ] - */ - -load('jstests/libs/fail_point_util.js'); - -(function() { -"use strict"; -const dbName = "test"; -const collName = "collTest"; -const renamedCollName = "collTest1"; -const st = new ShardingTest({shards: 2, mongos: 1, config: 1}); -const db = st.s0.getDB(dbName); -assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); -st.ensurePrimaryShard(dbName, st.shard0.shardName); -const coll = st.s.getDB(dbName).getCollection(collName); -coll.insert({x: 1}); -assert.eq(coll.count(), 1); - -function waitUntilOpCountIs(opFilter, num, st) { - assert.soon(() => { - let ops = st.s.getDB('admin') - .aggregate([ - {$currentOp: {allUsers: true}}, - {$match: opFilter}, - ]) - .toArray(); - if (ops.length != num) { - jsTest.log("Num operations: " + ops.length + ", expected: " + num); - jsTest.log(ops); - return false; - } - return true; - }); -} - -let ddlCoordinatorFailPoint = - configureFailPoint(st.getPrimaryShard(dbName), 'hangBeforeRunningCoordinatorInstance'); - -let codeToRun = () => { - const collName = "collTest"; - let sourceNss = db[collName].getFullName(); - let destNss = sourceNss + "1"; - assert.commandWorked(db.adminCommand({renameCollection: sourceNss, to: destNss})); -}; - -let ddlOpHandle = startParallelShell(codeToRun, st.s.port); - -ddlCoordinatorFailPoint.wait(); - -assert.commandWorked(st.s.adminCommand({fsync: 1, lock: true})); - -waitUntilOpCountIs({desc: 'RenameCollectionCoordinator'}, 1, st); - -ddlCoordinatorFailPoint.off(); -assert.commandWorked(st.s.adminCommand({fsyncUnlock: 1})); - -ddlOpHandle(); -assert.commandWorked(db[renamedCollName].insert({x: 2})); -assert.eq(db[renamedCollName].count(), 2); -st.stop(); -}()); diff --git a/jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js b/jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js deleted file mode 100644 index b6c797c18fd..00000000000 --- a/jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * This test script - * - Verifies that fsync with lock: true fails when a DDL operation is in progress. - * - * @tags: [ - * requires_fsync, - * ] - */ - -load('jstests/libs/fail_point_util.js'); - -(function() { -"use strict"; -const dbName = "test"; -const collName = "collTest"; -const ns = dbName + "." + collName; -const st = new ShardingTest({shards: 2, mongos: 1, config: 1}); -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard1.shardName})); -assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {_id: 1}})); -const coll = st.s.getDB(dbName).getCollection(collName); -coll.insert({x: 1}); -assert.eq(coll.count(), 1); - -// Start refineCollectionShardKey DDL operation -let newShardKey = {_id: 1, x: 1}; -st.s.getCollection(ns).createIndex(newShardKey); -let ddlOpThread = new Thread((mongosConnString, nss, newShardKey) => { - let mongos = new Mongo(mongosConnString); - mongos.adminCommand({refineCollectionShardKey: nss, key: newShardKey}); -}, st.s0.host, ns, newShardKey); -let ddlCoordinatorFailPoint = - configureFailPoint(st.rs1.getPrimary(), 'hangBeforeRemovingCoordinatorDocument'); - -ddlOpThread.start(); -ddlCoordinatorFailPoint.wait(); - -// Run fsync command, should fail when DDL op is in progress -let fsyncLockCommand = assert.commandFailed(st.s.adminCommand({fsync: 1, lock: true})); -const errmsg = "Cannot take lock while DDL operation is in progress"; -assert.eq(fsyncLockCommand.errmsg.includes(errmsg), true); - -ddlCoordinatorFailPoint.off(); -ddlOpThread.join(); - -// ensure writes are allowed since fsync failed -coll.insert({x: 2}); -assert.eq(coll.count(), 2); - -st.stop(); -}()); diff --git a/jstests/sharding/fsync_lock_unlock.js b/jstests/sharding/fsync_lock_unlock.js deleted file mode 100644 index b30c0f9f073..00000000000 --- a/jstests/sharding/fsync_lock_unlock.js +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Verifies the fsync with lock+unlock command on mongos. - * @tags: [ - * requires_fsync, - * uses_parallel_shell, - * ] - */ - -(function() { -"use strict"; - -const dbName = "test"; -const collName = "collTest"; -const ns = dbName + "." + collName; -const st = new ShardingTest({shards: 2, mongos: 1, config: 1}); -const adminDB = st.s.getDB('admin'); -const distributed_txn_insert_count = 10; - -function waitUntilOpCountIs(opFilter, num, st) { - assert.soon(() => { - let ops = st.s.getDB('admin') - .aggregate([ - {$currentOp: {}}, - {$match: opFilter}, - ]) - .toArray(); - if (ops.length != num) { - jsTest.log("Num operations: " + ops.length + ", expected: " + num); - jsTest.log(ops); - return false; - } - return true; - }); -} - -function runTransaction() { - load("jstests/libs/auto_retry_transaction_in_sharding.js"); - // Start the transaction and insert a document. - const sessionOptions = {causalConsistency: false}; - const session = db.getSiblingDB("test").getMongo().startSession(sessionOptions); - const sessionDb = session.getDatabase("test"); - const sessionColl = sessionDb["collTest"]; - - session.endSession(); - - withTxnAndAutoRetryOnMongos(session, () => { - for (let i = 0; i < 10; i++) { - assert.commandWorked(sessionColl.insert({x: i})); - } - }, {}); - - jsTest.log("END txn in parallel shell"); -} - -let collectionCount = 1; -const performFsyncLockUnlockWithReadWriteOperations = function() { - // lock then unlock - assert.commandWorked(st.s.adminCommand({fsync: 1, lock: true})); - - // Make sure writes are blocked. Spawn a write operation in a separate shell and make sure it - // is blocked. There is really no way to do that currently, so just check that the write didn't - // go through. - let codeToRun = () => { - assert.commandWorked(db.getSiblingDB("test").getCollection("collTest").insert({x: 1})); - }; - - let writeOpHandle = startParallelShell(codeToRun, st.s.port); - - waitUntilOpCountIs({op: 'insert', ns: 'test.collTest', waitingForLock: true}, 1, st); - - // Make sure reads can still run even though there is a pending write and also that the write - // didn't get through. - assert.eq(collectionCount, coll.count()); - assert.commandWorked(st.s.adminCommand({fsyncUnlock: 1})); - - writeOpHandle(); - - // ensure writers are allowed after the cluster is unlocked - assert.commandWorked(coll.insert({x: 1})); - collectionCount += 2; - assert.eq(coll.count(), collectionCount); - - // Ensure that distributed transactions are blocked when the cluster is locked - assert.commandWorked(st.s.adminCommand({fsync: 1, lock: true})); - - let txnOpHandle = startParallelShell(runTransaction, st.s.port); - - // Verify that txns are unsuccessful when cluster is locked. - assert.eq(collectionCount, st.s.getCollection(coll.getFullName()).countDocuments({})); - assert.commandWorked(st.s.adminCommand({fsyncUnlock: 1})); - - txnOpHandle(); - collectionCount += distributed_txn_insert_count; - - // Verify that txns are successful after cluster is unlocked. - assert.eq(collectionCount, st.s.getCollection(coll.getFullName()).countDocuments({})); - - // Ensure that fsync (lock: false) still works by performing a write after invoking the command, - // and checking the write is successful, showing the cluster does not need to be unlocked. - assert.commandWorked(st.s.adminCommand({fsync: 1, lock: false})); - assert.commandWorked(coll.insert({x: 2})); - collectionCount += 1; - assert.eq(coll.count(), collectionCount); -}; - -jsTest.log("Insert some data."); -const coll = st.s0.getDB(dbName)[collName]; -assert.commandWorked(coll.insert({x: 1})); - -// unlock before lock should fail -let ret = assert.commandFailed(st.s.adminCommand({fsyncUnlock: 1})); -let errmsg = "fsyncUnlock called when not locked"; -assert.eq(ret.errmsg.includes(errmsg), true); - -performFsyncLockUnlockWithReadWriteOperations(); - -st.stop(); -}()); diff --git a/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js b/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js index 66433f99037..5b4b582f164 100644 --- a/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js +++ b/jstests/sharding/health_monitor/progress_monitor_doesnt_crash_on_not_critical.js @@ -1,7 +1,7 @@ /* * @tags: [multiversion_incompatible] */ -const PROGRESS_TIMEOUT_SECONDS = 3; +const PROGRESS_TIMEOUT_SECONDS = 5; (function() { 'use strict'; @@ -45,7 +45,7 @@ assert.commandWorked( st.s1.adminCommand({"configureFailPoint": 'hangTestHealthObserver', "mode": "alwaysOn"})); // Wait for the progress monitor timeout to elapse. -sleep(2.1 * PROGRESS_TIMEOUT_SECONDS * 1000); +sleep(1.1 * PROGRESS_TIMEOUT_SECONDS * 1000); // Servers should be still be alive. jsTestLog("Expect monogs processes to still be alive"); @@ -55,7 +55,7 @@ jsTestLog("Change observer to critical"); changeObserverIntensity(st.s1, 'test', 'critical'); // Wait for the progress monitor timeout to elapse. -sleep((2.1 * PROGRESS_TIMEOUT_SECONDS * 1000) + 1000); +sleep(1.1 * PROGRESS_TIMEOUT_SECONDS * 1000); jsTestLog("Done sleeping"); // Servers should be still be alive. diff --git a/jstests/sharding/hedged_reads.js b/jstests/sharding/hedged_reads.js index 10b655d5608..bd121680d0a 100644 --- a/jstests/sharding/hedged_reads.js +++ b/jstests/sharding/hedged_reads.js @@ -1,10 +1,5 @@ /** * Tests hedging metrics in the serverStatus output. - * @tags: [ - * # This test is known to be racey due to implementation of hedged reads (SERVER-65329). - * # Disable windows testing as this feature is deprecated in v8.0. - * incompatible_with_windows_tls, - * ] */ (function() { "use strict"; @@ -100,9 +95,7 @@ try { setCommandDelay(sortedNodes[0], "count", kBlockCmdTimeMS, ns); // Make the hedged request block for a while to allow the operation to start on the other node. - // The delay is intentionally large so we avoid the race where a killOp can arrive before the - // request. - setCommandDelay(sortedNodes[1], "count", 1000, ns); + setCommandDelay(sortedNodes[1], "count", 100, ns); const comment = "test_kill_initial_request_" + ObjectId(); assert.commandWorked(testDB.runCommand({ @@ -133,9 +126,7 @@ try { setCommandDelay(sortedNodes[1], "count", kBlockCmdTimeMS, ns); // Make the initial request block for a while to allow the operation to start on the other node. - // The delay is intentionally large so we avoid the race where a killOp can arrive before the - // request. - setCommandDelay(sortedNodes[0], "count", 1000, ns); + setCommandDelay(sortedNodes[0], "count", 100, ns); const comment = "test_kill_additional_request_" + ObjectId(); assert.commandWorked(testDB.runCommand({ diff --git a/jstests/sharding/hidden_index.js b/jstests/sharding/hidden_index.js deleted file mode 100644 index 5dfdce03dfb..00000000000 --- a/jstests/sharding/hidden_index.js +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Test to validate that a shard key index cannot be hidden if it cannot be dropped. - * @tags: [ - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; -load("jstests/libs/index_catalog_helpers.js"); // For IndexCatalogHelpers.findByName. - -// Test to validate the correct behaviour of hiding an index in a sharded cluster with a shard key. -function validateHiddenIndexBehaviour() { - let index_type = 1; - let index_name = "a_" + index_type; - assert.commandWorked(coll.createIndex({"a": index_type})); - - let idxSpec = IndexCatalogHelpers.findByName(coll.getIndexes(), index_name); - assert.eq(idxSpec.hidden, undefined); - - assert.commandWorked(coll.hideIndex(index_name)); - idxSpec = IndexCatalogHelpers.findByName(coll.getIndexes(), index_name); - assert(idxSpec.hidden); - - assert.commandWorked(coll.unhideIndex(index_name)); - idxSpec = IndexCatalogHelpers.findByName(coll.getIndexes(), index_name); - assert.eq(idxSpec.hidden, undefined); - - assert.commandWorked(coll.dropIndex(index_name)); - assert.commandWorked(coll.createIndex({"a": index_type}, {hidden: true})); - - idxSpec = IndexCatalogHelpers.findByName(coll.getIndexes(), index_name); - assert(idxSpec.hidden); - assert.commandWorked(coll.dropIndexes()); -} - -// Check that command will fail when we try to hide the only shard key index of the collection -function validateOneShardKeyHiddenIndexBehaviour() { - assert.commandFailedWithCode(coll.hideIndex({skey: 1}), ErrorCodes.InvalidOptions); - assert.commandFailedWithCode(coll.hideIndex("skey_1"), ErrorCodes.InvalidOptions); - assert.commandFailedWithCode( - testDb.runCommand({"collMod": coll.getName(), "index": {"name": "skey_1", "hidden": true}}), - ErrorCodes.InvalidOptions); -} - -// Check that command will fail when we try to hide or drop an index that is the last shard key -// index of the collection -function validateDifferentHiddenIndexesBehaviour() { - // Create index on skey - assert.commandWorked(coll.createIndex({skey: 1, anotherkey: 1})); - - // Check that is possible to hide a shard key index using its key pattern - assert.commandWorked(coll.hideIndex({skey: 1})); - assert.commandWorked(coll.unhideIndex({skey: 1})); - - // Check that is possible to hide a shard key index using its name - assert.commandWorked(testDb.runCommand( - {"collMod": coll.getName(), "index": {"name": "skey_1", "hidden": true}})); - - assert.commandFailedWithCode( - testDb.runCommand( - {"collMod": coll.getName(), "index": {"name": "skey_1_anotherkey_1", "hidden": true}}), - ErrorCodes.InvalidOptions); - - assert.commandFailed(coll.dropIndex("skey_1_anotherkey_1")); -} - -// Configure initial sharded cluster -const st = new ShardingTest({shards: 2}); -const mongos = st.s; -const testDb = mongos.getDB("test"); -const coll = testDb.getCollection("foo"); - -// Enable sharding at collection 'foo' and create a new shard key -assert.commandWorked(st.s.adminCommand({enableSharding: testDb.getName()})); - -// Crate a new shard key -assert.commandWorked( - st.s.adminCommand({shardcollection: testDb.getName() + '.' + coll.getName(), key: {skey: 1}})); - -validateHiddenIndexBehaviour(); -validateOneShardKeyHiddenIndexBehaviour(); -validateDifferentHiddenIndexesBehaviour(); - -st.stop(); -})(); diff --git a/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js b/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js deleted file mode 100644 index f500091c7cf..00000000000 --- a/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js +++ /dev/null @@ -1,56 +0,0 @@ -(function() { -"use strict"; - -function shardKnowledgeIsShardedOrUnknown(shard, nss) { - let res = assert.commandWorked(shard.adminCommand({getShardVersion: nss, fullMetadata: true})); - return (typeof res.global == 'string' && res.global == 'UNKNOWN') || - (typeof res.metadata == 'object' && typeof res.metadata.collVersion != 'undefined'); -} - -const st = new ShardingTest({shards: 2, mongos: 1}); - -void function testOptimizedShardCollection() { - const dbName = 'testDB1'; - const collName = 'testColl1'; - - jsTest.log("Testing that implicit collection creation triggered by optimized " + - "shardCollection leaves all shards with the expected knowledge"); - - assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); - - assert.commandWorked( - st.s.adminCommand({shardCollection: `${dbName}.${collName}`, key: {_id: 'hashed'}})); - - assert(shardKnowledgeIsShardedOrUnknown(st.shard0, `${dbName}.${collName}`), - "Unexpected sharding state in Shard 0"); - assert(shardKnowledgeIsShardedOrUnknown(st.shard1, `${dbName}.${collName}`), - "Unexpected sharding state in Shard 1"); -}(); - -void function testmovePrimary() { - const dbName = 'testDB2'; - const collName = 'testColl2'; - - jsTest.log("Testing that implicit collection creation triggered by movePrimary " + - "leaves all shards with the expected knowledge"); - - assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); - - assert.commandWorked( - st.s.adminCommand({shardCollection: `${dbName}.${collName}`, key: {_id: 1}})); - - assert.commandWorked(st.s.adminCommand({ - movePrimary: dbName, - to: st.shard1.name, - })); - - assert(shardKnowledgeIsShardedOrUnknown(st.shard0, `${dbName}.${collName}`), - "Unexpected sharding state in Shard 0"); - assert(shardKnowledgeIsShardedOrUnknown(st.shard1, `${dbName}.${collName}`), - "Unexpected sharding state in Shard 1"); -}(); - -st.stop(); -})();
\ No newline at end of file diff --git a/jstests/sharding/index_and_collection_option_propagation.js b/jstests/sharding/index_and_collection_option_propagation.js index f3fd2d00e3c..d58d5dba446 100644 --- a/jstests/sharding/index_and_collection_option_propagation.js +++ b/jstests/sharding/index_and_collection_option_propagation.js @@ -137,8 +137,7 @@ assert.eq(undefined, res.raw[st.shard2.host], tojson(res)); assert.commandWorked(res); checkShardIndexes("idx1", [], [st.shard1, st.shard2]); -// collMod targets all shards, regardless of whether they have chunks. The shards that have no -// chunks for the collection will not be included in the responses. +// collMod const validationOption2 = { dummyField2: {$type: "string"} }; @@ -150,9 +149,9 @@ res = st.s.getDB(dbName).runCommand({ }); assert.commandWorked(res); assert.eq(undefined, res.raw[st.shard0.host], tojson(res)); -assert.eq(1, res.raw[st.shard1.host].ok, tojson(res)); +assert.eq(res.raw[st.shard1.host].ok, 1, tojson(res)); assert.eq(undefined, res.raw[st.shard2.host], tojson(res)); -checkShardCollOption("validator", validationOption2, [st.shard0, st.shard1], [st.shard2]); +checkShardCollOption("validator", validationOption2, [st.shard1], [st.shard2]); // Check that errors from shards are aggregated correctly. diff --git a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js index d2fad117f10..44ffd313d71 100644 --- a/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js +++ b/jstests/sharding/index_operations_abort_concurrent_outgoing_migrations.js @@ -1,31 +1,25 @@ /* * Test that the index commands abort concurrent outgoing migrations. */ - (function() { "use strict"; load('jstests/libs/chunk_manipulation_util.js'); load("jstests/libs/parallelTester.js"); load("jstests/sharding/libs/sharded_index_util.js"); -load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // Test deliberately inserts orphans outside of migration. TestData.skipCheckOrphans = true; -// TODO (SERVER-91380): remove skipCheckingIndexesConsistentAcrossCluster flag. -TestData.skipCheckingIndexesConsistentAcrossCluster = true; /* * Runs moveChunk on the host to move the chunk to the given shard. */ -function runMoveChunk(host, ns, fromShard, toShard, findOneChunkFunction) { +function runMoveChunk(host, ns, findCriteria, toShard) { const mongos = new Mongo(host); - const chunk = findOneChunkFunction(mongos.getDB('config'), ns, {shard: fromShard}); let res, hasRetriableError; do { hasRetriableError = false; - res = mongos.adminCommand({moveChunk: ns, bounds: [chunk.min, chunk.max], to: toShard}); + res = mongos.adminCommand({moveChunk: ns, find: findCriteria, to: toShard}); // If a migration is interrupted by an index build, the test may run another migration // before the recipient discovers the first one failed, leading to transient // ConflictingOperationInProgress errors. @@ -47,12 +41,7 @@ function assertCommandAbortsConcurrentOutgoingMigration(st, stepName, ns, cmdFun // Turn on the fail point and wait for moveChunk to hit the fail point. pauseMoveChunkAtStep(fromShard, stepName); - let moveChunkThread = new Thread(runMoveChunk, - st.s.host, - ns, - fromShard.shardName, - toShard.shardName, - findChunksUtil.findOneChunkByNs); + let moveChunkThread = new Thread(runMoveChunk, st.s.host, ns, {_id: MinKey}, toShard.shardName); moveChunkThread.start(); waitForMoveChunkStep(fromShard, stepName); @@ -111,38 +100,6 @@ stepNames.forEach((stepName) => { } }); -if (!jsTestOptions().shardMixedBinVersions && - !jsTest.options().useRandomBinVersionsWithinReplicaSet) { - stepNames.forEach((stepName) => { - jsTest.log( - `Testing that single phase createIndexes aborts concurrent outgoing migrations that are in step ${ - stepName}...`); - const collName = "testSinglePhaseCreateIndexesMoveChunkStep" + stepName; - const ns = dbName + "." + collName; - - assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: shardKey})); - - assertCommandAbortsConcurrentOutgoingMigration(st, stepName, ns, () => { - const coll = st.s.getCollection(ns); - - assert.commandWorked(coll.createIndexes([index])); - }); - - // Verify that the index command succeeds. - ShardedIndexUtil.assertIndexExistsOnShard(st.shard0, dbName, collName, index); - - // If createIndexes is run after the migration has reached the steady state, shard1 - // will not have the index created by the command because the index just does not - // exist when shard1 clones the collection options and indexes from shard0. However, - // if createIndexes is run after the cloning step starts but before the steady state - // is reached, shard0 may have the index when shard1 does the cloning so shard1 may - // or may not have the index. - if (stepName == moveChunkStepNames.reachedSteadyState) { - ShardedIndexUtil.assertIndexDoesNotExistOnShard(st.shard1, dbName, collName, index); - } - }); -} - stepNames.forEach((stepName) => { jsTest.log(`Testing that dropIndexes aborts concurrent outgoing migrations that are in step ${ stepName}...`); @@ -172,31 +129,5 @@ stepNames.forEach((stepName) => { } }); -// TODO: Remove feature flag check once project is backported. -if (FeatureFlagUtil.isEnabled(st.shard0.getDB('admin'), "ShardKeyIndexOptionalHashedSharding") && - FeatureFlagUtil.isEnabled(st.shard1.getDB('admin'), "ShardKeyIndexOptionalHashedSharding")) { - stepNames.forEach((stepName) => { - jsTest.log( - `Testing that dropIndex of a hashed shard key index aborts concurrent outgoing migrations that are in step ${ - stepName}...`); - const collName = "testDropHashedShardKeyIndexMoveChunkStep" + stepName; - const ns = dbName + "." + collName; - const hashedShardKey = {_id: "hashed"}; - - assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: hashedShardKey})); - ShardedIndexUtil.assertIndexExistsOnShard(st.shard0, dbName, collName, hashedShardKey); - ShardedIndexUtil.assertIndexExistsOnShard(st.shard1, dbName, collName, hashedShardKey); - - assertCommandAbortsConcurrentOutgoingMigration(st, stepName, ns, () => { - assert.commandWorked(st.s.getCollection(ns).dropIndexes(hashedShardKey)); - }); - - // Verify dropping the shard key index succeeds. - ShardedIndexUtil.assertIndexDoesNotExistOnShard( - st.shard0, dbName, collName, hashedShardKey); - // TODO (SERVER-91380): assert the shard key is not present on recipient shard1 as well. - }); -} - st.stop(); })(); diff --git a/jstests/sharding/internal_txns/incomplete_transaction_history_during_migration.js b/jstests/sharding/internal_txns/incomplete_transaction_history_during_migration.js deleted file mode 100644 index c9b85d2c426..00000000000 --- a/jstests/sharding/internal_txns/incomplete_transaction_history_during_migration.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Test that chunk migration can migrate a retryable internal transaction whose oplog entries have - * been truncated. - * - * @tags: [requires_fcv_60, uses_transactions, requires_persistence] - */ -(function() { -'use strict'; - -// This test involves writing directly to the config.transactions collection which is not allowed -// in a session. -TestData.disableImplicitSessions = true; - -const st = new ShardingTest({shards: 2, rs: {nodes: 1}}); - -const dbName = 'testDb'; -const collName = 'testColl'; -const ns = dbName + '.' + collName; -const testDB = st.s.getDB(dbName); - -assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); -st.ensurePrimaryShard(dbName, st.shard0.name); - -assert.commandWorked(st.s.getCollection(ns).createIndex({x: 1})); -assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}})); - -assert.commandWorked(st.s.getDB(dbName).runCommand({insert: collName, documents: [{x: 1}]})); - -const parentLsid = { - id: UUID() -}; -const parentTxnNumber = NumberLong(35); - -const originalChildLsid = { - id: parentLsid.id, - txnNumber: parentTxnNumber, - txnUUID: UUID() -}; -const childTxnNumber = NumberLong(1); - -const updateCmdObj = { - update: collName, - updates: [{q: {x: 1}, u: {$set: {y: 1}}}], - stmtId: NumberInt(0), -}; -const res0 = assert.commandWorked(testDB.runCommand(Object.assign({}, updateCmdObj, { - lsid: originalChildLsid, - txnNumber: childTxnNumber, - autocommit: false, - startTransaction: true -}))); -assert.eq(res0.nModified, 1, res0); -assert.commandWorked(st.s.adminCommand({ - commitTransaction: 1, - lsid: originalChildLsid, - txnNumber: childTxnNumber, - autocommit: false, -})); - -// Manually update the config.transactions document for the retryable internal transaction to point -// to an invalid op time. -const shard0ConfigTxnsColl = st.rs0.getPrimary().getCollection("config.transactions"); -const res1 = assert.commandWorked(shard0ConfigTxnsColl.update( - {"_id.txnUUID": originalChildLsid.txnUUID}, - {$set: {lastWriteOpTime: {ts: new Timestamp(100, 1), t: NumberLong(1)}}})); -assert.eq(res1.nModified, 1, res1); - -assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); -assert.commandWorked(st.s.adminCommand({moveChunk: ns, find: {x: 0}, to: st.shard1.shardName})); - -assert.commandFailedWithCode(testDB.runCommand(Object.assign( - {}, updateCmdObj, {lsid: parentLsid, txnNumber: parentTxnNumber})), - ErrorCodes.IncompleteTransactionHistory); - -const retryChildLsid = { - id: parentLsid.id, - txnNumber: parentTxnNumber, - txnUUID: UUID() -}; -assert.commandFailedWithCode(testDB.runCommand(Object.assign({}, updateCmdObj, { - lsid: retryChildLsid, - txnNumber: childTxnNumber, - autocommit: false, - startTransaction: true -})), - ErrorCodes.IncompleteTransactionHistory); - -st.stop(); -})(); diff --git a/jstests/sharding/internal_txns/libs/chunk_migration_test.js b/jstests/sharding/internal_txns/libs/chunk_migration_test.js index a96b030890a..cc78170153e 100644 --- a/jstests/sharding/internal_txns/libs/chunk_migration_test.js +++ b/jstests/sharding/internal_txns/libs/chunk_migration_test.js @@ -360,7 +360,7 @@ function InternalTransactionChunkMigrationTest(storeFindAndModifyImagesInSideCol testCase.setUpFunc(); const lsid = getTransactionSessionId(txnType, testCase); - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { const txnNumber = getNextTxnNumber(txnType, testCase); for (let i = 0; i < testCase.commands.length; i++) { @@ -401,7 +401,7 @@ function InternalTransactionChunkMigrationTest(storeFindAndModifyImagesInSideCol const lsid = getTransactionSessionId(txnType, testCase); // Give the session a different txnUUID to simulate a retry from a different mongos. lsid.txnUUID = UUID(); - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { const txnNumber = getNextTxnNumber(txnType, testCase); for (let i = 0; i < testCase.commands.length; i++) { diff --git a/jstests/sharding/internal_txns/libs/fixture_helpers.js b/jstests/sharding/internal_txns/libs/fixture_helpers.js index 8b215b98d14..5e418726fc6 100644 --- a/jstests/sharding/internal_txns/libs/fixture_helpers.js +++ b/jstests/sharding/internal_txns/libs/fixture_helpers.js @@ -5,7 +5,28 @@ function runTxnRetryOnTransientError(txnFunc) { return true; } catch (e) { if (e.hasOwnProperty('errorLabels') && - e.errorLabels.includes('TransientTransactionError')) { + e.errorLabels.includes('TransientTransactionError') && + e.code != ErrorCodes.NoSuchTransaction) { + // Don't retry on a NoSuchTransaction error since it implies the transaction was + // aborted so we should propagate the error instead. + jsTest.log("Failed to run transaction due to a transient error " + tojson(e)); + return false; + } else { + throw e; + } + } + }); +} + +function runTxnRetryOnLockTimeoutError(txnFunc) { + assert.soon(() => { + try { + txnFunc(); + return true; + } catch (e) { + if (e.hasOwnProperty('errorLabels') && + e.errorLabels.includes('TransientTransactionError') && + e.code == ErrorCodes.LockTimeout) { jsTest.log("Failed to run transaction due to a transient error " + tojson(e)); return false; } else { diff --git a/jstests/sharding/internal_txns/libs/retryable_internal_transaction_test.js b/jstests/sharding/internal_txns/libs/retryable_internal_transaction_test.js index 1051bf1fc28..0332843f72c 100644 --- a/jstests/sharding/internal_txns/libs/retryable_internal_transaction_test.js +++ b/jstests/sharding/internal_txns/libs/retryable_internal_transaction_test.js @@ -129,7 +129,7 @@ function RetryableInternalTransactionTest(collectionOptions = {}) { // Initial try. const initialLsid = txnOptions.makeSessionIdFunc(); let initialTxnNumber = 0; - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { initialTxnNumber++; setTxnFields(cmdObj, initialLsid, initialTxnNumber); assert.commandWorked(mongosTestDB.runCommand(cmdObj)); @@ -175,7 +175,7 @@ function RetryableInternalTransactionTest(collectionOptions = {}) { const initialLsid = txnOptions.makeSessionIdFunc(); let initialTxnNumber = 0; let initialRes; - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { initialTxnNumber++; setTxnFields(cmdObj, initialLsid, initialTxnNumber); initialRes = assert.commandWorked(mongosTestDB.runCommand(cmdObj)); @@ -202,7 +202,7 @@ function RetryableInternalTransactionTest(collectionOptions = {}) { // different txnUUID) to simulate a retry from a different mongos. const retryLsid = Object.assign({}, initialLsid, {txnUUID: UUID()}); let retryTxnNumber = 0; - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { retryTxnNumber++; setTxnFields(cmdObj, retryLsid, retryTxnNumber); const retryRes = assert.commandWorked(mongosTestDB.runCommand(cmdObj)); @@ -271,7 +271,7 @@ function RetryableInternalTransactionTest(collectionOptions = {}) { const initialLsid = txnOptions.makeSessionIdFunc(); let initialTxnNumber = 0; let initialRes; - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { initialTxnNumber++; setTxnFields(cmdObjToRetry, initialLsid, initialTxnNumber); insertCmdObjs.forEach(cmdObj => setTxnFields(cmdObj, initialLsid, initialTxnNumber)); @@ -314,7 +314,7 @@ function RetryableInternalTransactionTest(collectionOptions = {}) { // different txnUUID) to simulate a retry from a different mongos. const retryLsid = Object.assign({}, initialLsid, {txnUUID: UUID()}); let retryTxnNumber = 0; - runTxnRetryOnTransientError(() => { + runTxnRetryOnLockTimeoutError(() => { retryTxnNumber++; setTxnFields(cmdObjToRetry, retryLsid, retryTxnNumber); insertCmdObjs.forEach(cmdObj => setTxnFields(cmdObj, retryLsid, retryTxnNumber)); diff --git a/jstests/sharding/internal_txns/retryable_findAndModify_commit_and_abort_prepared_txns_after_failover_and_restart.js b/jstests/sharding/internal_txns/retryable_findAndModify_commit_and_abort_prepared_txns_after_failover_and_restart.js index 49d90c683b0..27cf8aef667 100644 --- a/jstests/sharding/internal_txns/retryable_findAndModify_commit_and_abort_prepared_txns_after_failover_and_restart.js +++ b/jstests/sharding/internal_txns/retryable_findAndModify_commit_and_abort_prepared_txns_after_failover_and_restart.js @@ -184,15 +184,15 @@ function runTest(st, stepDownShard0PrimaryFunc, testOpts = { } { - jsTest.log("Test when a participant shard restarts"); - const st = new ShardingTest({shards: 1, rs: {nodes: 2}}); + jsTest.log("Test when the old primary restarts"); + const st = new ShardingTest({shards: 1, rs: {nodes: 1}}); const restartShard0Func = () => { st.rs0.stopSet(null /* signal */, true /*forRestart */); st.rs0.startSet({restart: true}); st.rs0.getPrimary(); - // Wait for replication to recover the lastCommittedOpTime since it is illegal to run - // commitTransaction before the prepare oplog entry has been majority committed. - st.rs0.awaitLastOpCommitted(); + // Wait for replication since it is illegal to run commitTransaction before the prepare + // oplog entry has been majority committed. + st.rs0.awaitReplication(); }; // Test findAnModify without pre/post image. diff --git a/jstests/sharding/internal_txns/session_pooling.js b/jstests/sharding/internal_txns/session_pooling.js index e34597e54c4..38d3a165776 100644 --- a/jstests/sharding/internal_txns/session_pooling.js +++ b/jstests/sharding/internal_txns/session_pooling.js @@ -17,22 +17,14 @@ assert.commandWorked(mongosTestColl.insert({x: 1})); // Set up the collection. const sessionsColl = st.s.getCollection("config.system.sessions"); const transactionsCollOnShard = shard0Primary.getCollection("config.transactions"); -function assertNumEntries( - {sessionUUID, numSessionsCollEntries, numTransactionsCollEntries, allowedDelta}) { +function assertNumEntries({sessionUUID, numSessionsCollEntries, numTransactionsCollEntries}) { const filter = {"_id.id": sessionUUID}; - - // Assert the number of entries is within a range because sometimes the session created by an - // internal transaction is returned after the next internal transaction is created resulting in - // more sessions than expected. - const actualNumSessionsCollEntries = sessionsColl.find(filter).itcount(); - assert(actualNumSessionsCollEntries <= (numSessionsCollEntries + allowedDelta) && - actualNumSessionsCollEntries >= numSessionsCollEntries, - tojson(sessionsColl.find().toArray())); - - const actualNumTransactionsCollEntries = transactionsCollOnShard.find(filter).itcount(); - assert(actualNumTransactionsCollEntries <= (numTransactionsCollEntries + allowedDelta) && - actualNumTransactionsCollEntries >= numTransactionsCollEntries, - tojson(transactionsCollOnShard.find().toArray())); + assert.eq(numSessionsCollEntries, + sessionsColl.find(filter).itcount(), + tojson(sessionsColl.find().toArray())); + assert.eq(numTransactionsCollEntries, + transactionsCollOnShard.find(filter).itcount(), + tojson(transactionsCollOnShard.find().toArray())); } function runInternalTxn(conn, lsid) { @@ -52,12 +44,8 @@ function runTest(conn) { assert.commandWorked(conn.adminCommand({refreshLogicalSessionCacheNow: 1})); assert.commandWorked(shard0Primary.adminCommand({refreshLogicalSessionCacheNow: 1})); - assertNumEntries({ - sessionUUID: parentLsid.id, - numSessionsCollEntries: 1, - numTransactionsCollEntries: 1, - allowedDelta: 0, - }); + assertNumEntries( + {sessionUUID: parentLsid.id, numSessionsCollEntries: 1, numTransactionsCollEntries: 1}); // Run more transactions and verify the number of sessions used doesn't change. @@ -67,12 +55,8 @@ function runTest(conn) { assert.commandWorked(conn.adminCommand({refreshLogicalSessionCacheNow: 1})); assert.commandWorked(shard0Primary.adminCommand({refreshLogicalSessionCacheNow: 1})); - assertNumEntries({ - sessionUUID: parentLsid.id, - numSessionsCollEntries: 1, - numTransactionsCollEntries: 1, - allowedDelta: 1 - }); + assertNumEntries( + {sessionUUID: parentLsid.id, numSessionsCollEntries: 1, numTransactionsCollEntries: 1}); // Verify other sessions can be pooled concurrently. @@ -91,23 +75,17 @@ function runTest(conn) { assert.commandWorked(conn.adminCommand({refreshLogicalSessionCacheNow: 1})); assert.commandWorked(shard0Primary.adminCommand({refreshLogicalSessionCacheNow: 1})); - assertNumEntries({ - sessionUUID: parentLsid.id, - numSessionsCollEntries: 1, - numTransactionsCollEntries: 1, - allowedDelta: 1 - }); + assertNumEntries( + {sessionUUID: parentLsid.id, numSessionsCollEntries: 1, numTransactionsCollEntries: 1}); assertNumEntries({ sessionUUID: otherParentLsid1.id, numSessionsCollEntries: 1, - numTransactionsCollEntries: 1, - allowedDelta: 1, + numTransactionsCollEntries: 1 }); assertNumEntries({ sessionUUID: otherParentLsid2.id, numSessionsCollEntries: 1, - numTransactionsCollEntries: 1, - allowedDelta: 1, + numTransactionsCollEntries: 1 }); } diff --git a/jstests/sharding/jumbo1.js b/jstests/sharding/jumbo1.js new file mode 100644 index 00000000000..fa5d13b9f2a --- /dev/null +++ b/jstests/sharding/jumbo1.js @@ -0,0 +1,46 @@ +(function() { +'use strict'; + +load("jstests/sharding/libs/find_chunks_util.js"); + +var s = new ShardingTest({shards: 2, other: {chunkSize: 1}}); + +assert.commandWorked(s.s.adminCommand({enablesharding: "test", primaryShard: s.shard1.shardName})); +assert.commandWorked( + s.s.adminCommand({addShardToZone: s.shard0.shardName, zone: 'finalDestination'})); + +// Set the chunk range with a zone that will cause the chunk to be in the wrong place so the +// balancer will be forced to attempt to move it out. +assert.commandWorked(s.s.adminCommand({shardcollection: "test.foo", key: {x: 1}})); +assert.commandWorked(s.s.adminCommand( + {updateZoneKeyRange: 'test.foo', min: {x: 0}, max: {x: MaxKey}, zone: 'finalDestination'})); + +var db = s.getDB("test"); + +const big = 'X'.repeat(1024 * 1024); // 1MB + +// Insert 3MB of documents to create a jumbo chunk, and use the same shard key in all of +// them so that the chunk cannot be split. +var bulk = db.foo.initializeUnorderedBulkOp(); +for (var i = 0; i < 3; i++) { + bulk.insert({x: 0, big: big}); +} + +assert.commandWorked(bulk.execute()); + +s.startBalancer(); + +// Wait for the balancer to try to move the chunk and mark it as jumbo. +assert.soon(() => { + let chunk = findChunksUtil.findOneChunkByNs(s.getDB('config'), 'test.foo', {min: {x: 0}}); + if (chunk == null) { + // Balancer hasn't run and enforce the zone boundaries yet. + return false; + } + + assert.eq(s.shard1.shardName, chunk.shard, `${tojson(chunk)} was moved by the balancer`); + return chunk.jumbo; +}); + +s.stop(); +})(); diff --git a/jstests/sharding/jumbo_chunks.js b/jstests/sharding/jumbo_chunks.js deleted file mode 100644 index 4e03817b68e..00000000000 --- a/jstests/sharding/jumbo_chunks.js +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Test jumbo chunks - * - * @tags: [ - * # Command `configureCollectionBalancing` was added in v5.3 - * requires_fcv_53, - * ] - */ - -(function() { -'use strict'; - -load("jstests/sharding/libs/find_chunks_util.js"); - -function bulkInsert(coll, keyValue, sizeMBytes) { - const big = 'X'.repeat(1024 * 1024); // 1MB - var bulk = coll.initializeUnorderedBulkOp(); - for (var i = 0; i < sizeMBytes; i++) { - bulk.insert({x: keyValue, big: big}); - } - assert.commandWorked(bulk.execute()); -} - -function assertNumJumboChunks(configDB, ns, expectedNumJumboChunks) { - assert.eq(findChunksUtil.countChunksForNs(configDB, ns, {jumbo: true}), expectedNumJumboChunks); -} - -function setGlobalChunkSize(st, chunkSizeMBytes) { - // Set global chunk size - assert.commandWorked( - st.s.getDB("config").settings.update({_id: 'chunksize'}, - {$set: {value: chunkSizeMBytes}}, - {upsert: true, writeConcern: {w: 'majority'}})); -} - -function setCollectionChunkSize(st, ns, chunkSizeMBytes) { - assert.commandWorked( - st.s.adminCommand({configureCollectionBalancing: ns, chunkSize: chunkSizeMBytes})); -} - -// Test setup -var st = new ShardingTest({shards: 2, other: {chunkSize: 1}}); - -assert.commandWorked( - st.s.adminCommand({enablesharding: "test", primaryShard: st.shard1.shardName})); -assert.commandWorked(st.s.adminCommand({addShardToZone: st.shard0.shardName, zone: 'zoneShard0'})); - -// Try to move unsuccessfully a 3MB chunk and check it gets marked as jumbo -{ - // Set the chunk range with a zone that will cause the chunk to be in the wrong place so the - // balancer will be forced to attempt to move it out. - assert.commandWorked(st.s.adminCommand({shardcollection: "test.foo", key: {x: 1}})); - assert.commandWorked(st.s.adminCommand( - {updateZoneKeyRange: 'test.foo', min: {x: 0}, max: {x: MaxKey}, zone: 'zoneShard0'})); - - var db = st.getDB("test"); - - const big = 'X'.repeat(1024 * 1024); // 1MB - - // Insert 3MB of documents to create a jumbo chunk, and use the same shard key in all of - // them so that the chunk cannot be split. - var bulk = db.foo.initializeUnorderedBulkOp(); - for (var i = 0; i < 3; i++) { - bulk.insert({x: 0, big: big}); - } - - assert.commandWorked(bulk.execute()); - - st.startBalancer(); - - // Wait for the balancer to try to move the chunk and check it gets marked as jumbo. - assert.soon(() => { - let chunk = findChunksUtil.findOneChunkByNs(st.getDB('config'), 'test.foo', {min: {x: 0}}); - if (chunk == null) { - // Balancer hasn't run and enforce the zone boundaries yet. - return false; - } - - assert.eq(st.shard1.shardName, chunk.shard, `${tojson(chunk)} was moved by the balancer`); - return chunk.jumbo; - }); - - st.stopBalancer(); -} - -// Move successfully a 3MB chunk -// Collection chunkSize must prevail over global chunkSize setting -// global chunkSize -> 1MB -// collection chunkSize -> 5MB -{ - const collName = "collA"; - const coll = st.s.getDB("test").getCollection(collName); - const configDB = st.s.getDB("config"); - const splitPoint = 0; - - assert.commandWorked(st.s.adminCommand({shardcollection: coll.getFullName(), key: {x: 1}})); - assert.commandWorked(st.s.adminCommand({ - updateZoneKeyRange: coll.getFullName(), - min: {x: splitPoint}, - max: {x: MaxKey}, - zone: 'zoneShard0' - })); - - bulkInsert(coll, splitPoint, 3); - - setCollectionChunkSize(st, coll.getFullName(), 5); - setGlobalChunkSize(st, 1); - - // Move the 3MB chunk to shard0 - st.startBalancer(); - st.awaitCollectionBalance(coll); - st.stopBalancer(); - - const chunk = - findChunksUtil.findOneChunkByNs(configDB, coll.getFullName(), {min: {x: splitPoint}}); - - // Verify chunk has been moved to shard0 - assert.eq(st.shard0.shardName, - chunk.shard, - `${tojson(chunk)} was not moved to ${tojson(st.shard0.shardName)}`); - assertNumJumboChunks(configDB, coll.getFullName(), 0); - - coll.drop(); -} - -// Try to move unsuccessfully a 3MB chunk and mark it as jumbo -// Collection chunkSize must prevail over global chunkSize setting -// global chunkSize -> 5MB -// collection chunkSize -> 1MB -{ - const collName = "collB"; - const coll = st.s.getDB("test").getCollection(collName); - const configDB = st.s.getDB("config"); - const splitPoint = 0; - - assert.commandWorked(st.s.adminCommand({shardcollection: coll.getFullName(), key: {x: 1}})); - assert.commandWorked(st.s.adminCommand({ - updateZoneKeyRange: coll.getFullName(), - min: {x: splitPoint}, - max: {x: MaxKey}, - zone: 'zoneShard0' - })); - - bulkInsert(coll, splitPoint, 3); - - setCollectionChunkSize(st, coll.getFullName(), 1); - setGlobalChunkSize(st, 5); - - // Try to move the 3MB chunk and mark it as jumbo - st.startBalancer(); - - assert.soon(() => { - const chunk = - findChunksUtil.findOneChunkByNs(configDB, coll.getFullName(), {min: {x: splitPoint}}); - if (chunk == null) { - // Balancer hasn't run and enforce the zone boundaries yet. - return false; - } - - return chunk.jumbo; - }); - - st.stopBalancer(); - - const chunk = - findChunksUtil.findOneChunkByNs(configDB, coll.getFullName(), {min: {x: splitPoint}}); - - // Verify chunk hasn't been moved to shard0 and it's jumbo - assert.eq(st.shard1.shardName, - chunk.shard, - `${tojson(chunk)} was moved to ${tojson(st.shard0.shardName)}`); - assertNumJumboChunks(configDB, coll.getFullName(), 1); - - coll.drop(); -} - -st.stop(); -})(); diff --git a/jstests/sharding/libs/defragmentation_util.js b/jstests/sharding/libs/defragmentation_util.js index 943f2eb462d..52ac1333ef7 100644 --- a/jstests/sharding/libs/defragmentation_util.js +++ b/jstests/sharding/libs/defragmentation_util.js @@ -21,9 +21,7 @@ var defragmentationUtil = (function() { } createAndDistributeChunks(mongos, ns, numChunks, chunkSpacing); - // Created zones will line up exactly with existing chunks so as not to trigger zone - // violations in the balancer. - createRandomZones(mongos, ns, numZones); + createRandomZones(mongos, ns, numZones, chunkSpacing); fillChunksToRandomSize(mongos, ns, docSizeBytes, maxChunkFillMB); const beginningNumberChunks = findChunksUtil.countChunksForNs(mongos.getDB('config'), ns); @@ -51,18 +49,19 @@ var defragmentationUtil = (function() { } }; - let createRandomZones = function(mongos, ns, numZones) { - let existingChunks = findChunksUtil.findChunksByNs(mongos.getDB('config'), ns); - existingChunks = Array.shuffle(existingChunks.toArray()); - for (let i = 0; i < numZones; i++) { + let createRandomZones = function(mongos, ns, numZones, chunkSpacing) { + for (let i = -Math.floor(numZones / 2); i < Math.ceil(numZones / 2); i++) { let zoneName = "Zone" + i; - let shardForZone = existingChunks[i].shard; + let shardForZone = + findChunksUtil + .findOneChunkByNs(mongos.getDB('config'), ns, {min: {key: i * chunkSpacing}}) + .shard; assert.commandWorked( mongos.adminCommand({addShardToZone: shardForZone, zone: zoneName})); assert.commandWorked(mongos.adminCommand({ updateZoneKeyRange: ns, - min: existingChunks[i].min, - max: existingChunks[i].max, + min: {key: i * chunkSpacing}, + max: {key: i * chunkSpacing + chunkSpacing}, zone: zoneName })); } @@ -203,13 +202,6 @@ var defragmentationUtil = (function() { assert.soon(function() { let balancerStatus = assert.commandWorked(mongos.adminCommand({balancerCollectionStatus: ns})); - - if (balancerStatus.balancerCompliant) { - // As we can't rely on `balancerCompliant` due to orphan counter non atomic update, - // we need to ensure the collection is balanced by some extra checks - sh.awaitCollectionBalance(mongos.getCollection(ns)); - } - return balancerStatus.balancerCompliant || balancerStatus.firstComplianceViolation !== 'defragmentingChunks'; }); diff --git a/jstests/sharding/libs/last_lts_mongod_commands.js b/jstests/sharding/libs/last_lts_mongod_commands.js index f8c0802af04..c25af3a8275 100644 --- a/jstests/sharding/libs/last_lts_mongod_commands.js +++ b/jstests/sharding/libs/last_lts_mongod_commands.js @@ -20,12 +20,8 @@ const commandsAddedToMongodSinceLastLTS = [ "clusterGetMore", "clusterInsert", "clusterUpdate", - "createSearchIndexes", - "dropSearchIndex", "getClusterParameter", - "listSearchIndexes", "rotateCertificates", "setClusterParameter", "setUserWriteBlockMode", - "updateSearchIndex", ]; diff --git a/jstests/sharding/libs/last_lts_mongos_commands.js b/jstests/sharding/libs/last_lts_mongos_commands.js index 6acf8dba820..f7a72743e6b 100644 --- a/jstests/sharding/libs/last_lts_mongos_commands.js +++ b/jstests/sharding/libs/last_lts_mongos_commands.js @@ -18,17 +18,12 @@ const commandsAddedToMongosSinceLastLTS = [ "commitReshardCollection", "compactStructuredEncryptionData", "configureCollectionBalancing", - "createSearchIndexes", - "dropSearchIndex", - "fsyncUnlock", "getClusterParameter", - "listSearchIndexes", "moveRange", "reshardCollection", "rotateCertificates", "setAllowMigrations", "setClusterParameter", - "setProfilingFilterGlobally", // TODO SERVER-73305 "setUserWriteBlockMode", "testDeprecation", "testDeprecationInVersion2", @@ -36,5 +31,4 @@ const commandsAddedToMongosSinceLastLTS = [ "testRemoval", "testVersions1And2", "testVersion2", - "updateSearchIndex", ]; diff --git a/jstests/sharding/libs/mongos_api_params_util.js b/jstests/sharding/libs/mongos_api_params_util.js index 7de6a36dcda..e1ca18df3e6 100644 --- a/jstests/sharding/libs/mongos_api_params_util.js +++ b/jstests/sharding/libs/mongos_api_params_util.js @@ -75,7 +75,7 @@ let MongosAPIParametersUtil = (function() { function awaitRemoveShard(shardName) { assert.commandWorked(st.startBalancer()); - st.awaitBalancerRound(); + st.waitForBalancer(true, 60000); assert.soon(() => { const res = st.s.adminCommand({removeShard: shardName}); jsTestLog(`removeShard result: ${tojson(res)}`); @@ -789,11 +789,7 @@ let MongosAPIParametersUtil = (function() { jsTestLog(`Waiting for "find" on "${st.rs0.name}" ` + `with comment ${uuidStr} in currentOp`); assert.soon(() => { - const filter = { - "command.find": "collection", - "command.comment": uuidStr, - shard: st.rs0.name - }; + const filter = {"command.comment": uuidStr, shard: st.rs0.name}; const inprog = adminDb.currentOp(filter).inprog; if (inprog.length === 1) { jsTestLog(`Found it! findOpId ${inprog[0].opid}`); @@ -1219,8 +1215,9 @@ let MongosAPIParametersUtil = (function() { } }, { - commandName: "setProfilingFilterGlobally", - skip: "executes locally on mongos (not sent to any remote node)", + commandName: "setFreeMonitoring", + skip: "explicitly fails for mongos, primary mongod only", + conditional: true }, { commandName: "setParameter", @@ -1412,7 +1409,7 @@ let MongosAPIParametersUtil = (function() { }; }); - const st = new ShardingTest({mongos: 1, shards: 2, config: 1, rs: {nodes: 1}}); + const st = new ShardingTest({mongos: 1, shards: 2, rs: {nodes: 1}}); const listCommandsRes = st.s0.adminCommand({listCommands: 1}); assert.commandWorked(listCommandsRes); diff --git a/jstests/sharding/libs/remove_shard_util.js b/jstests/sharding/libs/remove_shard_util.js deleted file mode 100644 index 4de2c17a461..00000000000 --- a/jstests/sharding/libs/remove_shard_util.js +++ /dev/null @@ -1,34 +0,0 @@ -function removeShard(shardingTestOrConn, shardName, timeout) { - if (timeout == undefined) { - timeout = 10 * 60 * 1000; // 10 minutes - } - - var s; - if (shardingTestOrConn instanceof ShardingTest) { - s = shardingTestOrConn.s; - } else { - s = shardingTestOrConn; - } - - assert.soon(function() { - let res; - if (TestData.configShard && shardName == "config") { - // Need to use transitionToDedicatedConfigServer if trying - // to remove config server as a shard - res = s.adminCommand({transitionToDedicatedConfigServer: shardName}); - } else { - res = s.adminCommand({removeShard: shardName}); - } - if (!res.ok && res.code === ErrorCodes.ShardNotFound) { - // If the config server primary steps down right after removing the config.shards doc - // for the shard but before responding with "state": "completed", the mongos would retry - // the _configsvrRemoveShard command against the new config server primary, which would - // not find the removed shard in its ShardRegistry if it has done a ShardRegistry reload - // after the config.shards doc for the shard was removed. This would cause the command - // to fail with ShardNotFound. - return true; - } - assert.commandWorked(res); - return res.state == 'completed'; - }, "failed to remove shard " + shardName + " within " + timeout + "ms", timeout); -} diff --git a/jstests/sharding/libs/resharding_test_fixture.js b/jstests/sharding/libs/resharding_test_fixture.js index cd658ae9508..13d39674f0f 100644 --- a/jstests/sharding/libs/resharding_test_fixture.js +++ b/jstests/sharding/libs/resharding_test_fixture.js @@ -85,11 +85,11 @@ var ReshardingTest = class { /** @private */ this._newShardKey = undefined; /** @private */ - this._pauseCoordinatorBeforeBlockingWritesFailpoints = []; + this._pauseCoordinatorBeforeBlockingWrites = undefined; /** @private */ - this._pauseCoordinatorBeforeDecisionPersistedFailpoints = []; + this._pauseCoordinatorBeforeDecisionPersistedFailpoint = undefined; /** @private */ - this._pauseCoordinatorBeforeCompletionFailpoints = []; + this._pauseCoordinatorBeforeCompletionFailpoint = undefined; /** @private */ this._reshardingThread = undefined; /** @private */ @@ -283,11 +283,6 @@ var ReshardingTest = class { return sourceCollection; } - get tempNs() { - assert.neq(undefined, this._tempNs, "createShardedCollection must be called first"); - return this._tempNs; - } - /** * Reshards an existing collection using the specified new shard key and new chunk ranges. * @@ -321,19 +316,13 @@ var ReshardingTest = class { this._newShardKey = Object.assign({}, newShardKeyPattern); - this._pauseCoordinatorBeforeBlockingWritesFailpoints = []; - this._pauseCoordinatorBeforeDecisionPersistedFailpoints = []; - this._pauseCoordinatorBeforeCompletionFailpoints = []; - this._st.forEachConfigServer((configServer) => { - this._pauseCoordinatorBeforeBlockingWritesFailpoints.push( - configureFailPoint(configServer, "reshardingPauseCoordinatorBeforeBlockingWrites")); - this._pauseCoordinatorBeforeDecisionPersistedFailpoints.push(configureFailPoint( - configServer, "reshardingPauseCoordinatorBeforeDecisionPersisted")); - this._pauseCoordinatorBeforeCompletionFailpoints.push( - configureFailPoint(configServer, - "reshardingPauseCoordinatorBeforeCompletion", - {"sourceNamespace": this._ns})); - }); + const configPrimary = this._st.configRS.getPrimary(); + this._pauseCoordinatorBeforeBlockingWrites = + configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeBlockingWrites"); + this._pauseCoordinatorBeforeDecisionPersistedFailpoint = + configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeDecisionPersisted"); + this._pauseCoordinatorBeforeCompletionFailpoint = configureFailPoint( + configPrimary, "reshardingPauseCoordinatorBeforeCompletion", {}, {times: 1}); this._commandDoneSignal = new CountDownLatch(1); @@ -463,9 +452,9 @@ var ReshardingTest = class { try { fn(); } catch (duringReshardingError) { - for (const fp of [...this._pauseCoordinatorBeforeBlockingWritesFailpoints, - ...this._pauseCoordinatorBeforeDecisionPersistedFailpoints, - ...this._pauseCoordinatorBeforeCompletionFailpoints]) { + for (const fp of [this._pauseCoordinatorBeforeBlockingWrites, + this._pauseCoordinatorBeforeDecisionPersistedFailpoint, + this._pauseCoordinatorBeforeCompletionFailpoint]) { try { fp.off(); } catch (disableFailpointError) { @@ -514,9 +503,7 @@ var ReshardingTest = class { * proceeding to the next stage. This helper returns after either: * * 1) The node's waitForFailPoint returns successfully or - * 2) The `reshardCollection` command has returned a response or - * 3) The ReshardingCoordinator is blocked on the reshardingPauseCoordinatorBeforeCompletion - * failpoint and won't ever satisfy the supplied failpoint. + * 2) The `reshardCollection` command has returned a response. * * The function returns true when we returned because the server reached the failpoint. The * function returns false when the `reshardCollection` command is no longer running. @@ -525,20 +512,9 @@ var ReshardingTest = class { * @private */ _waitForFailPoint(fp) { - const completionFailpoint = this._pauseCoordinatorBeforeCompletionFailpoints.find( - completionFailpoint => completionFailpoint.conn.host === fp.conn.host); - assert.soon( () => { - if (this._commandDoneSignal.getCount() === 0 || fp.waitWithTimeout(1000)) { - return true; - } - - if (completionFailpoint !== fp && completionFailpoint.waitWithTimeout(1000)) { - completionFailpoint.off(); - } - - return false; + return this._commandDoneSignal.getCount() === 0 || fp.waitWithTimeout(1000); }, "Timed out waiting for failpoint to be hit. Failpoint: " + fp.failPointName, undefined, @@ -553,19 +529,6 @@ var ReshardingTest = class { postCheckConsistencyFn = () => {}, postDecisionPersistedFn = () => {}, afterReshardingFn = () => {}) { - // The CSRS primary may have changed as a result of running the duringReshardingFn() - // callback function. The failpoints will only be triggered on the new CSRS primary so we - // detect which node that is here. - const configPrimary = this._st.configRS.getPrimary(); - const primaryIdx = this._pauseCoordinatorBeforeBlockingWritesFailpoints.findIndex( - fp => fp.conn.host === configPrimary.host); - // The CSRS secondaries may be going through replication rollback which closes their - // connections to the test client. We wait for any replication rollbacks to complete and for - // the test client to have reconnected so the failpoints can be turned off on all of the - // nodes later on. - this._st.configRS.awaitSecondaryNodes(); - this._st.configRS.awaitReplication(); - let performCorrectnessChecks = true; if (expectedErrorCode === ErrorCodes.OK) { this._callFunctionSafely(() => { @@ -576,18 +539,17 @@ var ReshardingTest = class { // reshardingPauseCoordinatorBeforeDecisionPersisted failpoint to wait for all of // the recipient shards to have applied through all of the oplog entries from all of // the donor shards. - if (!this._waitForFailPoint( - this._pauseCoordinatorBeforeBlockingWritesFailpoints[primaryIdx])) { + if (!this._waitForFailPoint(this._pauseCoordinatorBeforeBlockingWrites)) { performCorrectnessChecks = false; } - this._pauseCoordinatorBeforeBlockingWritesFailpoints.forEach(fp => fp.off()); + this._pauseCoordinatorBeforeBlockingWrites.off(); // A resharding command that returned a failure will not hit the "Decision // Persisted" failpoint. If the command has returned, don't require that the // failpoint was entered. This ensures that following up by joining the // `_reshardingThread` will succeed. if (!this._waitForFailPoint( - this._pauseCoordinatorBeforeDecisionPersistedFailpoints[primaryIdx])) { + this._pauseCoordinatorBeforeDecisionPersistedFailpoint)) { performCorrectnessChecks = false; } @@ -600,21 +562,22 @@ var ReshardingTest = class { postCheckConsistencyFn(); } - this._pauseCoordinatorBeforeDecisionPersistedFailpoints.forEach(fp => fp.off()); + this._pauseCoordinatorBeforeDecisionPersistedFailpoint.off(); postDecisionPersistedFn(); - this._pauseCoordinatorBeforeCompletionFailpoints.forEach(fp => fp.off()); + this._pauseCoordinatorBeforeCompletionFailpoint.off(); }); } else { this._callFunctionSafely(() => { - this._pauseCoordinatorBeforeBlockingWritesFailpoints.forEach( - fp => this.retryOnceOnNetworkError(fp.off)); + this.retryOnceOnNetworkError( // + () => this._pauseCoordinatorBeforeBlockingWrites.off()); + postCheckConsistencyFn(); - this._pauseCoordinatorBeforeDecisionPersistedFailpoints.forEach( - fp => this.retryOnceOnNetworkError(fp.off)); + this.retryOnceOnNetworkError( + () => this._pauseCoordinatorBeforeDecisionPersistedFailpoint.off()); postDecisionPersistedFn(); - this._pauseCoordinatorBeforeCompletionFailpoints.forEach( - fp => this.retryOnceOnNetworkError(fp.off)); + this.retryOnceOnNetworkError( + () => this._pauseCoordinatorBeforeCompletionFailpoint.off()); }); } @@ -637,11 +600,7 @@ var ReshardingTest = class { /** @private */ _checkConsistency() { - // The "available" read concern level won't block this find cmd behind the critical section. - // Tests for resharding are not expected to have unowned documents in the collection being - // resharded. - const nsCursor = - this._st.s.getCollection(this._ns).find().readConcern("available").sort({_id: 1}); + const nsCursor = this._st.s.getCollection(this._ns).find().sort({_id: 1}); const tempNsCursor = this._st.s.getCollection(this._tempNs).find().sort({_id: 1}); const diff = ((diff) => { @@ -659,8 +618,7 @@ var ReshardingTest = class { docsExtraAfterResharding: [], docsMissingAfterResharding: [], }, - "existing sharded collection " + this._ns + - " and temporary resharding collection " + this._tempNs + " had different" + + "existing sharded collection and temporary resharding collection had different" + " contents"); } diff --git a/jstests/sharding/libs/with_partial_shard_key_util.js b/jstests/sharding/libs/with_partial_shard_key_util.js deleted file mode 100644 index bcfeae08926..00000000000 --- a/jstests/sharding/libs/with_partial_shard_key_util.js +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Utilities for the find_and_modify_with_partial_shard_key.js and delete_with_partial_shard_key.js - * tests. - */ -"use strict"; - -/** - * Runs an explain on the `cmdObj` and checks that the explain targets the shard given be - * `expectedShardName`. - */ -function assertExplainTargetsCorrectShard(db, cmdObj, expectedShardName) { - var res = db.runCommand({explain: cmdObj}); - assert.eq(res.queryPlanner.winningPlan.shards.length, 1); - assert.eq(res.queryPlanner.winningPlan.shards[0].shardName, expectedShardName); -} - -/** - * Performs a split given by the `splitDoc`, and then moves the chunk containg `moveShard0Doc` to - * shard0 and the chunk containing `moveShard1Doc` to shard1. - */ -function splitAndMoveChunks(st, splitDoc, moveShard0Doc, moveShard1Doc) { - assert.commandWorked(st.s0.adminCommand({split: "test.sharded_coll", middle: splitDoc})); - - assert.commandWorked(st.s0.adminCommand( - {moveChunk: "test.sharded_coll", find: moveShard0Doc, to: st.shard0.shardName})); - assert.commandWorked(st.s0.adminCommand( - {moveChunk: "test.sharded_coll", find: moveShard1Doc, to: st.shard1.shardName})); -} diff --git a/jstests/sharding/linearizable_read_concern.js b/jstests/sharding/linearizable_read_concern.js index 23541c0ba63..a5720681aeb 100644 --- a/jstests/sharding/linearizable_read_concern.js +++ b/jstests/sharding/linearizable_read_concern.js @@ -27,6 +27,11 @@ load("jstests/libs/write_concern_util.js"); (function() { "use strict"; +// Skip db hash check and shard replication since this test leaves a replica set shard +// partitioned. +TestData.skipCheckDBHashes = true; +TestData.skipAwaitingReplicationOnShardsBeforeCheckingUUIDs = true; + var testName = "linearizable_read_concern"; var st = new ShardingTest({ @@ -119,9 +124,5 @@ var result = testDB.runReadCommand({ }); assert.commandFailedWithCode(result, ErrorCodes.MaxTimeMSExpired); -// Reconnect to allow potential write operations triggered by consistency checks. -secondaries[0].reconnect(primary); -secondaries[1].reconnect(primary); - st.stop(); })(); diff --git a/jstests/sharding/log_remote_op_wait.js b/jstests/sharding/log_remote_op_wait.js index c514b211261..78241bf53c9 100644 --- a/jstests/sharding/log_remote_op_wait.js +++ b/jstests/sharding/log_remote_op_wait.js @@ -130,6 +130,30 @@ assert(!csCursor.hasNext()); assert.gte(remoteOpWait, 900); } +// An equivalent .find() does not include remoteOpWaitMillis. +const findComment = 'example_find_should_not_have_remote_op_wait'; +coll.find().sort({x: 1}).comment(findComment).next(); +{ + const mongosLog = assert.commandWorked(st.s.adminCommand({getLog: "global"})); + const lines = + [...findMatchingLogLines(mongosLog.log, {msg: "Slow query", comment: findComment})]; + const line = lines.find(line => line.match(/command.{1,4}find/)); + assert(line, "Failed to find a 'find' log line matching the comment"); + assert(!line.match(/remoteOpWait/), `Log line unexpectedly contained remoteOpWait: ${line}`); +} + +// Other cursor-producing commands do not include remoteOpWaitMillis, such as listCollections. +const listCollectionsComment = 'example_listCollections_should_not_have_remote_op_wait'; +coll.runCommand({listCollections: 1, comment: listCollectionsComment}); +{ + const mongosLog = assert.commandWorked(st.s.adminCommand({getLog: "global"})); + const lines = [...findMatchingLogLines(mongosLog.log, + {msg: "Slow query", comment: listCollectionsComment})]; + const line = lines.find(line => line.match(/command.{1,4}listCollections/)); + assert(line, "Failed to find a 'listCollections' log line matching the comment"); + assert(!line.match(/remoteOpWait/), `Log line unexpectedly contained remoteOpWait: ${line}`); +} + // A query that merges on a shard logs remoteOpWaitMillis on the shard. const pipeline2 = [{$sort: {x: 1}}, {$group: {_id: "$y"}}]; const pipelineComment2 = 'example_pipeline2_should_have_remote_op_wait'; diff --git a/jstests/sharding/log_remote_op_wait_for_other_commands.js b/jstests/sharding/log_remote_op_wait_for_other_commands.js deleted file mode 100644 index a8793d790ce..00000000000 --- a/jstests/sharding/log_remote_op_wait_for_other_commands.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Tests that command log lines which execute remote operations include remoteOpWaitMillis: the - * amount of time the merger spent waiting for results from shards. - * - * @tags: [ - * # The 'remoteOpWaitMillis' was added to explain output for commands other than $mergeCursor - * # aggregation stage in 6.0. - * requires_fcv_60 - * ] - */ -(function() { - -load("jstests/libs/log.js"); // For findMatchingLogLine. - -const st = new ShardingTest({shards: 2, rs: {nodes: 1}}); -st.stopBalancer(); - -const dbName = st.s.defaultDB; -const coll = st.s.getDB(dbName).getCollection('profile_remote_op_wait'); - -coll.drop(); -assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); - -// Shards the test collection and splits it into two chunks: one that contains all {shard: 1} -// documents and one that contains all {shard: 2} documents. -st.shardColl(coll.getName(), - {shard: 1} /* shard key */, - {shard: 2} /* split at */, - {shard: 2} /* move the chunk containing {shard: 2} to its own shard */, - dbName, - true); - -assert.commandWorked( - coll.insert(Array.from({length: 100}, (_, i) => ({_id: i, shard: (i % 2) + 1, x: i})))); - -// Sets the slow query logging threshold (slowMS) to -1 to ensure every query gets logged. -st.s.getDB('admin').setProfilingLevel(0, -1); - -function getRemoteOpWait(logLine) { - const pattern = /remoteOpWaitMillis"?:([0-9]+)/; - const match = logLine.match(pattern); - assert(match, `pattern ${pattern} did not match line: ${logLine}`); - const millis = parseInt(match[1]); - assert.gte(millis, 0, match); - return millis; -} - -function getDuration(logLine) { - const pattern = /durationMillis"?:([0-9]+)/; - const match = logLine.match(pattern); - assert(match, `pattern ${pattern} did not match line: ${logLine}`); - const millis = parseInt(match[1]); - assert.gte(millis, 0, match); - return millis; -} - -// An .find() includes remoteOpWaitMillis. -const findComment = 'example_find_should_have_remote_op_wait_too'; -coll.find().sort({x: 1}).comment(findComment).next(); -{ - const mongosLog = assert.commandWorked(st.s.adminCommand({getLog: "global"})); - const lines = - [...findMatchingLogLines(mongosLog.log, {msg: "Slow query", comment: findComment})]; - const line = lines.find(line => line.match(/command.{1,4}find/)); - assert(line, "Failed to find a 'find' log line matching the comment"); - assert(line.match(/remoteOpWait/), `Log line does not contain remoteOpWait: ${line}`); - const remoteOpWait = getRemoteOpWait(line); - const duration = getDuration(line); - assert.lte(remoteOpWait, duration); -} - -// Other commands which execute remote operations also include remoteOpWaitMillis, such as -// listCollections. -const listCollectionsComment = 'example_listCollections_should_have_remote_op_wait_too'; -coll.runCommand({listCollections: 1, comment: listCollectionsComment}); -{ - const mongosLog = assert.commandWorked(st.s.adminCommand({getLog: "global"})); - const lines = [...findMatchingLogLines(mongosLog.log, - {msg: "Slow query", comment: listCollectionsComment})]; - const line = lines.find(line => line.match(/command.{1,4}listCollections/)); - assert(line, "Failed to find a 'listCollections' log line matching the comment"); - assert(line.match(/remoteOpWait/), `Log line does not contain remoteOpWait: ${line}`); - const remoteOpWait = getRemoteOpWait(line); - const duration = getDuration(line); - assert.lte(remoteOpWait, duration); -} - -st.stop(); -})(); diff --git a/jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js b/jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js new file mode 100644 index 00000000000..0c0d67b9554 --- /dev/null +++ b/jstests/sharding/max_time_ms_does_not_leak_shard_cursor.js @@ -0,0 +1,75 @@ +// Tests that if a mongoS cursor exceeds the maxTimeMs timeout, the cursors on the shards will be +// cleaned up. Exercises the fix for the bug described in SERVER-62710. +// +// @tags: [] + +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); // for 'configureFailPoint()' + +function getIdleCursors(conn, collName) { + return conn.getDB('admin') + .aggregate([ + {$currentOp: {idleCursors: true}}, + {$match: {$and: [{type: "idleCursor"}, {"cursor.originatingCommand.find": collName}]}} + ]) + .toArray(); +} + +function assertNoIdleCursors(conn, collName) { + const sleepTimeMS = 10 * 1000; + const retries = 2; + assert.soon(() => { + return getIdleCursors(conn, collName).length === 0; + }, tojson(getIdleCursors(conn, collName)), retries * sleepTimeMS, sleepTimeMS, { + runHangAnalyzer: false + }); +} + +const st = new ShardingTest({shards: 1, mongos: 1, config: 1}); + +const dbName = "test"; +const collName = jsTestName(); + +const coll = st.s.getCollection(dbName + "." + collName); + +assert.commandWorked(coll.insert(Array.from({length: 1000}, _ => ({a: 1})))); + +// Perform a query that sleeps after retrieving each document. +// This is guaranteed to exceed the specified maxTimeMS limit. +// The timeout may happen either on mongoS or on shard. +const curs = coll.find({ + $where: function() { + sleep(1); + return true; + } + }) + .batchSize(2) + .maxTimeMS(100); +assert.eq(getIdleCursors(st.shard0, collName).length, 0); +assert.throwsWithCode(() => { + curs.itcount(); +}, ErrorCodes.MaxTimeMSExpired); +assertNoIdleCursors(st.shard0, collName); + +// Ensure the timeout happens on mongoS. +const cursTestMongoS = coll.find({}).batchSize(2).maxTimeMS(100); +const fpTestMongoS = configureFailPoint(st.s, "maxTimeAlwaysTimeOut", {}, "alwaysOn"); +assert.throwsWithCode(() => { + cursTestMongoS.itcount(); +}, ErrorCodes.MaxTimeMSExpired); +fpTestMongoS.off(); +assertNoIdleCursors(st.shard0, collName); + +// Ensure the timeout happens on the shard. +const cursTestShard0 = coll.find({}).batchSize(2).maxTimeMS(100); +const fpTestShard0 = configureFailPoint(st.shard0, "maxTimeAlwaysTimeOut", {}, "alwaysOn"); +assert.throwsWithCode(() => { + cursTestShard0.itcount(); +}, ErrorCodes.MaxTimeMSExpired); +fpTestShard0.off(); +assertNoIdleCursors(st.shard0, collName); + +st.stop(); +})(); diff --git a/jstests/sharding/merge_let_params_size_estimation.js b/jstests/sharding/merge_let_params_size_estimation.js deleted file mode 100644 index 66f30d38335..00000000000 --- a/jstests/sharding/merge_let_params_size_estimation.js +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Test which verifies that $merge accounts for the size of let parameters and runtime constants - * when it serializes writes to send to other nodes. - * - * @tags: [ - * # The $merge in this test targets the '_id' field, and requires a unique index. - * expects_explicit_underscore_id_index, - * ] - */ -(function() { -"use strict"; - -load('jstests/libs/fixture_helpers.js'); // For isReplSet(). - -// Function to run the test against a test fixture. Accepts an object that contains the following -// fields: -// - testFixture: The fixture to run the test against. -// - conn: Connection to the test fixture specified above. -// - shardLocal and shardOutput: Indicates whether the local/output collection should be sharded in -// this test run (ignored when not running against a sharded cluster). -function runTest({testFixture, conn, shardLocal, shardOutput}) { - const dbName = "db"; - const collName = "merge_let_params"; - const dbCollName = dbName + "." + collName; - const outCollName = "outcoll"; - const dbOutCollName = dbName + "." + outCollName; - const admin = conn.getDB("admin"); - const isReplSet = FixtureHelpers.isReplSet(admin); - - function shardColls() { - // When running against a sharded cluster, configure the collections according to - // 'shardLocal' and 'shardOutput'. - if (!isReplSet) { - assert.commandWorked(admin.runCommand({enableSharding: dbName})); - testFixture.ensurePrimaryShard(dbName, testFixture.shard0.shardName); - if (shardLocal) { - testFixture.shardColl(collName, {_id: 1}, {_id: 0}, {_id: 0}, dbName); - } - if (shardOutput) { - testFixture.shardColl(outCollName, {_id: 1}, {_id: 0}, {_id: 0}, dbName); - } - } - } - const coll = conn.getCollection(dbCollName); - const outColl = conn.getCollection(dbOutCollName); - coll.drop(); - outColl.drop(); - shardColls(); - - // Insert two large documents in both collections. By inserting the documents with the same _id - // values in both collections and splitting these documents between chunks, this will guarantee - // that we need to serialize and send update command(s) across the wire when targeting the - // output collection. - const kOneMB = 1024 * 1024; - const kDataString = "a".repeat(4 * kOneMB); - const kDocs = [{_id: 2, data: kDataString}, {_id: -2, data: kDataString}]; - assert.commandWorked(coll.insertMany(kDocs)); - assert.commandWorked(outColl.insertMany(kDocs)); - - // The sizes of the different update command components are deliberately chosen to test the - // batching logic when the update is targeted to another node in the cluster. In particular, the - // update command will contain the 10MB 'outFieldValue' and we will be updating two 4MB - // documents. The 18MB total exceeds the 16MB size limit, so we expect the batching logic to - // split the two documents into separate batches of 14MB each. - const outFieldValue = "a".repeat(10 * kOneMB); - let aggCommand = { - pipeline: [{ - $merge: { - into: {db: "db", coll: outCollName}, - on: "_id", - whenMatched: [{$addFields: {out: "$$outField"}}], - whenNotMatched: "insert" - } - }], - cursor: {}, - let : {"outField": outFieldValue} - }; - - // If this is a replica set, we need to target a secondary node to force writes to go over - // the wire. - const aggColl = isReplSet ? testFixture.getSecondary().getCollection(dbCollName) : coll; - - if (isReplSet) { - aggCommand["$readPreference"] = {mode: "secondary"}; - } - - // The aggregate should not fail. - assert.commandWorked(aggColl.runCommand("aggregate", aggCommand)); - - // Verify that each document in the output collection contains the value of 'outField'. - let outContents = outColl.find().toArray(); - for (const res of outContents) { - const out = res["out"]; - assert.eq(out, outFieldValue, outContents); - } - - assert(coll.drop()); - assert(outColl.drop()); - shardColls(); - - // Insert four large documents in both collections. As before, this will force updates to be - // sent across the wire, but this will generate double the batches. - const kMoreDocs = [ - {_id: -2, data: kDataString}, - {_id: -1, data: kDataString}, - {_id: 1, data: kDataString}, - {_id: 2, data: kDataString}, - ]; - - assert.commandWorked(coll.insertMany(kMoreDocs)); - assert.commandWorked(outColl.insertMany(kMoreDocs)); - - // The aggregate should not fail. - assert.commandWorked(aggColl.runCommand("aggregate", aggCommand)); - - // Verify that each document in the output collection contains the value of 'outField'. - outContents = outColl.find().toArray(); - for (const res of outContents) { - const out = res["out"]; - assert.eq(out, outFieldValue, outContents); - } - - assert(coll.drop()); - assert(outColl.drop()); - shardColls(); - - // If the documents and the let parameters are large enough, the $merge is expected to fail. - const kVeryLargeDataString = "a".repeat(10 * kOneMB); - const kLargeDocs = - [{_id: 2, data: kVeryLargeDataString}, {_id: -2, data: kVeryLargeDataString}]; - assert.commandWorked(coll.insertMany(kLargeDocs)); - assert.commandWorked(outColl.insertMany(kLargeDocs)); - assert.commandFailedWithCode(aggColl.runCommand("aggregate", aggCommand), - ErrorCodes.BSONObjectTooLarge); -} - -// Test against a replica set. -const rst = new ReplSetTest({nodes: 2}); -rst.startSet(); -rst.initiate(); -rst.awaitSecondaryNodes(); - -runTest({testFixture: rst, conn: rst.getPrimary()}); - -rst.stopSet(); - -// Test against a sharded cluster. -const st = new ShardingTest({shards: 2, mongos: 1}); -runTest({testFixture: st, conn: st.s0, shardLocal: false, shardOutput: false}); -runTest({testFixture: st, conn: st.s0, shardLocal: true, shardOutput: false}); -runTest({testFixture: st, conn: st.s0, shardLocal: false, shardOutput: true}); -runTest({testFixture: st, conn: st.s0, shardLocal: true, shardOutput: true}); - -st.stop(); -})(); diff --git a/jstests/sharding/merge_with_drop_shard.js b/jstests/sharding/merge_with_drop_shard.js index bc0b18715ce..bc50b8fc058 100644 --- a/jstests/sharding/merge_with_drop_shard.js +++ b/jstests/sharding/merge_with_drop_shard.js @@ -29,6 +29,7 @@ function setAggHang(mode) { function removeShard(shard) { // We need the balancer to drain all the chunks out of the shard that is being removed. assert.commandWorked(st.startBalancer()); + st.waitForBalancer(true, 60000); var res = st.s.adminCommand({removeShard: shard.shardName}); assert.commandWorked(res); assert.eq('started', res.state); @@ -44,6 +45,7 @@ function removeShard(shard) { st.configRS.awaitLastOpCommitted(); assert.commandWorked(st.s.adminCommand({flushRouterConfig: 1})); assert.commandWorked(st.stopBalancer()); + st.waitForBalancer(false, 60000); } function addShard(shard) { diff --git a/jstests/sharding/migrateBig.js b/jstests/sharding/migrateBig.js index aa237832dd3..60306797dfd 100644 --- a/jstests/sharding/migrateBig.js +++ b/jstests/sharding/migrateBig.js @@ -1,10 +1,7 @@ (function() { 'use strict'; -load("jstests/libs/feature_flag_util.js"); - -var s = new ShardingTest( - {name: "migrateBig", shards: 2, other: {chunkSize: 1, enableAutoSplit: false}}); +var s = new ShardingTest({name: "migrateBig", shards: 2, other: {chunkSize: 1}}); assert.commandWorked( s.config.settings.update({_id: "balancer"}, {$set: {_waitForDelete: true}}, true)); @@ -61,7 +58,11 @@ s.printShardingStatus(); s.startBalancer(); -s.awaitBalance('foo', 'test', 60 * 1000); +assert.soon(function() { + var x = s.chunkDiff("foo", "test"); + print("chunk diff: " + x); + return x < 2; +}, "no balance happened", 8 * 60 * 1000, 2000); s.stop(); })(); diff --git a/jstests/sharding/migrateBig_balancer.js b/jstests/sharding/migrateBig_balancer.js new file mode 100644 index 00000000000..2517a11a70c --- /dev/null +++ b/jstests/sharding/migrateBig_balancer.js @@ -0,0 +1,64 @@ +/** + * This test is labeled resource intensive because its total io_write is 95MB compared to a median + * of 5MB across all sharding tests in wiredTiger. + * @tags: [resource_intensive] + */ +(function() { +"use strict"; + +load("jstests/sharding/libs/find_chunks_util.js"); + +var st = new ShardingTest( + {name: 'migrateBig_balancer', shards: 2, other: {enableBalancer: true, chunkSize: 64}}); +var mongos = st.s; +var admin = mongos.getDB("admin"); +var db = mongos.getDB("test"); +var coll = db.getCollection("stuff"); + +assert.commandWorked(admin.runCommand({enablesharding: coll.getDB().getName()})); +st.ensurePrimaryShard(coll.getDB().getName(), st.shard1.shardName); + +var data = "x"; +var nsq = 16; +var n = 255; + +for (var i = 0; i < nsq; i++) + data += data; + +var dataObj = {}; +for (var i = 0; i < n; i++) + dataObj["data-" + i] = data; + +var bulk = coll.initializeUnorderedBulkOp(); +for (var i = 0; i < 40; i++) { + bulk.insert({data: dataObj}); +} + +assert.commandWorked(bulk.execute()); +assert.eq(40, coll.count(), "prep1"); + +assert.commandWorked(admin.runCommand({shardcollection: "" + coll, key: {_id: 1}})); +st.printShardingStatus(); + +assert.lt(5, + findChunksUtil.findChunksByNs(mongos.getDB("config"), "test.stuff").count(), + "not enough chunks"); + +assert.soon(() => { + const aggMatch = (function() { + const collMetadata = mongos.getDB("config").collections.findOne({_id: "test.stuff"}); + if (collMetadata.timestamp) { + return {$match: {uuid: collMetadata.uuid}}; + } else { + return {$match: {ns: "test.stuff"}}; + } + }()); + let res = mongos.getDB("config") + .chunks.aggregate([aggMatch, {$group: {_id: "$shard", nChunks: {$sum: 1}}}]) + .toArray(); + printjson(res); + return res.length > 1 && Math.abs(res[0].nChunks - res[1].nChunks) <= 3; +}, "never migrated", 10 * 60 * 1000, 1000); + +st.stop(); +})(); diff --git a/jstests/sharding/min_max_key.js b/jstests/sharding/min_max_key.js deleted file mode 100644 index 7235216580e..00000000000 --- a/jstests/sharding/min_max_key.js +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Test writing and targeting of Max/Min key values - */ - -(function() { -'use strict'; - -const st = new ShardingTest({}); -const coll = st.s.getDB(jsTestName())['coll']; - -st.shardColl(coll, {x: 1}); - -assert.commandWorked(coll.insert({x: MaxKey})); -assert.eq(1, coll.countDocuments({})); -assert.eq(1, coll.countDocuments({x: MaxKey})); -assert.commandWorked(coll.remove({x: MaxKey})); -assert.eq(0, coll.countDocuments({})); -assert.eq(0, coll.countDocuments({x: MaxKey})); - -assert.commandWorked(coll.insert({x: MinKey})); -assert.eq(1, coll.countDocuments({})); -assert.eq(1, coll.countDocuments({x: MinKey})); -assert.commandWorked(coll.remove({x: MinKey})); -assert.eq(0, coll.countDocuments({})); -assert.eq(0, coll.countDocuments({x: MinKey})); - -st.stop(); -})(); diff --git a/jstests/sharding/mongos_noop_writes_wait_for_write_concern.js b/jstests/sharding/mongos_noop_writes_wait_for_write_concern.js deleted file mode 100644 index 0e177ac9e60..00000000000 --- a/jstests/sharding/mongos_noop_writes_wait_for_write_concern.js +++ /dev/null @@ -1,88 +0,0 @@ -// Sharding version of jstests/replsets/noop_writes_wait_for_write_concern.js. -// @tags: [ -// multiversion_incompatible -// ] - -load("jstests/libs/noop_write_commands.js"); -load("jstests/libs/write_concern_util.js"); - -// Create a shard with 3 nodes and stop one of the secondaries. This will allow majority write -// concern to be met, but w: 3 will always time out. -var st = new ShardingTest({mongos: 1, shards: 1, rs: {nodes: 3}}); -const secondary = st.rs0.getSecondary(); -st.rs0.stop(secondary); - -const mongos = st.s; -var dbName = 'testDB'; -var db = mongos.getDB(dbName); -var collName = 'testColl'; -var coll = db[collName]; -const commands = getNoopWriteCommands(coll); - -function dropTestCollection() { - coll.drop(); - assert.eq(0, coll.find().itcount(), "test collection not empty"); -} - -function testCommandWithWriteConcern(cmd) { - if ("applyOps" in cmd.req) { - // applyOps is not available through mongos. - return; - } - - if ("findAndModify" in cmd.req) { - // TODO SERVER-80103: findAndModify does not return write concern errors in the presence of - // other errors. - return; - } - - if ("dropDatabase" in cmd.req || "drop" in cmd.req) { - // TODO SERVER-80103: dropDatabase and dropCollection do not respect user supplied write - // concern and instead always use majority. - return; - } - - if ("create" in cmd.req) { - // TODO SERVER-80103: create returns WriteConcernFailed as an ordinary error code instead of - // using the writeConcernError field. - return; - } - - // Provide a small wtimeout that we expect to time out. - cmd.req.writeConcern = {w: 3, wtimeout: 1000}; - jsTest.log("Testing command: " + tojson(cmd.req)); - - dropTestCollection(); - - cmd.setupFunc(); - - // We run the command on a different connection. If the the command were run on the - // same connection, then the client last op for the noop write would be set by the setup - // operation. By using a fresh connection the client last op begins as null. - // This test explicitly tests that write concern for noop writes works when the - // client last op has not already been set by a duplicate operation. - const shell2 = new Mongo(mongos.host); - - // We check the error code of 'res' in the 'confirmFunc'. - const res = "bulkWrite" in cmd.req ? shell2.adminCommand(cmd.req) - : shell2.getDB(dbName).runCommand(cmd.req); - - try { - // Tests that the command receives a write concern error. If we don't wait for write - // concern on noop writes then we won't get a write concern error. - assertWriteConcernError(res); - cmd.confirmFunc(res); - } catch (e) { - // Make sure that we print out the response. - printjson(res); - throw e; - } -} - -commands.forEach(function(cmd) { - testCommandWithWriteConcern(cmd); -}); - -// Restart the node so that consistency checks performed by st.stop() can succeed. -st.rs0.restart(secondary); -st.stop(); diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js index 890fe0533ea..66c2133c90a 100644 --- a/jstests/sharding/mongos_rs_shard_failure_tolerance.js +++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js @@ -19,15 +19,7 @@ TestData.skipCheckOrphans = true; (function() { 'use strict'; -var st = new ShardingTest({ - shards: 3, - other: { - rs: true, - // Disables elections to avoid secondaries becoming primaries after stepdowns. The test - // relies on specific topology changes done explicitly. - rsOptions: {nodes: 2, settings: {electionTimeoutMillis: ReplSetTest.kForeverMillis}} - }, -}); +var st = new ShardingTest({shards: 3, mongos: 1, other: {rs: true, rsOptions: {nodes: 2}}}); var mongos = st.s0; var admin = mongos.getDB("admin"); @@ -138,19 +130,7 @@ jsTest.log("Testing active connection with second primary down..."); // Reads with read prefs mongosConnActive.setSecondaryOk(); assert.neq(null, mongosConnActive.getCollection(collSharded.toString()).findOne({_id: -1})); -// This special retry logic is to mimic connection pool timeout in v8.0 and later. In earlier -// versions, the connection pool may time out with NetworkInterfaceExceededTimeLimit, which -// is not a retryable error. This is necessary because the now downed primary may time out -// with NetworkInterfaceExceededTimeLimit instead of HostUnreachable. -var res = null; -try { - res = mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1}); -} catch (e) { - // RSM marks failed host. - assert.commandFailedWithCode(e, ErrorCodes.NetworkInterfaceExceededTimeLimit); - res = mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1}); -} -assert.neq(null, res); +assert.neq(null, mongosConnActive.getCollection(collSharded.toString()).findOne({_id: 1})); assert.neq(null, mongosConnActive.getCollection(collUnsharded.toString()).findOne({_id: 1})); mongosConnActive.setSecondaryOk(false); diff --git a/jstests/sharding/mongos_validate_writes.js b/jstests/sharding/mongos_validate_writes.js index fc6d4dccab3..0852dccf763 100644 --- a/jstests/sharding/mongos_validate_writes.js +++ b/jstests/sharding/mongos_validate_writes.js @@ -22,8 +22,8 @@ st.ensurePrimaryShard(coll.getDB().getName(), st.shard1.shardName); coll.createIndex({a: 1}); // Shard the collection on {a: 1} and move one chunk to another shard. Updates need to be across -// two shards to trigger an error, otherwise they are versioned and will succeed after raising a -// StaleConfig error. +// two shards to trigger an error, otherwise they are versioned and will succeed after raising +// a StaleConfigException. st.shardColl(coll, {a: 1}, {a: 0}, {a: 1}, coll.getDB(), true); // Let the stale mongos see the collection state diff --git a/jstests/sharding/move_chunk_allowMigrations.js b/jstests/sharding/move_chunk_allowMigrations.js index 06dae4bc2c1..6395a7a7424 100644 --- a/jstests/sharding/move_chunk_allowMigrations.js +++ b/jstests/sharding/move_chunk_allowMigrations.js @@ -6,19 +6,17 @@ * * @tags: [ * does_not_support_stepdowns, - * requires_fcv_60, * ] */ (function() { 'use strict'; -load("jstests/libs/feature_flag_util.js"); load('jstests/libs/fail_point_util.js'); load('jstests/libs/parallel_shell_helpers.js'); load("jstests/sharding/libs/find_chunks_util.js"); load("jstests/sharding/libs/shard_versioning_util.js"); -const st = new ShardingTest({shards: 2, other: {chunkSize: 1, enableAutoSplit: false}}); +const st = new ShardingTest({shards: 2}); const configDB = st.s.getDB("config"); // Resets database dbName and enables sharding and establishes shard0 as primary, test case agnostic @@ -147,14 +145,12 @@ function testAllowMigrationsFalseDisablesBalancer(allowMigrations, collBSetNoBal assert.commandWorked(st.s.adminCommand({shardCollection: collA.getFullName(), key: {_id: 1}})); assert.commandWorked(st.s.adminCommand({shardCollection: collB.getFullName(), key: {_id: 1}})); - const bigString = 'X'.repeat(1024 * 1024); // 1MB - // Split both collections into 4 chunks so balancing can occur. for (let coll of [collA, collB]) { - coll.insert({_id: 1, s: bigString}); - coll.insert({_id: 10, s: bigString}); - coll.insert({_id: 20, s: bigString}); - coll.insert({_id: 30, s: bigString}); + coll.insert({_id: 1}); + coll.insert({_id: 10}); + coll.insert({_id: 20}); + coll.insert({_id: 30}); assert.commandWorked(st.splitAt(coll.getFullName(), {_id: 10})); assert.commandWorked(st.splitAt(coll.getFullName(), {_id: 20})); @@ -181,9 +177,20 @@ function testAllowMigrationsFalseDisablesBalancer(allowMigrations, collBSetNoBal })); st.startBalancer(); - st.awaitBalance(collAName, dbName, 10 * 60000 /* 10min timeout */); + assert.soon(() => { + st.awaitBalancerRound(); + const shard0Chunks = + findChunksUtil + .findChunksByNs(configDB, collA.getFullName(), {shard: st.shard0.shardName}) + .itcount(); + const shard1Chunks = + findChunksUtil + .findChunksByNs(configDB, collA.getFullName(), {shard: st.shard1.shardName}) + .itcount(); + jsTestLog(`shard0 chunks ${shard0Chunks}, shard1 chunks ${shard1Chunks}`); + return shard0Chunks == 2 && shard1Chunks == 2; + }, `Balancer failed to balance ${collA.getFullName()}`, 1000 * 60 * 10); st.stopBalancer(); - st.verifyCollectionIsBalanced(collA); const collABalanceStatus = assert.commandWorked(st.s.adminCommand({balancerCollectionStatus: collA.getFullName()})); diff --git a/jstests/sharding/move_chunk_basic.js b/jstests/sharding/move_chunk_basic.js index 90e13f76733..cddec5c18aa 100644 --- a/jstests/sharding/move_chunk_basic.js +++ b/jstests/sharding/move_chunk_basic.js @@ -27,22 +27,6 @@ assert.commandFailed(mongos.adminCommand({moveChunk: 'a.b', find: {_id: 1}, to: assert.commandFailed( mongos.adminCommand({moveChunk: kDbName + '.xxx', find: {_id: 1}, to: shard1})); -assert.commandFailedWithCode( - st.rs0.getPrimary().adminCommand( - {setParameter: 1, chunkMigrationFetcherMaxBufferedSizeBytesPerThread: -5}), - ErrorCodes.InvalidOptions); - -assert.commandFailedWithCode( - st.rs0.getPrimary().adminCommand( - {setParameter: 1, chunkMigrationFetcherMaxBufferedSizeBytesPerThread: 1000}), - ErrorCodes.InvalidOptions); - -assert.commandWorked(st.rs0.getPrimary().adminCommand( - {setParameter: 1, chunkMigrationFetcherMaxBufferedSizeBytesPerThread: 20 * 1024 * 1024})); - -assert.commandWorked(st.rs0.getPrimary().adminCommand( - {setParameter: 1, chunkMigrationFetcherMaxBufferedSizeBytesPerThread: 0})); - function testHashed() { var ns = kDbName + '.fooHashed'; assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {_id: 'hashed'}})); diff --git a/jstests/sharding/move_chunk_concurrent_cloning.js b/jstests/sharding/move_chunk_concurrent_cloning.js deleted file mode 100644 index 5d1c1c249e1..00000000000 --- a/jstests/sharding/move_chunk_concurrent_cloning.js +++ /dev/null @@ -1,119 +0,0 @@ -/** - * @tags: [ - * featureFlagConcurrencyInChunkMigration, - * requires_fcv_60, - * ] - */ -(function() { -"use strict"; - -load('./jstests/libs/chunk_manipulation_util.js'); - -const runParallelMoveChunk = (numThreads) => { - // For startParallelOps to write its state - let staticMongod = MongoRunner.runMongod({}); - - let st = new ShardingTest({shards: 2}); - st.stopBalancer(); - - const kThreadCount = numThreads; - const kPadding = new Array(1024).join("x"); - - let testDB = st.s.getDB('test'); - assert.commandWorked(testDB.adminCommand({enableSharding: 'test'})); - st.ensurePrimaryShard('test', st.shard0.shardName); - assert.commandWorked(testDB.adminCommand({shardCollection: 'test.user', key: {x: 1}})); - - let shardKeyVal = 0; - const kDocsInBatch = 8 * 1000; - const kMinCollSize = 128 * 1024 * 1024; - let approxInsertedSize = 0; - while (approxInsertedSize < kMinCollSize) { - var bulk = testDB.user.initializeUnorderedBulkOp(); - for (let docs = 0; docs < kDocsInBatch; docs++) { - shardKeyVal++; - bulk.insert({_id: shardKeyVal, x: shardKeyVal, padding: kPadding}); - } - assert.commandWorked(bulk.execute()); - - approxInsertedSize = approxInsertedSize + (kDocsInBatch * 1024); - } - - const kInitialLoadFinalKey = shardKeyVal; - - print(`Running tests with chunkMigrationConcurrency == ${kThreadCount}`); - const kMaxBufferBytes = 17 * 1024 * 1024; - st._rs.forEach((replSet) => { - assert.commandWorked(replSet.test.getPrimary().adminCommand({ - setParameter: 1, - chunkMigrationConcurrency: kThreadCount, - chunkMigrationFetcherMaxBufferedSizeBytesPerThread: kMaxBufferBytes - })); - }); - - const configCollEntry = - st.s.getDB('config').getCollection('collections').findOne({_id: 'test.user'}); - let chunks = st.s.getDB('config').chunks.find({uuid: configCollEntry.uuid}).toArray(); - assert.eq(1, chunks.length, tojson(chunks)); - - let joinMoveChunk = - moveChunkParallel(staticMongod, st.s0.host, {x: 0}, null, 'test.user', st.shard1.shardName); - - // Migration cloning scans by shard key order. Perform some writes against the collection on - // both the lower and upper ends of the shard key values while migration is happening to - // exercise xferMods logic. - const kDeleteIndexOffset = kInitialLoadFinalKey - 3000; - const kUpdateIndexOffset = kInitialLoadFinalKey - 5000; - for (let x = 0; x < 1000; x++) { - assert.commandWorked(testDB.user.remove({x: x})); - assert.commandWorked(testDB.user.update({x: 4000 + x}, {$set: {updated: true}})); - - assert.commandWorked(testDB.user.remove({x: kDeleteIndexOffset + x})); - assert.commandWorked( - testDB.user.update({x: kUpdateIndexOffset + x}, {$set: {updated: true}})); - - let newShardKey = kInitialLoadFinalKey + x + 1; - assert.commandWorked(testDB.user.insert({_id: newShardKey, x: newShardKey})); - } - - joinMoveChunk(); - - let shardKeyIdx = 1000; // Index starts at 1k since we deleted the first 1k docs. - let cursor = testDB.user.find().sort({x: 1}); - - while (cursor.hasNext()) { - let next = cursor.next(); - assert.eq(next.x, shardKeyIdx); - - if ((shardKeyIdx >= 4000 && shardKeyIdx < 5000) || - (shardKeyIdx >= kUpdateIndexOffset && shardKeyIdx < (kUpdateIndexOffset + 1000))) { - assert.eq(true, next.updated, tojson(next)); - } - - shardKeyIdx++; - - if (shardKeyIdx == kDeleteIndexOffset) { - shardKeyIdx += 1000; - } - } - - shardKeyIdx--; - assert.eq(shardKeyIdx, kInitialLoadFinalKey + 1000); - - // server Status on the receiving shard - var serverStatus = st.shard1.getDB('admin').runCommand({serverStatus: 1}); - - assert.eq(kThreadCount, - serverStatus.shardingStatistics.chunkMigrationConcurrency, - tojson(serverStatus)); - st.stop(); - MongoRunner.stopMongod(staticMongod); -}; - -runParallelMoveChunk(1); - -// Run test a few times with random concurrency levels. -for (let i = 1; i <= 4; i++) { - runParallelMoveChunk(Math.floor(Math.random() * 31) + 1); -} -})(); diff --git a/jstests/sharding/move_chunk_deferred_lookup.js b/jstests/sharding/move_chunk_deferred_lookup.js deleted file mode 100644 index 7e3a9149c94..00000000000 --- a/jstests/sharding/move_chunk_deferred_lookup.js +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Ensure that updates are not lost if they are made between processing deferred updates and reading - * from the updates list in _transferMods. - * - * @tags: [uses_transactions, uses_prepare_transaction, requires_persistence] - */ - -(function() { -"use strict"; -load('jstests/libs/chunk_manipulation_util.js'); -load("jstests/libs/fail_point_util.js"); -load('jstests/replsets/rslib.js'); -load('jstests/sharding/libs/create_sharded_collection_util.js'); - -const dbName = "test"; -const collName = "user"; -const staticMongod = MongoRunner.runMongod({}); -const st = new ShardingTest({shards: {rs0: {nodes: 2}, rs1: {nodes: 1}}}); -const collection = st.s.getDB(dbName).getCollection(collName); -const lsid = { - id: UUID() -}; -const txnNumber = 0; - -function setup() { - CreateShardedCollectionUtil.shardCollectionWithChunks(collection, {_id: 1}, [ - {min: {_id: MinKey}, max: {_id: 10}, shard: st.shard0.shardName}, - {min: {_id: 10}, max: {_id: MaxKey}, shard: st.shard1.shardName}, - ]); - - for (let i = 0; i < 20; i++) { - assert.commandWorked(collection.insertOne({_id: i, x: i})); - } -} - -function prepareTransactionAndTriggerFailover() { - assert.commandWorked(st.s.getDB(dbName).runCommand({ - update: collName, - updates: [ - {q: {_id: 1}, u: {$set: {x: 5}}}, - {q: {_id: 2}, u: {$set: {x: -10}}}, - ], - lsid: lsid, - txnNumber: NumberLong(txnNumber), - stmtId: NumberInt(0), - startTransaction: true, - autocommit: false, - })); - - const result = assert.commandWorked(st.shard0.getDB(dbName).adminCommand({ - prepareTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - writeConcern: {w: "majority"}, - })); - - let oldSecondary = st.rs0.getSecondary(); - - st.rs0.stepUp(oldSecondary); - - awaitRSClientHosts(st.s, oldSecondary, {ok: true, ismaster: true}); - - return result.prepareTimestamp; -} - -function commitPreparedTransaction(prepareTimestamp) { - assert.commandWorked( - st.shard0.getDB(dbName).adminCommand(Object.assign({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - }, - {commitTimestamp: prepareTimestamp}))); -} - -function runMoveChunkAndCommitTransaction() { - const joinMoveChunk = moveChunkParallel( - staticMongod, st.s.host, {_id: 1}, null, 'test.user', st.shard1.shardName); - pauseMigrateAtStep(st.shard1, migrateStepNames.catchup); - waitForMoveChunkStep(st.shard0, moveChunkStepNames.startedMoveChunk); - commitPreparedTransaction(prepareTimestamp); - unpauseMigrateAtStep(st.shard1, migrateStepNames.catchup); - return joinMoveChunk; -} - -setup(); -const prepareTimestamp = prepareTransactionAndTriggerFailover(); -const fp = configureFailPoint(st.rs0.getPrimary(), "hangAfterProcessingDeferredXferMods"); -const joinMoveChunk = runMoveChunkAndCommitTransaction(); -fp.wait(); -assert.commandWorked(st.s.getDB(dbName).getCollection(collName).update({_id: 4}, {$set: {x: 501}})); -fp.off(); -joinMoveChunk(); -assert.eq(collection.findOne({_id: 4}).x, 501); - -st.stop(); - -MongoRunner.stopMongod(staticMongod); -})(); diff --git a/jstests/sharding/move_chunk_interrupt_postimage.js b/jstests/sharding/move_chunk_interrupt_postimage.js deleted file mode 100644 index 69537bb9ce0..00000000000 --- a/jstests/sharding/move_chunk_interrupt_postimage.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Tests that chunk migration interruption before processing post/pre image oplog entry leads - * to consistent config.transactions data across primary and secondaries (SERVER-67492) - */ - -(function() { -"use strict"; - -load("jstests/sharding/libs/create_sharded_collection_util.js"); -load("jstests/libs/fail_point_util.js"); -load('jstests/libs/parallel_shell_helpers.js'); - -const st = new ShardingTest({mongos: 1, config: 1, shards: 2, rs: {nodes: 2}}); -const interruptBeforeProcessingPrePostImageOriginatingOpFP = - configureFailPoint(st.rs1.getPrimary(), "interruptBeforeProcessingPrePostImageOriginatingOp"); - -const collection = st.s.getCollection("test.mycoll"); -CreateShardedCollectionUtil.shardCollectionWithChunks(collection, {x: 1}, [ - {min: {x: MinKey}, max: {x: 0}, shard: st.shard0.shardName}, - {min: {x: 0}, max: {x: 100}, shard: st.shard0.shardName}, - {min: {x: 100}, max: {x: MaxKey}, shard: st.shard1.shardName}, -]); - -assert.commandWorked(collection.insert({_id: 0, x: 10})); -assert.commandWorked(collection.runCommand("findAndModify", { - query: {x: 10}, - update: {$set: {y: 2}}, - new: true, - txnNumber: NumberLong(1), -})); - -const res = st.s.adminCommand({ - moveChunk: collection.getFullName(), - find: {x: 10}, - to: st.shard1.shardName, -}); -assert.commandFailedWithCode(res, ErrorCodes.CommandFailed); -interruptBeforeProcessingPrePostImageOriginatingOpFP.wait(); - -interruptBeforeProcessingPrePostImageOriginatingOpFP.off(); -st.stop(); -})(); diff --git a/jstests/sharding/move_chunk_permitMigrations.js b/jstests/sharding/move_chunk_permitMigrations.js index b2a2f545f25..e984a9fbdfb 100644 --- a/jstests/sharding/move_chunk_permitMigrations.js +++ b/jstests/sharding/move_chunk_permitMigrations.js @@ -4,7 +4,7 @@ * * @tags: [ * does_not_support_stepdowns, - * requires_fcv_60, + * requires_fcv_52, * ] */ (function() { @@ -15,7 +15,7 @@ load('jstests/libs/parallel_shell_helpers.js'); load("jstests/sharding/libs/find_chunks_util.js"); load("jstests/sharding/libs/shard_versioning_util.js"); -const st = new ShardingTest({shards: 2, other: {chunkSize: 1, enableAutoSplit: false}}); +const st = new ShardingTest({shards: 2}); const configDB = st.s.getDB("config"); const dbName = 'AllowMigrations'; @@ -65,14 +65,12 @@ const testBalancer = function(setAllowMigrations, collBSetNoBalanceParam) { assert.commandWorked(st.s.adminCommand({shardCollection: collA.getFullName(), key: {_id: 1}})); assert.commandWorked(st.s.adminCommand({shardCollection: collB.getFullName(), key: {_id: 1}})); - const bigString = 'X'.repeat(1024 * 1024); // 1MB - // Split both collections into 4 chunks so balancing can occur. for (let coll of [collA, collB]) { - coll.insert({_id: 1, s: bigString}); - coll.insert({_id: 10, s: bigString}); - coll.insert({_id: 20, s: bigString}); - coll.insert({_id: 30, s: bigString}); + coll.insert({_id: 1}); + coll.insert({_id: 10}); + coll.insert({_id: 20}); + coll.insert({_id: 30}); assert.commandWorked(st.splitAt(coll.getFullName(), {_id: 10})); assert.commandWorked(st.splitAt(coll.getFullName(), {_id: 20})); @@ -98,9 +96,20 @@ const testBalancer = function(setAllowMigrations, collBSetNoBalanceParam) { setAllowMigrationsCmd(collB.getFullName(), setAllowMigrations); st.startBalancer(); - st.awaitBalance(collAName, dbName); + assert.soon(() => { + st.awaitBalancerRound(); + const shard0Chunks = + findChunksUtil + .findChunksByNs(configDB, collA.getFullName(), {shard: st.shard0.shardName}) + .itcount(); + const shard1Chunks = + findChunksUtil + .findChunksByNs(configDB, collA.getFullName(), {shard: st.shard1.shardName}) + .itcount(); + jsTestLog(`shard0 chunks ${shard0Chunks}, shard1 chunks ${shard1Chunks}`); + return shard0Chunks == 2 && shard1Chunks == 2; + }, `Balancer failed to balance ${collA.getFullName()}`, 1000 * 60 * 10); st.stopBalancer(); - st.verifyCollectionIsBalanced(collA); const collABalanceStatus = assert.commandWorked(st.s.adminCommand({balancerCollectionStatus: collA.getFullName()})); diff --git a/jstests/sharding/move_primary_clone_test.js b/jstests/sharding/move_primary_clone_test.js index 5ce928c9580..404562e7cd1 100644 --- a/jstests/sharding/move_primary_clone_test.js +++ b/jstests/sharding/move_primary_clone_test.js @@ -58,12 +58,7 @@ function checkCollectionsCopiedCorrectly(fromShard, toShard, sharded, barUUID, f var indexes = res.cursor.firstBatch; indexes.sort(sortByName); - // TODO SERVER-74252: once 7.0 becomes LastLTS we can assume that the movePrimary will never - // copy indexes of sharded collections. - if (sharded) - assert(indexes.length == 1 || indexes.length == 2); - else - assert(indexes.length == 2); + assert.eq(indexes.length, 2); indexes.forEach((index, i) => { var expected; diff --git a/jstests/sharding/move_range_basic.js b/jstests/sharding/move_range_basic.js index 401516bbf73..25cd0da3497 100644 --- a/jstests/sharding/move_range_basic.js +++ b/jstests/sharding/move_range_basic.js @@ -2,6 +2,7 @@ * Basic tests for moveRange. * * @tags: [ + * featureFlagNoMoreAutoSplitter, * requires_fcv_60, * ] */ diff --git a/jstests/sharding/mr_single_reduce_split.js b/jstests/sharding/mr_single_reduce_split.js deleted file mode 100644 index 1a6d20f0e1f..00000000000 --- a/jstests/sharding/mr_single_reduce_split.js +++ /dev/null @@ -1,55 +0,0 @@ -/** - * This jstest verifies that the mrEnableSingleReduceOptimization flag works properly in a sharded - * cluster when there are documents on multiple chunks that need to be merged. - * @tags: [ - * backport_required_multiversion, - * ] - */ -(function() { -const st = new ShardingTest({ - shards: 2, - mongos: 1, - other: { - mongosOptions: {setParameter: {mrEnableSingleReduceOptimization: true}}, - shardOptions: {setParameter: {mrEnableSingleReduceOptimization: true}}, - } -}); - -const mongosDB = st.s0.getDB(jsTestName()); -const mongosColl = mongosDB[jsTestName()]; - -assert.commandWorked(mongosDB.dropDatabase()); -assert.commandWorked(mongosDB.adminCommand({enableSharding: mongosDB.getName()})); -st.ensurePrimaryShard(mongosDB.getName(), st.shard0.shardName); - -assert.commandWorked( - mongosDB.adminCommand({shardCollection: mongosColl.getFullName(), key: {_id: 1}})); - -// Split the collection into 2 chunks: [minKey, 0) and [0, maxKey]. -assert.commandWorked(mongosDB.adminCommand({split: mongosColl.getFullName(), middle: {_id: 0}})); - -// Move the [0, MaxKey) chunk to the second shard. -assert.commandWorked(mongosDB.adminCommand( - {moveChunk: mongosColl.getFullName(), find: {_id: 50}, to: st.shard1.shardName})); - -assert.commandWorked(mongosColl.insert({_id: -1})); -const map = function() { - emit(0, {val: "mapped value"}); -}; - -const reduce = function(key, values) { - return {val: "reduced value"}; -}; - -let res = assert.commandWorked(mongosDB.runCommand( - {mapReduce: mongosColl.getName(), map: map, reduce: reduce, out: {inline: 1}})); -assert.eq(res.results[0], {_id: 0, value: {val: "mapped value"}}); - -assert.commandWorked(mongosColl.insert({_id: 1})); - -res = assert.commandWorked(mongosDB.runCommand( - {mapReduce: mongosColl.getName(), map: map, reduce: reduce, out: {inline: 1}})); -assert.eq(res.results[0], {_id: 0, value: {val: "reduced value"}}); - -st.stop(); -}()); diff --git a/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js b/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js deleted file mode 100644 index fbd7dbd7d58..00000000000 --- a/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Tests that multi-document transactions fail with MigrationConflict (and TransientTransactionError - * label) if a collection or database placement changes have occurred later than the transaction - * data snapshot timestamp. - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallel_shell_helpers.js"); - -const st = new ShardingTest({mongos: 1, shards: 2}); - -// Test transaction with concurrent chunk migration. -{ - const dbName = 'test'; - const collName1 = 'foo'; - const collName2 = 'bar'; - const ns1 = dbName + '.' + collName1; - const ns2 = dbName + '.' + collName2; - - let coll1 = st.s.getDB(dbName)[collName1]; - let coll2 = st.s.getDB(dbName)[collName2]; - // Setup initial state: - // ns1: unsharded collection on shard0, with documents: {a: 0} - // ns2: sharded collection with chunks both on shard0 and shard1, with documents: {x: -1}, {x: - // 1} - st.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName}); - - st.adminCommand({shardCollection: ns2, key: {x: 1}}); - assert.commandWorked(st.splitAt(ns2, {x: 0})); - assert.commandWorked(st.moveChunk(ns2, {x: -1}, st.shard0.shardName)); - assert.commandWorked(st.moveChunk(ns2, {x: 0}, st.shard1.shardName)); - - assert.commandWorked(coll1.insert({a: 1})); - - assert.commandWorked(coll2.insert({x: -1})); - assert.commandWorked(coll2.insert({x: 1})); - - // Start a multi-document transaction and make one read on shard0 - const session = st.s.startSession(); - const sessionDB = session.getDatabase(dbName); - const sessionColl1 = sessionDB.getCollection(collName1); - const sessionColl2 = sessionDB.getCollection(collName2); - session.startTransaction(); // Default is local RC. With snapshot RC there's no bug. - assert.eq(1, sessionColl1.find().itcount()); - - // While the transaction is still open, move ns2's [0, 100) chunk to shard0. - assert.commandWorked(st.moveChunk(ns2, {x: 0}, st.shard0.shardName)); - // Refresh the router so that it doesn't send a stale SV to the shard, which would cause the txn - // to be aborted. - assert.eq(2, coll2.find().itcount()); - - // Trying to read coll2 will result in an error. Note that this is not retryable even with - // enableStaleVersionAndSnapshotRetriesWithinTransactions enabled because the first statement - // aleady had an active snapshot open on the same shard this request is trying to contact. - let err = assert.throwsWithCode(() => { - sessionColl2.find().itcount(); - }, ErrorCodes.MigrationConflict); - - assert.contains("TransientTransactionError", err.errorLabels, tojson(err)); -} - -// Test transaction with concurrent move primary. -{ - const dbName1 = 'test'; - const dbName2 = 'test2'; - const collName1 = 'foo'; - const collName2 = 'foo'; - - function runTest(readConcernLevel) { - st.getDB(dbName1).dropDatabase(); - st.getDB(dbName2).dropDatabase(); - st.adminCommand({enableSharding: dbName1, primaryShard: st.shard0.shardName}); - st.adminCommand({enableSharding: dbName2, primaryShard: st.shard1.shardName}); - - const coll1 = st.getDB(dbName1)[collName1]; - coll1.insert({x: 1, c: 0}); - - const coll2 = st.getDB(dbName2)[collName2]; - coll2.insert({x: 2, c: 0}); - - // Set failpoint to hang the movePrimary cloner after having created the collections on the - // destination shard but before cloning their documents. - let clonerFp = configureFailPoint(st.rs0.getPrimary(), - "movePrimaryClonerHangBeforeStartCloneDocuments"); - - let awaitMovePrimary = startParallelShell( - funWithArgs(function(dbName, toShard) { - assert.commandWorked(db.adminCommand({movePrimary: dbName, to: toShard})); - }, dbName2, st.shard0.shardName), st.s.port); - - clonerFp.wait(); - - // Start a multi-document transaction. Execute one statement that will target shard0. - let session = st.s.startSession(); - session.startTransaction({readConcern: {level: readConcernLevel}}); - assert.eq(1, session.getDatabase(dbName1)[collName1].find().itcount()); - - // Wait movePrimary to commit moving dbName2 from shard1 to shard0. - clonerFp.off(); - awaitMovePrimary(); - - // Make sure the router has fresh routing info to avoid causing the transaction to fail due - // to StaleConfig. - assert.eq(1, coll2.find().itcount()); - - // Execute a second statement, now on dbName2. This statement will be routed to shard0 - // (since there's no historical routing for databases). Expect it to fail with - // MigrationConflict error. - let err = assert.throwsWithCode(() => { - session.getDatabase(dbName2)[collName2].find().itcount(); - }, ErrorCodes.MigrationConflict); - - assert.contains("TransientTransactionError", err.errorLabels, tojson(err)); - } - - runTest('majority'); - runTest('snapshot'); -} - -st.stop(); -})(); diff --git a/jstests/sharding/multi_writes_with_shard_version_ignored_dont_bubble_up_critical_section.js b/jstests/sharding/multi_writes_with_shard_version_ignored_dont_bubble_up_critical_section.js deleted file mode 100644 index 82155211bef..00000000000 --- a/jstests/sharding/multi_writes_with_shard_version_ignored_dont_bubble_up_critical_section.js +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Tests that multi-writes where the router attaches 'shardVersion: IGNORED' (i.e. if they need to - * target several shards AND are not part of a txn) do not bubble up StaleConfig errors due to - * ongoing critical sections. Instead, the shard yields and waits for the critical section to finish - * and then continues the write plan. - */ - -(function() { -"use strict"; - -load('jstests/libs/parallel_shell_helpers.js'); -load("jstests/libs/fail_point_util.js"); - -// Configure 'internalQueryExecYieldIterations' on both shards such that operations will yield on -// each 10th PlanExecuter iteration. -var st = new ShardingTest({ - shards: 2, - rs: {setParameter: {internalQueryExecYieldIterations: 10}}, - other: {enableBalancer: false} -}); - -const dbName = "test"; -const collName = "foo"; -const ns = dbName + "." + collName; -const numDocs = 100; -let coll = st.s.getCollection(ns); - -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); - -function setupTest() { - coll.drop(); - assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}})); - - // Create three chunks: - // - [MinKey, 0) initially on shard0 and has no documents. This chunk will be migrated during - // the test execution. - // - [0, numDocs) on shard 0. Contains 'numDocs' documents. - // - [numDocs, MaxKey) shard 1. Contains no documents. - assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); - assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: numDocs}})); - assert.commandWorked(st.s.adminCommand( - {moveChunk: ns, find: {x: numDocs}, to: st.shard1.shardName, waitForDelete: true})); - - jsTest.log("Inserting initial data."); - const bulkOp = coll.initializeOrderedBulkOp(); - for (let i = 0; i < numDocs; ++i) { - bulkOp.insert({x: i, c: 0}); - } - assert.commandWorked(bulkOp.execute()); - jsTest.log("Inserted initial data."); -} - -function runMigration() { - const awaitResult = startParallelShell( - funWithArgs(function(ns, toShard) { - jsTest.log("Starting migration."); - assert.commandWorked(db.adminCommand({moveChunk: ns, find: {x: -1}, to: toShard})); - jsTest.log("Completed migration."); - }, ns, st.shard1.shardName), st.s.port); - - return awaitResult; -} - -function updateOperationFn(shardColl, numInitialDocsOnShard0) { - load('jstests/sharding/libs/shard_versioning_util.js'); // For kIgnoredShardVersion - - jsTest.log("Begin multi-update."); - - // Send a multi-update with 'shardVersion: IGNORED' directly to the shard, as if we were a - // router. - const result = assert.commandWorked(shardColl.runCommand({ - update: shardColl.getName(), - updates: [{q: {}, u: {$inc: {c: 1}}, multi: true}], - shardVersion: ShardVersioningUtil.kIgnoredShardVersion - })); - - jsTest.log("End multi-update. Result: " + tojson(result)); - - // Check that all documents got updates. Despite the weak guarantees of {multi: true} writes - // concurrent with migrations, this has to be the case in this test because the migrated chunk - // does not contain any document. - assert.eq(numInitialDocsOnShard0, shardColl.find({c: 1}).itcount()); -} - -function deleteOperationFn(shardColl, numInitialDocsOnShard0) { - load('jstests/sharding/libs/shard_versioning_util.js'); // For kIgnoredShardVersion - - jsTest.log("Begin multi-delete"); - - // Send a multi-delete with 'shardVersion: IGNORED' directly to the shard, as if we were a - // router. - const result = assert.commandWorked(shardColl.runCommand({ - delete: shardColl.getName(), - deletes: [{q: {}, limit: 0}], - shardVersion: ShardVersioningUtil.kIgnoredShardVersion - })); - - jsTest.log("End multi-delete. Result: " + tojson(result)); - - // Check that all documents got deleted. Despite the weak guarantees of {multi: true} writes - // concurrent with migrations, this has to be the case in this test because the migrated chunk - // does not contain any document. - assert.eq(0, shardColl.find().itcount()); -} - -function runTest(writeOpFn) { - setupTest(); - - let fp1 = configureFailPoint( - st.rs0.getPrimary(), 'setYieldAllLocksHang', {namespace: coll.getFullName()}); - - const awaitWriteResult = startParallelShell( - funWithArgs(function(writeOpFn, dbName, collName, numDocs) { - const shardColl = db.getSiblingDB(dbName)[collName]; - writeOpFn(shardColl, numDocs); - }, writeOpFn, coll.getDB().getName(), coll.getName(), numDocs), st.rs0.getPrimary().port); - - // Wait for the write op to yield. - fp1.wait(); - jsTest.log("Multi-write yielded"); - - // Start chunk migration and wait for it to enter the critical section. - let failpointHangMigrationWhileInCriticalSection = - configureFailPoint(st.rs0.getPrimary(), 'moveChunkHangAtStep5'); - const awaitMigration = runMigration(); - failpointHangMigrationWhileInCriticalSection.wait(); - - // Let the multi-write resume from the yield. - jsTest.log("Resuming yielded multi-write"); - fp1.off(); - - // Let the multi-write run for a bit after the resuming from yield. It will encounter the - // critical section. - sleep(1000); - - // Let the migration continue and release the critical section. - jsTest.log("Letting migration exit its critical section and complete"); - failpointHangMigrationWhileInCriticalSection.off(); - awaitMigration(); - - // Wait for the write op to finish. It should succeed. - awaitWriteResult(); -} - -runTest(updateOperationFn); -runTest(deleteOperationFn); - -st.stop(); -})(); diff --git a/jstests/sharding/prefix_shard_key.js b/jstests/sharding/prefix_shard_key.js index 78486319784..20b8224b790 100644 --- a/jstests/sharding/prefix_shard_key.js +++ b/jstests/sharding/prefix_shard_key.js @@ -61,7 +61,6 @@ assert.commandWorked(s.s0.adminCommand({shardCollection: coll.getFullName(), key assert.eq(2, coll.getIndexes().length); // make sure balancing happens -s.startBalancer(); s.awaitBalance(coll.getName(), db.getName()); // Make sure our initial balance cleanup doesn't interfere with later migrations. diff --git a/jstests/sharding/prepare_transaction_then_migrate.js b/jstests/sharding/prepare_transaction_then_migrate.js index 4411b3014c1..034259d02be 100644 --- a/jstests/sharding/prepare_transaction_then_migrate.js +++ b/jstests/sharding/prepare_transaction_then_migrate.js @@ -3,14 +3,12 @@ * 1. Ignore multi-statement transaction prepare conflicts in the clone phase, and * 2. Pick up the changes for prepared transactions in the transfer mods phase. * - * @tags: [uses_transactions, uses_prepare_transaction, requires_persistence] + * @tags: [uses_transactions, uses_prepare_transaction] */ (function() { "use strict"; load('jstests/libs/chunk_manipulation_util.js'); -load('jstests/replsets/rslib.js'); -load('jstests/sharding/libs/create_sharded_collection_util.js'); load('jstests/sharding/libs/sharded_transactions_helpers.js'); const dbName = "test"; @@ -18,179 +16,58 @@ const collName = "user"; const staticMongod = MongoRunner.runMongod({}); // For startParallelOps. -const TestMode = { - kBasic: 'basic', - kWithStepUp: 'with stepUp', - kWithRestart: 'with restart', -}; - -let runTest = function(testMode) { - jsTest.log(`Running test in mode ${testMode}`); - - const st = new ShardingTest( - {shards: {rs0: {nodes: testMode == TestMode.kWithStepUp ? 2 : 1}, rs1: {nodes: 1}}}); - const collection = st.s.getDB(dbName).getCollection(collName); - - CreateShardedCollectionUtil.shardCollectionWithChunks(collection, {x: 1}, [ - {min: {x: MinKey}, max: {x: 0}, shard: st.shard0.shardName}, - {min: {x: 0}, max: {x: 1000}, shard: st.shard0.shardName}, - {min: {x: 1000}, max: {x: MaxKey}, shard: st.shard1.shardName}, - ]); - - assert.commandWorked(collection.insert([ - {_id: 1, x: -1, note: "move into chunk range being migrated"}, - {_id: 2, x: -2, note: "keep out of chunk range being migrated"}, - {_id: 3, x: 50, note: "move out of chunk range being migrated"}, - {_id: 4, x: 100, note: "keep in chunk range being migrated"}, - ])); - - const lsid = {id: UUID()}; - const txnNumber = 0; - let stmtId = 0; - - assert.commandWorked(st.s0.getDB(dbName).runCommand({ - insert: collName, - documents: [ - {_id: 5, x: -1.01, note: "move into chunk range being migrated"}, - {_id: 6, x: -2.01, note: "keep out of chunk range being migrated"}, - {_id: 7, x: 50.01, note: "move out of chunk range being migrated"}, - {_id: 8, x: 100.01, note: "keep in chunk range being migrated"}, - ], - lsid: lsid, - txnNumber: NumberLong(txnNumber), - stmtId: NumberInt(stmtId++), - startTransaction: true, - autocommit: false, - })); - - assert.commandWorked(st.s.getDB(dbName).runCommand({ - update: collName, - updates: [ - {q: {x: -1}, u: {$set: {x: 5}}}, - {q: {x: -2}, u: {$set: {x: -10}}}, - {q: {x: 50}, u: {$set: {x: -20}}}, - {q: {x: 100}, u: {$set: {x: 500}}}, - {q: {x: -1.01}, u: {$set: {x: 5.01}}}, - {q: {x: -2.01}, u: {$set: {x: -10.01}}}, - {q: {x: 50.01}, u: {$set: {x: -20.01}}}, - {q: {x: 100.01}, u: {$set: {x: 500.01}}}, - ], - lsid: lsid, - txnNumber: NumberLong(txnNumber), - stmtId: NumberInt(stmtId++), - autocommit: false, - })); - - const res = assert.commandWorked(st.shard0.getDB(dbName).adminCommand({ - prepareTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - writeConcern: {w: "majority"}, - })); - - let prepareTimestamp = res.prepareTimestamp; - - if (testMode == TestMode.kWithStepUp) { - st.rs0.stepUp(st.rs0.getSecondary()); - - // Wait for the config server to see the new primary. - // TODO SERVER-74177 Remove this once retry on NotWritablePrimary is implemented. - st.forEachConfigServer((conn) => { - awaitRSClientHosts(conn, st.rs0.getPrimary(), {ok: true, ismaster: true}); - }); - } else if (testMode == TestMode.kWithRestart) { - TestData.skipCollectionAndIndexValidation = true; - st.rs0.restart(st.rs0.getPrimary()); - st.rs0.waitForPrimary(); - TestData.skipCollectionAndIndexValidation = false; - - assert.soon(() => { - try { - st.shard0.getDB(dbName).getCollection(collName).findOne(); - return true; - } catch (ex) { - print("Caught expected once exception due to restart: " + tojson(ex)); - return false; - } - }); - } +const st = new ShardingTest({shards: {rs0: {nodes: 1}, rs1: {nodes: 1}}}); +st.adminCommand({enableSharding: 'test'}); +st.ensurePrimaryShard('test', st.shard0.shardName); +st.adminCommand({shardCollection: 'test.user', key: {_id: 1}}); - const joinMoveChunk = - moveChunkParallel(staticMongod, st.s.host, {x: 1}, null, 'test.user', st.shard1.shardName); +const session = st.s.startSession({causalConsistency: false}); +const sessionDB = session.getDatabase(dbName); +const sessionColl = sessionDB.getCollection(collName); - pauseMigrateAtStep(st.shard1, migrateStepNames.catchup); +assert.commandWorked(sessionColl.insert({_id: 1})); - // The donor shard only ignores prepare conflicts while scanning over the shard key index. We - // wait for donor shard to have finished buffering the RecordIds into memory from scanning over - // the shard key index before committing the transaction. Notably, the donor shard doesn't - // ignore prepare conflicts when fetching the full contents of the documents during calls to - // _migrateClone. - // - // TODO: SERVER-71028 Remove comment after making changes. - - waitForMoveChunkStep(st.shard0, moveChunkStepNames.startedMoveChunk); - - assert.commandWorked( - st.shard0.getDB(dbName).adminCommand(Object.assign({ - commitTransaction: 1, - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false, - }, - {commitTimestamp: prepareTimestamp}))); - - unpauseMigrateAtStep(st.shard1, migrateStepNames.catchup); - - joinMoveChunk(); - - class ArrayCursor { - constructor(arr) { - this.i = 0; - this.arr = arr; - } - - hasNext() { - return this.i < this.arr.length; - } - - next() { - return this.arr[this.i++]; - } - } - - const expected = new ArrayCursor([ - {_id: 1, x: 5, note: "move into chunk range being migrated"}, - {_id: 2, x: -10, note: "keep out of chunk range being migrated"}, - {_id: 3, x: -20, note: "move out of chunk range being migrated"}, - {_id: 4, x: 500, note: "keep in chunk range being migrated"}, - {_id: 5, x: 5.01, note: "move into chunk range being migrated"}, - {_id: 6, x: -10.01, note: "keep out of chunk range being migrated"}, - {_id: 7, x: -20.01, note: "move out of chunk range being migrated"}, - {_id: 8, x: 500.01, note: "keep in chunk range being migrated"}, - ]); - - const diff = ((diff) => { - return { - docsWithDifferentContents: diff.docsWithDifferentContents.map( - ({first, second}) => ({expected: first, actual: second})), - docsExtraAfterMigration: diff.docsMissingOnFirst, - docsMissingAfterMigration: diff.docsMissingOnSecond, - }; - })(DataConsistencyChecker.getDiff(expected, collection.find().sort({_id: 1, x: 1}))); - - assert.eq(diff, { - docsWithDifferentContents: [], - docsExtraAfterMigration: [], - docsMissingAfterMigration: [], - }); - - st.stop(); +const lsid = { + id: UUID() }; - -runTest(TestMode.kBasic); -runTest(TestMode.kWithStepUp); -runTest(TestMode.kWithRestart); - +const txnNumber = 0; +const stmtId = 0; + +assert.commandWorked(st.s0.getDB(dbName).runCommand({ + insert: collName, + documents: [{_id: 2}, {_id: 5}, {_id: 15}], + lsid: lsid, + txnNumber: NumberLong(txnNumber), + stmtId: NumberInt(stmtId), + startTransaction: true, + autocommit: false, +})); + +const res = assert.commandWorked(st.shard0.getDB(dbName).adminCommand({ + prepareTransaction: 1, + lsid: lsid, + txnNumber: NumberLong(txnNumber), + autocommit: false, +})); + +const joinMoveChunk = + moveChunkParallel(staticMongod, st.s.host, {_id: 1}, null, 'test.user', st.shard1.shardName); + +// Wait for catchup to verify that the migration has exited the clone phase. +waitForMigrateStep(st.shard1, migrateStepNames.catchup); + +assert.commandWorked(st.shard0.getDB(dbName).adminCommand({ + commitTransaction: 1, + lsid: lsid, + txnNumber: NumberLong(txnNumber), + autocommit: false, + commitTimestamp: res.prepareTimestamp, +})); + +joinMoveChunk(); + +assert.eq(sessionColl.find({_id: 2}).count(), 1); + +st.stop(); MongoRunner.stopMongod(staticMongod); })(); diff --git a/jstests/sharding/presplit.js b/jstests/sharding/presplit.js new file mode 100644 index 00000000000..32f30a6aba7 --- /dev/null +++ b/jstests/sharding/presplit.js @@ -0,0 +1,47 @@ +/* + * @tags: [ + * requires_fcv_51, + * ] + */ + +(function() { + +load("jstests/sharding/libs/find_chunks_util.js"); + +var s = new ShardingTest({name: "presplit", shards: 2, mongos: 1, other: {chunkSize: 1}}); + +s.adminCommand({enablesharding: "test"}); +s.ensurePrimaryShard('test', s.shard1.shardName); + +// Insert enough data in 'test.foo' to fill several chunks, if it was sharded. +bigString = ""; +while (bigString.length < 10000) { + bigString += "asdasdasdasdadasdasdasdasdasdasdasdasda"; +} + +db = s.getDB("test"); +inserted = 0; +num = 0; +var bulk = db.foo.initializeUnorderedBulkOp(); +while (inserted < (20 * 1024 * 1024)) { + bulk.insert({_id: num++, s: bigString}); + inserted += bigString.length; +} +assert.commandWorked(bulk.execute()); + +// Make sure that there's only one chunk holding all the data. +s.printChunks(); +primary = s.getPrimaryShard("test").getDB("test"); +assert.eq(0, s.config.chunks.count({"ns": "test.foo"}), "single chunk assertion"); +assert.eq(num, primary.foo.count()); + +s.adminCommand({shardcollection: "test.foo", key: {_id: 1}}); + +// Make sure the collection's original chunk got split +s.printChunks(); +assert.lte(20, findChunksUtil.countChunksForNs(s.config, "test.foo"), "many chunks assertion"); +assert.eq(num, primary.foo.count()); + +s.printChangeLog(); +s.stop(); +})(); diff --git a/jstests/sharding/query/agg_explain_fmt.js b/jstests/sharding/query/agg_explain_fmt.js index a1049e6f937..b11381a63d1 100644 --- a/jstests/sharding/query/agg_explain_fmt.js +++ b/jstests/sharding/query/agg_explain_fmt.js @@ -59,6 +59,7 @@ assert.eq(mergeCursors.nss, "test.agg_explain_fmt", mergeCursors); // This test manually sets collection and db at the top. assert.eq(mergeCursors.allowPartialResults, false, mergeCursors); +assert.eq(mergeCursors.recordRemoteOpWaitTime, false, mergeCursors); // Do a sharded explain from a mongod, not mongos, to ensure that it does not have a // SHARDING_FILTER stage."); diff --git a/jstests/sharding/query/aggregation_currentop.js b/jstests/sharding/query/aggregation_currentop.js index 207d93b42f2..58d63128329 100644 --- a/jstests/sharding/query/aggregation_currentop.js +++ b/jstests/sharding/query/aggregation_currentop.js @@ -13,7 +13,7 @@ * applicable. * * This test requires replica set configuration and user credentials to persist across a restart. - * @tags: [requires_persistence, uses_transactions, uses_prepare_transaction, requires_fcv_60] + * @tags: [requires_persistence, uses_transactions, uses_prepare_transaction] */ // Restarts cause issues with authentication for awaiting replication. @@ -411,8 +411,7 @@ function runCommonTests(conn, curOpSpec) { explain: true })); - let expectedStages = - [{$currentOp: {idleConnections: true, allUsers: false}}, {$match: {desc: {$eq: "test"}}}]; + let expectedStages = [{$currentOp: {idleConnections: true}}, {$match: {desc: {$eq: "test"}}}]; if (isRemoteShardCurOp) { assert.docEq(explainPlan.splitPipeline.shardsPart, expectedStages); diff --git a/jstests/sharding/query/delete_with_partial_shard_key.js b/jstests/sharding/query/delete_with_partial_shard_key.js deleted file mode 100644 index 8fb85a40900..00000000000 --- a/jstests/sharding/query/delete_with_partial_shard_key.js +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Tests that delete queries that are not an exact match on shard key and target only a single shard - * work. - */ - -(function() { -'use strict'; - -load('jstests/sharding/libs/with_partial_shard_key_util.js'); - -const st = new ShardingTest({shards: 2}); - -// Setup the test by creating two chunks: -// 1. The first chunk has documents with "a" from [min, 2) and is on shard0. -// 2. The second chunk has documents with "a" from [2, max) and is on shard1. -const dbName = "test"; -const coll = "sharded_coll"; -const ns = dbName + "." + coll; -const db = st.getDB(dbName); -const docsToInsert = [{a: 1, b: 1}, {a: 2, b: 1}, {a: 3, b: 1}, {a: 999, b: 1}]; -assert.commandWorked(st.s0.adminCommand({enableSharding: dbName})); -assert.commandWorked(st.s0.adminCommand({shardcollection: ns, key: {a: 1, b: 1}})); -db.sharded_coll.insert(docsToInsert); -splitAndMoveChunks(st, - {a: 2, b: 1} /* split point */, - {a: 1, b: 1} /* move chunk containing doc to shard0 */, - {a: 3, b: 1} /* move chunk containing doc to shard1 */); - -// deleteOne query without the full shard key and only one shard targeted should succeed. -assert.eq(db.sharded_coll.count({a: 999}), 1); -let cmdObj = {delete: coll, deletes: [{q: {a: 999}, limit: 1}]}; -assert.commandWorked(db.runCommand(cmdObj)); -assert.eq(db.sharded_coll.count({a: 999}), 0); -assertExplainTargetsCorrectShard(db, cmdObj, st.shard1.shardName); - -// deleteOne query without the full shard key and multiple shards targeted should fail. -cmdObj = { - delete: coll, - deletes: [{q: {a: {$gt: 0}}, limit: 1}] -}; -assert.commandFailedWithCode(db.runCommand(cmdObj), ErrorCodes.ShardKeyNotFound); -assert.commandFailedWithCode(db.runCommand({explain: cmdObj}), ErrorCodes.ShardKeyNotFound); - -// deleteOne query without the shard key should fail (as this would requiring targeting more than -// one shard). -cmdObj = { - delete: coll, - deletes: [{q: {nonShardKey: 12345}, limit: 1}] -}; -assert.commandFailedWithCode(db.runCommand(cmdObj), ErrorCodes.ShardKeyNotFound); -assert.commandFailedWithCode(db.runCommand({explain: cmdObj}), ErrorCodes.ShardKeyNotFound); - -// deleteMany query that targets only one shard should work correctly. -assert.eq(db.sharded_coll.count({a: 1}), 1); -cmdObj = { - delete: coll, - deletes: [{q: {a: 1}, limit: 0}] -}; -assert.commandWorked(db.runCommand(cmdObj)); -assert.eq(db.sharded_coll.count({a: 1}), 0); -assert.eq(db.sharded_coll.count({a: {$gt: 0}}), 2); // Check that other documents were not deleted. -assertExplainTargetsCorrectShard(db, cmdObj, st.shard0.shardName); - -st.stop(); -})(); diff --git a/jstests/sharding/query/find_and_modify_with_partial_shard_key.js b/jstests/sharding/query/find_and_modify_with_partial_shard_key.js deleted file mode 100644 index e0a0bb1b3f9..00000000000 --- a/jstests/sharding/query/find_and_modify_with_partial_shard_key.js +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Tests that findAndModify queries that are not an exact match on shard key and target only a - * single shard work. - */ - -(function() { -'use strict'; - -load('jstests/sharding/libs/with_partial_shard_key_util.js'); - -const st = new ShardingTest({shards: 2}); - -// Setup the test by creating two chunks: -// 1. The first chunk has documents with "a" from [min, 2) and is on shard0. -// 2. The second chunk has documents with "a" from [2, max) and is on shard1. -const dbName = "test"; -const coll = "sharded_coll"; -const ns = dbName + "." + coll; -const db = st.getDB(dbName); -const docsToInsert = [{a: 1, b: 1}, {a: 2, b: 1}, {a: 3, b: 1}, {a: 999, b: 1}]; -assert.commandWorked(st.s0.adminCommand({enableSharding: dbName})); -assert.commandWorked(st.s0.adminCommand({shardcollection: ns, key: {a: 1, b: 1}})); -db.sharded_coll.insert(docsToInsert); -splitAndMoveChunks(st, - {a: 2, b: 1} /* split point */, - {a: 1, b: 1} /* move chunk containing doc to shard0 */, - {a: 3, b: 1} /* move chunk containing doc to shard1 */); - -// Query without the full shard key and only one shard targeted should succeed. -assert.eq(db.sharded_coll.count({a: 1, name: "bob"}), 0); -let cmdObj = {findAndModify: coll, query: {a: 1}, update: {$set: {name: "bob"}}}; -assert.commandWorked(db.runCommand(cmdObj)); -assert.eq(db.sharded_coll.count({a: 1, name: "bob"}), 1); -assertExplainTargetsCorrectShard(db, cmdObj, st.shard0.shardName); - -// Query without the full shard key and multiple shards targeted should fail. -cmdObj = { - findAndModify: coll, - query: {a: {$gt: 0}}, - update: {$set: {name: "bob"}} -}; -assert.commandFailedWithCode(db.runCommand(cmdObj), ErrorCodes.ShardKeyNotFound); -assert.commandFailedWithCode(db.runCommand({explain: cmdObj}), ErrorCodes.ShardKeyNotFound); - -// Query without the shard key should fail (as this would requiring targeting more than one shard). -cmdObj = { - findAndModify: coll, - query: {c: "3"}, - update: {$set: {name: "bob"}} -}; -assert.commandFailedWithCode(db.runCommand(cmdObj), ErrorCodes.ShardKeyNotFound); -assert.commandFailedWithCode(db.runCommand({explain: cmdObj}), ErrorCodes.ShardKeyNotFound); - -st.stop(); -})(); diff --git a/jstests/sharding/query/lookup_graph_lookup_foreign_becomes_sharded.js b/jstests/sharding/query/lookup_graph_lookup_foreign_becomes_sharded.js index 1bc05f59555..c3924654a74 100644 --- a/jstests/sharding/query/lookup_graph_lookup_foreign_becomes_sharded.js +++ b/jstests/sharding/query/lookup_graph_lookup_foreign_becomes_sharded.js @@ -225,14 +225,14 @@ for (let testCase of testCases) { const newStaleConfigErrorCount = assert.commandWorked(primaryDB.runCommand({serverStatus: 1})) .shardingStatistics.countStaleConfigErrors; -// ... and a single StaleConfig error for the foreign namespace. Note that the 'ns' field of the +// ... and a single StaleConfig exception for the foreign namespace. Note that the 'ns' field of the // profiler entry is the source collection in both cases, because the $lookup's parent aggregation // produces the profiler entry, and it is always running on the source collection. // TODO SERVER-60018: When the feature flag is removed, remove the check and ensure the results are // expected. if (!isShardedLookupEnabled) { - // Both in the classic lookup and the SBE lookup, StaleConfig error happens. In the classic - // lookup, profiling can properly report the StaleConfig of the foreign collection. + // Both in the classic lookup and the SBE lookup, 'StaleConfig' error happens. In the classic + // lookup, profiling can properly report the 'StaleConfig' of the foreign collection. // // On the other hand, in the SBE lookup, profiling fails to report the error because the // profiling level is not set up properly according to the configured profiling level in case of diff --git a/jstests/sharding/query/lookup_mongod_unaware.js b/jstests/sharding/query/lookup_mongod_unaware.js index 35cc9d920cb..41e5f23047b 100644 --- a/jstests/sharding/query/lookup_mongod_unaware.js +++ b/jstests/sharding/query/lookup_mongod_unaware.js @@ -6,9 +6,8 @@ * We restart a mongod to cause it to forget that a collection was sharded. When restarted, we * expect it to still have all the previous data. * - * // TODO (SERVER-74380): Remove requires_fcv_70 once SERVER-74380 has been backported to v6.0 * @tags: [ - * requires_persistence, + * requires_persistence * ] * */ @@ -19,7 +18,7 @@ load("jstests/noPassthrough/libs/server_parameter_helpers.js"); // For setParam load("jstests/libs/discover_topology.js"); // For findDataBearingNodes. // Restarts the primary shard and ensures that it believes both collections are unsharded. -function restartPrimaryShard(rs, ...expectedCollections) { +function restartPrimaryShard(rs, localColl, foreignColl) { // Returns true if the shard is aware that the collection is sharded. function hasRoutingInfoForNs(shardConn, coll) { const res = shardConn.adminCommand({getShardVersion: coll, fullMetadata: true}); @@ -29,11 +28,8 @@ function restartPrimaryShard(rs, ...expectedCollections) { rs.restart(0); rs.awaitSecondaryNodes(); - - expectedCollections.forEach(function(coll) { - assert(!hasRoutingInfoForNs(rs.getPrimary(), coll.getFullName()), - 'Shard role not cleared for ' + coll.getFullName()); - }); + assert(!hasRoutingInfoForNs(rs.getPrimary(), localColl.getFullName())); + assert(!hasRoutingInfoForNs(rs.getPrimary(), foreignColl.getFullName())); } // Disable checking for index consistency to ensure that the config server doesn't trigger a @@ -174,55 +170,5 @@ assert.eq(mongos0LocalColl.aggregate(pipeline).toArray(), expectedResults); restartPrimaryShard(st.rs0, mongos0LocalColl, mongos0ForeignColl); assert.eq(mongos1LocalColl.aggregate(pipeline).toArray(), expectedResults); -// -// Test two-level $lookup with a stale shard handles the shard role recovery case. -// -jsTest.log("Running two-level $lookup with a shard that needs recovery"); - -assert.commandWorked(st.s0.adminCommand({enableSharding: 'D'})); -st.ensurePrimaryShard('D', st.shard0.shardName); - -const D = st.s0.getDB('D'); - -assert.commandWorked(D.A.insert({Key: 1, Value: 1})); -assert.commandWorked(D.B.insert({Key: 1, Value: 1})); -assert.commandWorked(D.C.insert({Key: 1, Value: 1})); -assert.commandWorked(D.D.insert({Key: 1, Value: 1})); - -const aggPipeline = [{ - $lookup: { - from: 'B', - localField: 'Key', - foreignField: 'Value', - as: 'Joined', - pipeline: [ - { - $lookup: { - from: 'C', - localField: 'Key', - foreignField: 'Value', - as: 'Joined', - } - }, - { - $lookup: { - from: 'D', - localField: 'Key', - foreignField: 'Value', - as: 'Joined', - } - }, - ], - } -}]; - -const resultBefore = D.A.aggregate(aggPipeline).toArray(); - -// Restarting the shard primary in order for the shard role's cache to be cleared -restartPrimaryShard(st.rs0, D.A, D.B, D.C, D.D); - -const resultAfter = D.A.aggregate(aggPipeline).toArray(); -assert.eq(resultBefore, resultAfter, "Before and after results do not match"); - st.stop(); })(); diff --git a/jstests/sharding/query/merge_nondefault_read_concern.js b/jstests/sharding/query/merge_nondefault_read_concern.js deleted file mode 100644 index 70df9207bd8..00000000000 --- a/jstests/sharding/query/merge_nondefault_read_concern.js +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Tests that $merge doesn't fail when a non-default readConcern is - * set on the session. - * @tags: [requires_fcv_71] - */ -(function() { -"use strict"; - -load("jstests/aggregation/extras/merge_helpers.js"); - -const st = new ShardingTest({shards: 2, rs: {nodes: 1}}); - -const mongosDB = st.s0.getDB("merge_nondefault_read_concern"); -const source = mongosDB["source"]; -const target = mongosDB["target"]; - -assert.commandWorked(mongosDB.adminCommand({enableSharding: mongosDB.getName()})); - -const baseMergeCommand = { - aggregate: "source", - pipeline: - [{$merge: {into: "target", on: "_id", whenMatched: "replace", whenNotMatched: "insert"}}], - cursor: {}, -}; - -// Test with command level override. -var withReadConcern = baseMergeCommand; -withReadConcern.readConcern = { - level: "majority" -}; -assert.commandWorked(mongosDB.runCommand(withReadConcern)); - -// Test with global override. -assert.commandWorked( - mongosDB.adminCommand({"setDefaultRWConcern": 1, "defaultReadConcern": {level: "majority"}})); -assert.commandWorked(mongosDB.runCommand(baseMergeCommand)); - -st.stop(); -}()); diff --git a/jstests/sharding/query/merge_write_concern.js b/jstests/sharding/query/merge_write_concern.js index a29c8b7e6b8..1aac599f0f8 100644 --- a/jstests/sharding/query/merge_write_concern.js +++ b/jstests/sharding/query/merge_write_concern.js @@ -17,21 +17,12 @@ assert.commandWorked(mongosDB.adminCommand({enableSharding: mongosDB.getName()}) st.ensurePrimaryShard(mongosDB.getName(), st.shard0.shardName); function testWriteConcernError(rs) { - // Split the target collection at {_id: 10} so that there'll be doc $merge-ed to both shards. - if (FixtureHelpers.isSharded(target)) { - mongosDB.adminCommand({split: target.getFullName(), middle: {_id: 10}}); - } - // Make sure that there are only 2 nodes up so w:3 writes will always time out. const stoppedSecondary = rs.getSecondary(); rs.stop(stoppedSecondary); // Test that $merge correctly returns a WC error. withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { - // When either mode is "fail", a different error rather than WC error is thrown. - if (whenMatchedMode === "fail" || whenNotMatchedMode === "fail") { - return; - } const res = mongosDB.runCommand({ aggregate: "source", pipeline: [{ @@ -45,24 +36,13 @@ function testWriteConcernError(rs) { cursor: {}, }); - jsTestLog("Testing Mode: " + tojson(whenMatchedMode) + tojson(whenNotMatchedMode)); - jsTestLog("Target collection after $merge: " + tojson(target.find().toArray())); - - let oplogEntries = shard0.getPrimary() - .getDB("local") - .oplog.rs.find({"ns": {$regex: "merge_write_concern.*"}}) - .toArray(); - jsTestLog("Shard0 oplog entries: " + tojson(oplogEntries)); - oplogEntries = shard1.getPrimary() - .getDB("local") - .oplog.rs.find({"ns": {$regex: "merge_write_concern.*"}}) - .toArray(); - jsTestLog("Shard1 oplog entries: " + tojson(oplogEntries)); - // $merge writeConcern errors are handled differently from normal writeConcern // errors. Rather than returing ok:1 and a WriteConcernError, the entire operation // fails. - assert.commandFailedWithCode(res, ErrorCodes.WriteConcernFailed); + assert.commandFailedWithCode(res, + whenNotMatchedMode == "fail" + ? [13113, ErrorCodes.WriteConcernFailed] + : ErrorCodes.WriteConcernFailed); assert.commandWorked(target.remove({})); }); diff --git a/jstests/sharding/query/metadata_removal.js b/jstests/sharding/query/metadata_removal.js deleted file mode 100644 index 7caba6433bc..00000000000 --- a/jstests/sharding/query/metadata_removal.js +++ /dev/null @@ -1,46 +0,0 @@ -// Ensure that metadata fields (and only metadata) are removed before documents are returned to -// customers. As we can't insert documents with metadata, this test focuses on queries that might -// inject metadata when communication results to mongos for merging. In addition, we want to ensure -// that other fields beginning with a '$' are not modified before returning them to customers. -// @tags: [ -// requires_fcv_60 -// ] - -(function() { -"use strict"; - -function runTest(coll, keyword) { - const document = ({_id: 5, x: 1, $set: {$inc: {x: 5}}}); - assert.commandWorked(coll.insert(document)); - - // Leaves $set intact without shard merging. - assert.eq([document], coll.find().hint({$natural: 1}).toArray()); - assert.eq([document], coll.find().hint({_id: 1}).toArray()); - - // Leaves $set intact when shard merging on agg query. - const docWithY = ({_id: 5, x: 1, $set: {$inc: {x: 5}}, y: 1}); - assert.eq([docWithY], coll.aggregate([{$addFields: {y: 1}}]).toArray()); - - // Leaves $set intact when shard merging on sort query. - assert.eq([document], coll.find().sort({_id: 1}).toArray()); - - // Leaves added fields intact when user adds $ prefixed field in an agg pipeline. - const docWithDollarY = ({_id: 5, x: 1, $set: {$inc: {x: 5}}, $y: 1}); - assert.eq( - [docWithDollarY], - coll.aggregate( - [{$replaceWith: {$setField: {field: {$literal: "$y"}, input: "$$ROOT", value: 1}}}]) - .toArray()); -} - -const st = new ShardingTest({shards: 2}); -try { - assert.commandWorked(st.s0.adminCommand({enableSharding: 'test'})); - st.ensurePrimaryShard('test', st.shard1.shardName); - assert.commandWorked(st.s0.adminCommand({shardCollection: 'test.coll', key: {x: 'hashed'}})); - - runTest(st.getDB("test").coll); -} finally { - st.stop(); -} -})(); diff --git a/jstests/sharding/query/shard_refuses_cursor_ownership.js b/jstests/sharding/query/shard_refuses_cursor_ownership.js deleted file mode 100644 index 30e6e88d896..00000000000 --- a/jstests/sharding/query/shard_refuses_cursor_ownership.js +++ /dev/null @@ -1,83 +0,0 @@ -/** - * This test runs and unsharded $out query which results in mongos setting up a cursor and giving - * it to a shard to complete. mongos assumes the shard will kill the cursor, but if a shard doesn't - * accept ownership of the cursor then previously no one would kill it. this test ensures mongos - * will kill the cursor if a shard doesn't accept ownership. - * @tags: [ - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallel_shell_helpers.js"); - -const st = new ShardingTest({shards: 2}); - -const dbName = jsTestName(); -const collName = "foo"; -const ns = dbName + "." + collName; - -let db = st.s.getDB(dbName); -assert.commandWorked(db.dropDatabase()); -let coll = db[collName]; - -st.shardColl(collName, {x: 1}, {x: 0}, {x: 1}, dbName, true); - -assert.commandWorked(coll.insert([{x: -2}, {x: -1}, {x: 1}, {x: 2}])); - -const primary = st.getPrimaryShard(dbName); -const other = st.getOther(st.getPrimaryShard(dbName)); - -// Start an aggregation that requires merging on a shard. Let it run until the shard cursors have -// been established but make it hang right before opening the merge cursor. -let shardedAggregateHangBeforeDispatchMergingPipelineFP = - configureFailPoint(st.s, "shardedAggregateHangBeforeDispatchMergingPipeline"); -let awaitAggregationShell = startParallelShell( - funWithArgs((dbName, collName) => { - assert.eq( - 0, db.getSiblingDB(dbName)[collName].aggregate([{$out: collName + ".out"}]).itcount()); - }, dbName, collName), st.s.port); -shardedAggregateHangBeforeDispatchMergingPipelineFP.wait(); - -// Start a chunk migration, let it run until it enters the critical section. -let hangBeforePostMigrationCommitRefresh = - configureFailPoint(primary, "hangBeforePostMigrationCommitRefresh"); -let awaitMoveChunkShell = startParallelShell( - funWithArgs((recipientShard, ns) => { - assert.commandWorked(db.adminCommand({moveChunk: ns, find: {x: -1}, to: recipientShard})); - }, other.shardName, ns), st.s.port); -hangBeforePostMigrationCommitRefresh.wait(); - -// Let the aggregation continue and try to establish the merge cursor (it will first fail because -// the shard is in the critical section. Mongos will transparently retry). -shardedAggregateHangBeforeDispatchMergingPipelineFP.off(); - -// Let the migration exit the critical section and complete. -hangBeforePostMigrationCommitRefresh.off(); - -// The aggregation will be able to complete now. -awaitAggregationShell(); - -awaitMoveChunkShell(); - -// Did any cursor leak? -const idleCursors = primary.getDB("admin") - .aggregate([ - {$currentOp: {idleCursors: true, allUsers: true}}, - {$match: {type: "idleCursor", ns: ns}} - ]) - .toArray(); -assert.eq(0, idleCursors.length, "Found idle cursors: " + tojson(idleCursors)); - -// Check that range deletions can be completed (if a cursor was left open, the range deletion would -// not finish). -assert.soon( - () => { - return primary.getDB("config")["rangeDeletions"].find().itcount() === 0; - }, - "Range deletion tasks did not finish: + " + - tojson(primary.getDB("config")["rangeDeletions"].find().toArray())); - -st.stop(); -})(); diff --git a/jstests/sharding/range_deletions_has_index.js b/jstests/sharding/range_deletions_has_index.js index 9f00ca8970b..deef14a9550 100644 --- a/jstests/sharding/range_deletions_has_index.js +++ b/jstests/sharding/range_deletions_has_index.js @@ -14,7 +14,6 @@ const st = new ShardingTest({shards: {rs0: {nodes: 2}}}); // Test index gets created with an empty range deletions collection let newPrimary = st.rs0.getSecondaries()[0]; -st.rs0.awaitReplication(); assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1})); st.rs0.waitForPrimary(); const rangeDeletionColl = newPrimary.getDB("config").getCollection("rangeDeletions"); diff --git a/jstests/sharding/read_pref_with_hedging_mode.js b/jstests/sharding/read_pref_with_hedging_mode.js index 16fb1dfddf4..05c890885ee 100644 --- a/jstests/sharding/read_pref_with_hedging_mode.js +++ b/jstests/sharding/read_pref_with_hedging_mode.js @@ -29,11 +29,11 @@ assert.commandFailedWithCode(st.s.adminCommand({setParameter: 1, readHedgingMode ErrorCodes.BadValue); // Test setting maxTimeMS for hedged reads. -assert.commandWorked(st.s.adminCommand({setParameter: 1, maxTimeMSForHedgedReads: 1000})); +assert.commandWorked(st.s.adminCommand({setParameter: 1, maxTimeMSForHedgedReads: 100})); // Test hedging with maxTimeMS. assert.commandWorked(st.s.getDB(dbName).runCommand( - {query: {find: collName, maxTimeMS: 10000}, $readPreference: {mode: "nearest", hedge: {}}})); + {query: {find: collName, maxTimeMS: 1000}, $readPreference: {mode: "nearest", hedge: {}}})); // Test hedging without maxTimeMS. assert.commandWorked(st.s.getDB(dbName).runCommand({ diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js index 1f6effbcdcf..9b8b3a1efda 100644 --- a/jstests/sharding/read_write_concern_defaults_application.js +++ b/jstests/sharding/read_write_concern_defaults_application.js @@ -125,7 +125,6 @@ let testCases = { _configsvrShardCollection: {skip: "internal command"}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS _configsvrUpdateZoneKeyRange: {skip: "internal command"}, - _dropConnectionsToMongot: {skip: "internal command"}, _flushDatabaseCacheUpdates: {skip: "internal command"}, _flushDatabaseCacheUpdatesWithWriteConcern: {skip: "internal command"}, _flushReshardingStateChange: {skip: "internal command"}, @@ -139,7 +138,6 @@ let testCases = { _killOperations: {skip: "internal command"}, _mergeAuthzCollections: {skip: "internal command"}, _migrateClone: {skip: "internal command"}, - _mongotConnPoolStats: {skip: "internal command"}, _recvChunkAbort: {skip: "internal command"}, _recvChunkCommit: {skip: "internal command"}, _recvChunkReleaseCritSec: {skip: "internal command"}, @@ -342,7 +340,6 @@ let testCases = { shardedTargetsConfigServer: true, useLogs: true, }, - createSearchIndexes: {skip: "does not accept read or write concern"}, createUser: { command: {createUser: "foo", pwd: "bar", roles: []}, checkReadConcern: false, @@ -443,7 +440,6 @@ let testCases = { shardedTargetsConfigServer: true, useLogs: true, }, - dropSearchIndex: {skip: "does not accept read or write concern"}, dropUser: { setUp: function(conn) { assert.commandWorked(conn.getDB(db).runCommand( @@ -490,6 +486,7 @@ let testCases = { getDatabaseVersion: {skip: "does not accept read or write concern"}, getDefaultRWConcern: {skip: "does not accept read or write concern"}, getDiagnosticData: {skip: "does not accept read or write concern"}, + getFreeMonitoringStatus: {skip: "does not accept read or write concern"}, getLastError: {skip: "does not accept read or write concern"}, getLog: {skip: "does not accept read or write concern"}, getMore: {skip: "does not accept read or write concern"}, @@ -565,7 +562,6 @@ let testCases = { listCommands: {skip: "does not accept read or write concern"}, listDatabases: {skip: "does not accept read or write concern"}, listIndexes: {skip: "does not accept read or write concern"}, - listSearchIndexes: {skip: "does not accept read or write concern"}, listShards: {skip: "does not accept read or write concern"}, lockInfo: {skip: "does not accept read or write concern"}, logApplicationMessage: {skip: "does not accept read or write concern"}, @@ -615,6 +611,7 @@ let testCases = { checkReadConcern: false, checkWriteConcern: true, }, + repairDatabase: {skip: "does not accept read or write concern"}, repairShardedCollectionChunksHistory: {skip: "does not accept read or write concern"}, replSetAbortPrimaryCatchUp: {skip: "does not accept read or write concern"}, replSetFreeze: {skip: "does not accept read or write concern"}, @@ -700,7 +697,7 @@ let testCases = { setCommittedSnapshot: {skip: "internal command"}, setDefaultRWConcern: {skip: "special case (must run after all other commands)"}, setFeatureCompatibilityVersion: {skip: "does not accept read or write concern"}, - setProfilingFilterGlobally: {skip: "does not accept read or write concern"}, + setFreeMonitoring: {skip: "does not accept read or write concern"}, setIndexCommitQuorum: {skip: "does not accept read or write concern"}, setParameter: {skip: "does not accept read or write concern"}, setShardVersion: {skip: "internal command"}, @@ -747,7 +744,6 @@ let testCases = { shardedTargetsConfigServer: true, useLogs: true, }, - updateSearchIndex: {skip: "does not accept read or write concern"}, updateUser: { setUp: function(conn) { assert.commandWorked(conn.getDB(db).runCommand( @@ -893,11 +889,6 @@ function createProfileFilterForTestCase(test, targetId, explicitRWC) { function runScenario( desc, conn, regularCheckConn, configSvrCheckConn, {explicitRWC, explicitProvenance = false}) { let runCommandTest = function(cmdName, test) { - // These commands were removed but break this test in multiversion - if (cmdName === "getFreeMonitoringStatus" || cmdName === "setFreeMonitoring") { - return; - } - assert(test !== undefined, "coverage failure: must define a RWC defaults application test for " + cmdName); diff --git a/jstests/sharding/refine_collection_shard_key_basic.js b/jstests/sharding/refine_collection_shard_key_basic.js index 60b3a199b14..671493d99ee 100644 --- a/jstests/sharding/refine_collection_shard_key_basic.js +++ b/jstests/sharding/refine_collection_shard_key_basic.js @@ -1,8 +1,6 @@ // // Basic tests for refineCollectionShardKey. // -// Disabled in multiversion, see SERVER-69290 for more details. -// @tags: [multiversion_incompatible] (function() { 'use strict'; @@ -127,9 +125,11 @@ function validateCRUDAfterRefine() { assert.eq(4, sessionDB.getCollection(kCollName).findOne({c: -1}).b); mongos.setReadPref(null); - assert.commandWorked(sessionDB.getCollection(kCollName).remove({a: 1, b: 1}, true)); - assert.commandWorked(sessionDB.getCollection(kCollName).remove({a: -1, b: -1}, true)); - + // The full shard key is required when removing documents. + assert.writeErrorWithCode(sessionDB.getCollection(kCollName).remove({a: 1, b: 1}, true), + ErrorCodes.ShardKeyNotFound); + assert.writeErrorWithCode(sessionDB.getCollection(kCollName).remove({a: -1, b: -1}, true), + ErrorCodes.ShardKeyNotFound); assert.commandWorked(sessionDB.getCollection(kCollName).remove({a: 1, b: 2, c: 1, d: 1}, true)); assert.commandWorked( sessionDB.getCollection(kCollName).remove({a: -1, b: 4, c: -1, d: -1}, true)); @@ -249,8 +249,6 @@ function validateUnrelatedCollAfterRefine(oldCollArr, oldChunkArr, oldTagsArr) { jsTestLog('********** SIMPLE TESTS **********'); -var result; - // Should fail because arguments 'refineCollectionShardKey' and 'key' are invalid types. assert.commandFailedWithCode( mongos.adminCommand({refineCollectionShardKey: {_id: 1}, key: {_id: 1, aKey: 1}}), @@ -357,46 +355,36 @@ assert.commandFailedWithCode( dropAndReshardColl({_id: 1}); assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {sparse: true})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is sparse.")); - -// Should fail because index has a non-simple collation. -dropAndReshardColl({aKey: 1}); -assert.commandWorked(mongos.getCollection(kNsName).createIndex({aKey: 1, bKey: 1}, { - collation: { - locale: "en", - } -})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index has a non-simple collation.")); +assert.commandFailedWithCode( + mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}), + ErrorCodes.InvalidOptions); // Should fail because only a partial index exists for new shard key {_id: 1, aKey: 1}. dropAndReshardColl({_id: 1}); assert.commandWorked(mongos.getCollection(kNsName).createIndex( {_id: 1, aKey: 1}, {partialFilterExpression: {aKey: {$gt: 0}}})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is partial.")); +assert.commandFailedWithCode( + mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}), + ErrorCodes.InvalidOptions); // Should fail because only a multikey index exists for new shard key {_id: 1, aKey: 1}. dropAndReshardColl({_id: 1}); assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1})); assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is multikey.")); +assert.commandFailedWithCode( + mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}), + ErrorCodes.InvalidOptions); // Should fail because current shard key {a: 1} is unique, new shard key is {a: 1, b: 1}, and an // index only exists on {a: 1, b: 1, c: 1}. dropAndReshardCollUnique({a: 1}); assert.commandWorked(mongos.getCollection(kNsName).createIndex({a: 1, b: 1, c: 1})); -mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); +assert.commandFailedWithCode( + mongos.adminCommand({refineCollectionShardKey: kNsName, key: {a: 1, b: 1}}), + ErrorCodes.InvalidOptions); // Should work because current shard key {_id: 1} is not unique, new shard key is {_id: 1, aKey: // 1}, and an index exists on {_id: 1, aKey: 1, bKey: 1}. @@ -471,43 +459,6 @@ assert.commandFailedWithCode( mongos.adminCommand({refineCollectionShardKey: kNsName, key: {aKey: 1, bKey: 1}}), ErrorCodes.InvalidOptions); -// Should fail because index key is sparse and index has non-simple collation. -dropAndReshardColl({_id: 1}); -assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, { - sparse: true, - collation: { - locale: "en", - } -})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is sparse.") && - result.errmsg.includes("Index has a non-simple collation.")); - -// Should fail because index key is multikey and is partial. -dropAndReshardColl({_id: 1}); -assert.commandWorked(mongos.getCollection(kNsName).createIndex( - {_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}})); -assert.commandWorked( - mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1}, {name: "index_2"})); -assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]})); - -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is multikey.") && - result.errmsg.includes("Index key is partial.")); - -// Should fail because both indexes have keys that are incompatible: partial; sparse -dropAndReshardColl({_id: 1}); -assert.commandWorked(mongos.getCollection(kNsName).createIndex( - {_id: 1, aKey: 1}, {name: "index_1_part", partialFilterExpression: {aKey: {$gt: 0}}})); -assert.commandWorked(mongos.getCollection(kNsName).createIndex( - {_id: 1, aKey: 1}, {name: "index_2_sparse", sparse: true})); -result = mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}); -assert.commandFailedWithCode(result, ErrorCodes.InvalidOptions); -assert(result.errmsg.includes("Index key is partial.") && - result.errmsg.includes("Index key is sparse.")); - // Should work because a 'useful' index exists for new shard key {_id: 1, aKey: 1}. dropAndReshardColl({_id: 1}); assert.commandWorked(mongos.getCollection(kNsName).createIndex({_id: 1, aKey: 1})); diff --git a/jstests/sharding/rename.js b/jstests/sharding/rename.js index 472b333ff43..4759127de02 100644 --- a/jstests/sharding/rename.js +++ b/jstests/sharding/rename.js @@ -37,18 +37,30 @@ assert.commandWorked( assert.commandFailed(db.bar.renameCollection('shardedColl')); // Renaming unsharded collection to a different db with different primary shard. -db.unsharded.insert({x: 1}); +db.unSharded.insert({x: 1}); assert.commandFailedWithCode( - db.adminCommand({renameCollection: 'test.unsharded', to: 'otherDBDifferentPrimary.foo'}), + db.adminCommand({renameCollection: 'test.unSharded', to: 'otherDBDifferentPrimary.foo'}), [ErrorCodes.CommandFailed], "Source and destination collections must be on the same database."); // Renaming unsharded collection to a different db with same primary shard. assert.commandWorked( - db.adminCommand({renameCollection: 'test.unsharded', to: 'otherDBSamePrimary.foo'})); + db.adminCommand({renameCollection: 'test.unSharded', to: 'otherDBSamePrimary.foo'})); assert.eq(0, db.unsharded.countDocuments({})); assert.eq(1, s.getDB('otherDBSamePrimary').foo.countDocuments({})); +jsTest.log("Testing that rename operations involving views are not allowed"); +{ + assert.commandWorked(db.collForView.insert({_id: 1})); + assert.commandWorked(db.createView('view', 'collForView', [])); + + let toAView = db.unsharded.renameCollection('view', true /* dropTarget */); + assert.commandFailed(toAView); + + let fromAView = db.view.renameCollection('target'); + assert.commandFailed(fromAView); +} + // Rename a collection to itself fails, without loosing data { const sameCollName = 'sameColl'; @@ -61,35 +73,6 @@ assert.eq(1, s.getDB('otherDBSamePrimary').foo.countDocuments({})); assert.eq(1, sameColl.countDocuments({}), "Rename a collection to itself must not loose data"); } -const testDB = s.rs0.getPrimary().getDB('test'); -const fcvDoc = testDB.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}); -if (MongoRunner.compareBinVersions(fcvDoc.featureCompatibilityVersion.version, '6.0') >= 0) { - // Create collection on non-primary shard (shard1 for test db) to simulate wrong creation via - // direct connection: collection rename should fail since `badcollection` uuids are inconsistent - // across shards - jsTest.log("Testing uuid consistency across shards"); - assert.commandWorked( - s.shard1.getDB('test').badcollection.insert({_id: 1})); // direct connection - assert.commandWorked(s.s0.getDB('test').badcollection.insert({_id: 1})); // mongos connection - assert.commandFailedWithCode( - s.s0.getDB('test').badcollection.renameCollection('goodcollection'), - [ErrorCodes.InvalidUUID], - "collection rename should fail since test.badcollection uuids are inconsistent across shards"); - - // Target collection existing on non-primary shard: rename with `dropTarget=false` must fail - jsTest.log( - "Testing rename behavior when target collection [wrongly] exists on non-primary shards"); - assert.commandWorked( - s.shard1.getDB('test').superbadcollection.insert({_id: 1})); // direct connection - assert.commandWorked(s.s0.getDB('test').goodcollection.insert({_id: 1})); // mongos connection - assert.commandFailedWithCode( - s.s0.getDB('test').goodcollection.renameCollection('superbadcollection', false), - [ErrorCodes.NamespaceExists], - "Collection rename with `dropTarget=false` must have failed because target collection exists on a non-primary shard"); - // Target collection existing on non-primary shard: rename with `dropTarget=true` must succeed - assert.commandWorked( - s.s0.getDB('test').goodcollection.renameCollection('superbadcollection', true)); -} // Ensure write concern works by shutting down 1 node in a replica set shard jsTest.log("Testing write concern (2)"); diff --git a/jstests/sharding/rename_sharded.js b/jstests/sharding/rename_sharded.js index 6112bd2c9a4..35764906362 100644 --- a/jstests/sharding/rename_sharded.js +++ b/jstests/sharding/rename_sharded.js @@ -59,37 +59,11 @@ const st = new ShardingTest({shards: 3, mongos: 1, other: {enableBalancer: false const mongos = st.s0; -// Rename non-existing source collection to a target collection/view (dropTarget=false) must -// fail with NamespaceNotFound. Make sure the check on the source is done before any check on the -// target for consistency with replicaset. We cannot delegate this to passthrough suite since in -// those suites any non-existing collection will always be implicitely sharded at the first access +// Rename to non-existing target collection must succeed { - const dbName = 'notExistingSource'; - assert.commandWorked( - mongos.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); - - // Rename non-existing source to non-existing target - assert.commandFailedWithCode( - st.getDB(dbName).adminCommand( - {renameCollection: dbName + ".source", to: dbName + ".target"}), - ErrorCodes.NamespaceNotFound); - - // Rename non-existing source to existing collection - const toCollName = dbName + ".target"; - const toColl = mongos.getCollection(toCollName); - toColl.insert({a: 0}); - - assert.commandFailedWithCode( - st.getDB(dbName).adminCommand({renameCollection: dbName + ".source", to: toCollName}), - ErrorCodes.NamespaceNotFound); - - // Rename non-existing source to existing view - const toViewName = dbName + ".target_view"; - assert.commandWorked(st.getDB(dbName).createView(toViewName, toCollName, [])); - - assert.commandFailedWithCode( - st.getDB(dbName).adminCommand({renameCollection: dbName + ".source", to: toViewName}), - ErrorCodes.NamespaceNotFound); + const dbName = 'testRenameToNewCollection'; + const toNs = dbName + '.to'; + testRename(st, dbName, toNs, false /* dropTarget */, false /* mustFail */); } // Rename to existing sharded target collection with dropTarget=true must succeed @@ -140,6 +114,19 @@ const mongos = st.s0; testRename(st, dbName, toNs, false /* dropTarget */, true /* mustFail */); } +// Rename to existing sharded target collection with dropTarget=false must fail +{ + const dbName = 'testRenameToShardedCollectionWithoutDropTarget'; + const toNs = dbName + '.to'; + assert.commandWorked( + mongos.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); + assert.commandWorked(mongos.adminCommand({shardCollection: toNs, key: {a: 1}})); + const toColl = mongos.getCollection(toNs); + toColl.insert({a: 0}); + + testRename(st, dbName, toNs, false /* dropTarget */, true /* mustFail */); +} + // Rename unsharded collection to sharded target collection with dropTarget=true must succeed { const dbName = 'testRenameUnshardedToShardedTargetCollection'; diff --git a/jstests/sharding/replication_with_undefined_shard_key.js b/jstests/sharding/replication_with_undefined_shard_key.js index a363ff06b9e..f684bd29e29 100644 --- a/jstests/sharding/replication_with_undefined_shard_key.js +++ b/jstests/sharding/replication_with_undefined_shard_key.js @@ -22,9 +22,9 @@ jsTestLog("Doing writes that generate oplog entries including undefined document assert.commandWorked(mongosColl.update( {}, {$set: {a: 1}}, - {multi: true, writeConcern: {w: 2, wtimeout: ReplSetTest.kDefaultTimeoutMS}})); + {multi: true, writeConcern: {w: 2, wtimeout: ReplSetTest.kDefaultTimeoutMs}})); assert.commandWorked( - mongosColl.remove({}, {writeConcern: {w: 2, wtimeout: ReplSetTest.kDefaultTimeoutMS}})); + mongosColl.remove({}, {writeConcern: {w: 2, wtimeout: ReplSetTest.kDefaultTimeoutMs}})); st.stop(); })();
\ No newline at end of file diff --git a/jstests/sharding/reshard_collection_basic.js b/jstests/sharding/reshard_collection_basic.js index 4b527757ecd..c235243264a 100644 --- a/jstests/sharding/reshard_collection_basic.js +++ b/jstests/sharding/reshard_collection_basic.js @@ -150,14 +150,6 @@ let verifyAllShardingCollectionsRemoved = (tempReshardingCollName) => { .itcount()); }; -let verifyTagsDocumentsAfterOperationCompletes = (ns, shardKeyPattern) => { - const tagsArr = mongos.getCollection('config.tags').find({ns: ns}).toArray(); - for (let i = 0; i < tagsArr.length; ++i) { - assert.eq(Object.keys(tagsArr[i]["min"]), shardKeyPattern); - assert.eq(Object.keys(tagsArr[i]["max"]), shardKeyPattern); - } -}; - let assertReshardCollOkWithPreset = (commandObj, presetReshardedChunks) => { assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {oldKey: 1}})); @@ -174,9 +166,6 @@ let assertReshardCollOkWithPreset = (commandObj, presetReshardedChunks) => { verifyTemporaryReshardingCollectionExistsWithCorrectOptions( getAllShardIdsFromExpectedChunks(presetReshardedChunks)); - - verifyTagsDocumentsAfterOperationCompletes(ns, Object.keys(commandObj.key)); - verifyChunksMatchExpected(presetReshardedChunks.length, presetReshardedChunks); mongos.getDB(kDbName)[collName].drop(); @@ -196,8 +185,6 @@ let assertReshardCollOk = (commandObj, expectedChunks) => { assert.commandWorked(mongos.adminCommand(commandObj)); - verifyTagsDocumentsAfterOperationCompletes(ns, Object.keys(commandObj.key)); - verifyChunksMatchExpected(expectedChunks); mongos.getDB(kDbName)[collName].drop(); @@ -259,14 +246,6 @@ assert.commandFailedWithCode(mongos.adminCommand({ }), 4952607); -jsTest.log("Fail if zone provided is invalid for storage."); -assert.commandFailedWithCode(mongos.adminCommand({ - reshardCollection: ns, - key: {"_id": "hashed"}, - zones: [{min: {"_id": {"$minKey": 1}}, max: {"_id": {"$maxKey": 1}}, zone: "Namezone"}] -}), - ErrorCodes.BadValue); - jsTestLog("Fail if splitting collection into multiple chunks while it is still empty."); assert.commandFailedWithCode( mongos.adminCommand({reshardCollection: ns, key: {b: 1}, numInitialChunks: 2}), 4952606); @@ -370,20 +349,6 @@ assertReshardCollOk({ }, 1); -jsTest.log("Succeed if zones are not empty."); -assert.commandWorked( - mongos.adminCommand({addShardToZone: st.shard1.shardName, zone: existingZoneName})); -assert.commandWorked(st.s.adminCommand( - {updateZoneKeyRange: ns, min: {oldKey: 0}, max: {oldKey: 5}, zone: existingZoneName})); -assertReshardCollOk({ - reshardCollection: ns, - key: {oldKey: 1, newKey: 1}, - unique: false, - collation: {locale: 'simple'}, - zones: [{zone: existingZoneName, min: {oldKey: 0}, max: {oldKey: 5}}] -}, - 3); - jsTest.log("Succeed with hashed shard key that provides enough cardinality."); assert.commandWorked( mongos.adminCommand({shardCollection: ns, key: {a: "hashed"}, numInitialChunks: 5})); diff --git a/jstests/sharding/resharding_abort_command.js b/jstests/sharding/resharding_abort_command.js index 7c5f4a95cb0..f8e7aeaf9f5 100644 --- a/jstests/sharding/resharding_abort_command.js +++ b/jstests/sharding/resharding_abort_command.js @@ -177,7 +177,7 @@ const runAbortWithFailpoint = (failpointName, failpointNodeType, abortLocation, // Resharding has not been attempted yet, so resharding metrics will not be reported. This means // shardingStatistics will be empty, and thus not reported. So we assert that the serverStatus // does not have shardingStatistics yet. - assert(!status.shardingStatistics.hasOwnProperty("resharding"), status); + assert(!status.hasOwnProperty('shardingStatistics'), status); let expectedAbortErrorCodes = ErrorCodes.OK; let expectedReshardingErrorCode = ErrorCodes.ReshardCollectionAborted; diff --git a/jstests/sharding/resharding_abort_in_preparing_to_donate.js b/jstests/sharding/resharding_abort_in_preparing_to_donate.js index 711dbb71860..167dcd3c67a 100644 --- a/jstests/sharding/resharding_abort_in_preparing_to_donate.js +++ b/jstests/sharding/resharding_abort_in_preparing_to_donate.js @@ -11,7 +11,6 @@ "use strict"; load("jstests/libs/discover_topology.js"); load("jstests/sharding/libs/resharding_test_fixture.js"); -load('jstests/libs/parallel_shell_helpers.js'); const originalCollectionNs = "reshardingDb.coll"; @@ -37,7 +36,6 @@ const configsvr = new Mongo(topology.configsvr.nodes[0]); const pauseAfterPreparingToDonateFP = configureFailPoint(configsvr, "reshardingPauseCoordinatorAfterPreparingToDonate"); -let awaitAbort; reshardingTest.withReshardingInBackground( { @@ -49,30 +47,13 @@ reshardingTest.withReshardingInBackground( }, () => { pauseAfterPreparingToDonateFP.wait(); - assert.neq(null, mongos.getCollection("config.reshardingOperations").findOne({ - ns: originalCollectionNs - })); + assert.commandWorked(mongos.adminCommand({abortReshardCollection: originalCollectionNs})); // Signaling abort will cause the // pauseAfterPreparingToDonateFP to throw, implicitly // allowing the coordinator to make progress without // explicitly turning off the failpoint. - awaitAbort = - startParallelShell(funWithArgs(function(sourceNamespace) { - db.adminCommand({abortReshardCollection: sourceNamespace}); - }, originalCollectionNs), mongos.port); - // Wait for the coordinator to remove coordinator document from config.reshardingOperations - // as a result of the recipients and donors transitioning to done due to abort. - assert.soon(() => { - const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({ - ns: originalCollectionNs - }); - return coordinatorDoc === null || coordinatorDoc.state === "aborting"; - }); }, {expectedErrorCode: ErrorCodes.ReshardCollectionAborted}); - -awaitAbort(); pauseAfterPreparingToDonateFP.off(); - reshardingTest.teardown(); })(); diff --git a/jstests/sharding/resharding_coordinator_recovers_abort_decision.js b/jstests/sharding/resharding_coordinator_recovers_abort_decision.js index 124e575208b..80e0ef14b77 100644 --- a/jstests/sharding/resharding_coordinator_recovers_abort_decision.js +++ b/jstests/sharding/resharding_coordinator_recovers_abort_decision.js @@ -21,8 +21,7 @@ const sourceCollection = reshardingTest.createShardedCollection({ }); const mongos = sourceCollection.getMongo(); -const ns = sourceCollection.getFullName(); -let topology = DiscoverTopology.findConnectedNodes(mongos); +const topology = DiscoverTopology.findConnectedNodes(mongos); const recipientShardNames = reshardingTest.recipientShardNames; const recipient = new Mongo(topology.shards[recipientShardNames[0]].primary); @@ -39,13 +38,6 @@ const shardsvrAbortReshardCollectionFailpoint = configureFailPoint(recipient, "f failCommands: ["_shardsvrAbortReshardCollection"], }); -// We pause the _configsvrReshardCollection command upon joining an existing ReshardingCoordinator -// instance on all of the config server replica set because we don't know which node will be elected -// primary from calling stepUpNewPrimaryOnShard(). -const configsvrConnections = topology.configsvr.nodes.map(host => new Mongo(host)); -const reshardCollectionJoinedFailPointsList = configsvrConnections.map( - conn => configureFailPoint(conn, "reshardCollectionJoinedExistingOperation")); - let awaitAbort; reshardingTest.withReshardingInBackground( { @@ -56,6 +48,7 @@ reshardingTest.withReshardingInBackground( // Wait until participants are aware of the resharding operation. reshardingTest.awaitCloneTimestampChosen(); + const ns = sourceCollection.getFullName(); awaitAbort = startParallelShell(funWithArgs(function(ns) { db.adminCommand({abortReshardCollection: ns}); }, ns), mongos.port); @@ -77,12 +70,13 @@ reshardingTest.withReshardingInBackground( // Mongos automatically retries the abortReshardCollection command on retryable errors. // We interrupt the abortReshardCollection command running on mongos to verify that the // ReshardingCoordinator recovers the decision on its own. - const ops = mongos.getDB("admin") - .aggregate([ - {$currentOp: {localOps: true}}, - {$match: {"command.abortReshardCollection": ns}} - ]) - .toArray(); + const ops = + mongos.getDB("admin") + .aggregate([ + {$currentOp: {localOps: true}}, + {$match: {"command.abortReshardCollection": sourceCollection.getFullName()}} + ]) + .toArray(); assert.neq([], ops, "failed to find abortReshardCollection command running on mongos"); assert.eq( @@ -93,34 +87,7 @@ reshardingTest.withReshardingInBackground( assert.commandWorked(mongos.getDB("admin").killOp(ops[0].opid)); - // Step down the config shard's primary. - let replSet = reshardingTest.getReplSetForShard(reshardingTest.configShardName); - let primary = replSet.getPrimary(); - assert.commandWorked( - primary.getDB("admin").runCommand({replSetStepDown: 60, force: true})); - - // After a stepdown, the _configsvrReshardCollection command will be retried by the - // primary shard. We use the reshardCollectionJoinedExistingOperation failpoint to - // ensure the primary shard upon retrying finds the ongoing resharding operation on the - // new config server primary. It would otherwise be possible for the - // reshardingPauseCoordinatorBeforeCompletion failpoint to be released by the - // ReshardingTest fixture after this function returns, for the ongoing resharding - // operation to complete, and for the retried _configsvrReshardCollection command to - // spawn an entirely new resharding operation which won't get aborted by the test - // client. - topology = DiscoverTopology.findConnectedNodes(mongos); - const configsvrPrimary = new Mongo(topology.configsvr.primary); - const idx = reshardCollectionJoinedFailPointsList.findIndex(fp => fp.conn.host === - configsvrPrimary.host); - reshardCollectionJoinedFailPointsList[idx].wait(); - - // Wait for secondaries to recover and catchup with primary before turning off the - // failpoints as a replication roll back can disconnect the test client. - const configRS = reshardingTest.getReplSetForShard(reshardingTest.configShardName); - configRS.awaitSecondaryNodes(); - configRS.awaitReplication(); - reshardCollectionJoinedFailPointsList.forEach( - fp => reshardingTest.retryOnceOnNetworkError(fp.off)); + reshardingTest.stepUpNewPrimaryOnShard(reshardingTest.configShardName); shardsvrAbortReshardCollectionFailpoint.off(); }, }); diff --git a/jstests/sharding/resharding_feature_flagging.js b/jstests/sharding/resharding_feature_flagging.js index 8b023168865..6e964a2237c 100644 --- a/jstests/sharding/resharding_feature_flagging.js +++ b/jstests/sharding/resharding_feature_flagging.js @@ -46,7 +46,7 @@ assert.commandFailedWithCode( const serverStatusCmd = ({serverStatus: 1, shardingStatistics: 1}); let res = assert.commandWorked(configPrimary.adminCommand(serverStatusCmd)); -assert(!res.shardingStatistics.hasOwnProperty("resharding"), res.shardingStatistics); +assert(!res.hasOwnProperty("shardingStatistics"), res.shardingStatistics); const shardPrimary = st.shard0.rs.getPrimary(); res = assert.commandWorked(shardPrimary.adminCommand(serverStatusCmd)); diff --git a/jstests/sharding/resharding_interrupt_before_create_state_machine.js b/jstests/sharding/resharding_interrupt_before_create_state_machine.js deleted file mode 100644 index 8b18d498ff2..00000000000 --- a/jstests/sharding/resharding_interrupt_before_create_state_machine.js +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Test that reshardCollection does not hang if its opCtx is interrupted between inserting the state - * document for its state machine and starting that state machine. See SERVER-74647. - */ -(function() { -"use strict"; -load("jstests/sharding/libs/resharding_test_fixture.js"); -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/discover_topology.js"); - -const sourceNs = "reshardingDb.coll"; - -const reshardingTest = new ReshardingTest(); -reshardingTest.setup(); - -const donorShardNames = reshardingTest.donorShardNames; -const recipientShardNames = reshardingTest.recipientShardNames; - -const inputCollection = reshardingTest.createShardedCollection({ - ns: sourceNs, - shardKeyPattern: {oldKey: 1}, - chunks: [ - {min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}, - ], -}); - -const mongos = inputCollection.getMongo(); -const topology = DiscoverTopology.findConnectedNodes(mongos); -const donorPrimary = new Mongo(topology.shards[donorShardNames[0]].primary); - -const failpoint = - configureFailPoint(donorPrimary, "reshardingInterruptAfterInsertStateMachineDocument"); - -reshardingTest.withReshardingInBackground({ - newShardKeyPattern: {newKey: 1}, - newChunks: [ - {min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}, - ] -}, - () => { - failpoint.wait(); - failpoint.off(); - }); - -reshardingTest.teardown(); -})(); diff --git a/jstests/sharding/resharding_large_number_of_initial_chunks.js b/jstests/sharding/resharding_large_number_of_initial_chunks.js index 0bda116b9c7..798131c0fb8 100644 --- a/jstests/sharding/resharding_large_number_of_initial_chunks.js +++ b/jstests/sharding/resharding_large_number_of_initial_chunks.js @@ -1,6 +1,6 @@ /** - * Tests that resharding can complete successfully when it has a large number - * of chunks being created during the process. + * Tests that resharding can complete successfully when the original collection has a large number + * of chunks. * * @tags: [ * uses_atclustertime, @@ -29,42 +29,41 @@ const kDbName = 'db'; const collName = 'foo'; const ns = kDbName + '.' + collName; const mongos = st.s; -const shard0 = st.shard0.shardName; -const shard1 = st.shard1.shardName; assert.commandWorked(mongos.adminCommand({enableSharding: kDbName})); assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {oldKey: 1}})); -let nChunks = 100000; -let newChunks = []; +let nZones = 10000; +let zones = []; +let shard0Zones = []; +let shard1Zones = []; +for (let i = 0; i < nZones; i++) { + let zoneName = "zone" + i; + zones.push({zone: zoneName, min: {"newKey": i}, max: {"newKey": i + 1}}); -newChunks.push({min: {newKey: MinKey}, max: {newKey: 0}, recipientShardId: shard0}); -for (let i = 0; i < nChunks; i++) { if (i % 2 == 0) { - newChunks.push({min: {newKey: i}, max: {newKey: i + 1}, recipientShardId: shard0}); + shard0Zones.push(zoneName); } else { - newChunks.push({min: {newKey: i}, max: {newKey: i + 1}, recipientShardId: shard1}); + shard1Zones.push(zoneName); } } -newChunks.push({min: {newKey: nChunks}, max: {newKey: MaxKey}, recipientShardId: shard1}); + +jsTestLog("Updating First Zone"); +assert.commandWorked( + mongos.getDB("config").shards.update({_id: st.shard0.shardName}, {$set: {tags: shard0Zones}})); +jsTestLog("Updating First Zone"); +assert.commandWorked( + mongos.getDB("config").shards.update({_id: st.shard1.shardName}, {$set: {tags: shard1Zones}})); jsTestLog("Resharding Collection"); -assert.commandWorked(mongos.adminCommand( - {reshardCollection: ns, key: {newKey: 1}, _presetReshardedChunks: newChunks})); +assert.commandWorked(mongos.adminCommand({reshardCollection: ns, key: {newKey: 1}, zones: zones})); -// Assert that the correct number of chunks documents exist after resharding 'db.foo'. -// There should be two more chunks docs to cover the ranges -// {newKey: minKey -> newKey : 0} and {newKey: nChunks -> newKey : maxKey} -assert.eq(findChunksUtil.countChunksForNs(mongos.getDB("config"), ns), nChunks + 2); +// Assert that the correct number of zones and chunks documents exist after resharding 'db.foo'. +// There should be two more chunks docs than zones docs created to cover the ranges +// {newKey: minKey -> newKey : 0} and {newKey: nZones -> newKey : maxKey} which are not associated +// with a zone. +assert.eq(mongos.getDB("config").tags.find({ns: ns}).itcount(), nZones); +assert.eq(findChunksUtil.countChunksForNs(mongos.getDB("config"), ns), nZones + 2); -// check_orphans_are_deleted.js is skipped because it takes 1 minute to run on an optimized build -// and this test doesn't insert any data for there to be unowned documents anyway. -TestData.skipCheckOrphans = true; -// check_uuids_consistent_across_cluster.js is skipped because it takes nearly 1 minute to run on an -// optimized build. -TestData.skipCheckingUUIDsConsistentAcrossCluster = true; -// check_routing_table_consistency.js is skipped because its $group + $lookup aggregation over the -// config.chunks documents exceeds 100MB and fails. -TestData.skipCheckRoutingTableConsistency = true; st.stop(); })(); diff --git a/jstests/sharding/resharding_metrics.js b/jstests/sharding/resharding_metrics.js index 7e1d3a830a2..c4cedb11711 100644 --- a/jstests/sharding/resharding_metrics.js +++ b/jstests/sharding/resharding_metrics.js @@ -50,7 +50,8 @@ function testMetricsArePresent(mongo, expectedMetrics, minOplogEntriesFetchedAnd function verifyStatsMissing(mongo) { const stats = mongo.getDB('admin').serverStatus({}); - assert(!stats.shardingStatistics.hasOwnProperty('resharding'), + assert(!stats.hasOwnProperty('shardingStatistics') || + !stats.shardingStatistics.hasOwnProperty('resharding'), `Resharding section not expected in ${tojson(stats)}`); } diff --git a/jstests/sharding/resharding_nonblocking_coordinator_rebuild.js b/jstests/sharding/resharding_nonblocking_coordinator_rebuild.js index 2ee6c76aaf1..dac1afc0014 100644 --- a/jstests/sharding/resharding_nonblocking_coordinator_rebuild.js +++ b/jstests/sharding/resharding_nonblocking_coordinator_rebuild.js @@ -109,7 +109,19 @@ reshardingTest.withReshardingInBackground( } }, { - expectedErrorCode: 5356800, + // As a result of the elections intentionally triggered on the config server replica sets, + // the primary shard of the database may retry the _configsvrReshardCollection command. It + // is possible for the resharding operation from the first _configsvrReshardCollection + // command to have entirely finished executing to the point of removing the coordinator + // state document. A retry of the _configsvrReshardCollection command in this situation will + // lead to a second resharding operation to run. The second resharding operation will have + // the duplicate documents cloned by the ReshardingCollectionCloner rather than applied by + // the ReshardingOplogApplier as intended. This results in the reshardCollection command + // failing with a DuplicateKey error rather than the error code for the stash collections + // being non-empty. The recipient must have been able to successfully update its state to + // "applying" in the first resharding operation even when the ReshardingCoordinatorService + // had yet to be rebuilt so we accept DuplicateKey as an error too. + expectedErrorCode: [5356800, ErrorCodes.DuplicateKey], }); reshardingTest.teardown(); diff --git a/jstests/sharding/resharding_prohibited_commands.js b/jstests/sharding/resharding_prohibited_commands.js index 9f7d73b57c3..d06a9561d2a 100644 --- a/jstests/sharding/resharding_prohibited_commands.js +++ b/jstests/sharding/resharding_prohibited_commands.js @@ -118,7 +118,6 @@ const waitUntilReshardingInitializedOnDonor = () => { * @param {Function} config.setup * @param {AfterReshardingCallback} afterReshardingFn */ - const withReshardingInBackground = (duringReshardingFn, {setup = () => {}, expectedErrorCode, afterReshardingFn = () => {}} = {}) => { @@ -133,34 +132,22 @@ const withReshardingInBackground = }, duringReshardingFn, {expectedErrorCode: expectedErrorCode, afterReshardingFn: afterReshardingFn}); + assertCommandsSucceedAfterReshardingOpFinishes(mongos.getDB(databaseName)); assert.commandWorked(sourceCollection.dropIndex(indexCreatedByTest)); }; // Tests that the prohibited commands work if the resharding operation is aborted. -let awaitAbort; withReshardingInBackground(() => { waitUntilReshardingInitializedOnDonor(); - assert.neq(null, - mongos.getCollection("config.reshardingOperations").findOne({ns: sourceNamespace})); - awaitAbort = startParallelShell(funWithArgs(function(sourceNamespace) { - db.adminCommand({abortReshardCollection: sourceNamespace}); - }, sourceNamespace), mongos.port); - // Wait for the coordinator to remove coordinator document from config.reshardingOperations - // as a result of the recipients and donors transitioning to done due to abort. - assert.soon(() => { - const coordinatorDoc = - mongos.getCollection("config.reshardingOperations").findOne({ns: sourceNamespace}); - return coordinatorDoc === null || coordinatorDoc.state === "aborting"; - }); + assert.commandWorked(mongos.adminCommand({abortReshardCollection: sourceNamespace})); }, { expectedErrorCode: ErrorCodes.ReshardCollectionAborted, }); -awaitAbort(); // Tests that the prohibited commands succeed if the resharding operation succeeds. During the -// operation it makes sure that the prohibited commands are rejected during the resharding +// operation it makes sures that the prohibited commands are rejected during the resharding // operation. withReshardingInBackground(() => { waitUntilReshardingInitializedOnDonor(); diff --git a/jstests/sharding/resharding_temp_ns_routing_info_unsharded.js b/jstests/sharding/resharding_temp_ns_routing_info_unsharded.js deleted file mode 100644 index d7616222920..00000000000 --- a/jstests/sharding/resharding_temp_ns_routing_info_unsharded.js +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Tests that write operations on the collection being resharded succeed even when the routing - * information for the associated temporary resharding collection is stale. - */ -(function() { -"use strict"; - -load("jstests/libs/discover_topology.js"); -load("jstests/libs/fail_point_util.js"); -load("jstests/sharding/libs/resharding_test_fixture.js"); - -const reshardingTest = new ReshardingTest({reshardInPlace: false}); -reshardingTest.setup(); - -const donorShardNames = reshardingTest.donorShardNames; -const recipientShardNames = reshardingTest.recipientShardNames; - -const sourceCollection = reshardingTest.createShardedCollection({ - ns: "reshardingDb.coll", - shardKeyPattern: {oldKey: 1}, - chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}], - primaryShardName: recipientShardNames[0], -}); - -const mongos = sourceCollection.getMongo(); -const topology = DiscoverTopology.findConnectedNodes(mongos); -const donor = new Mongo(topology.shards[donorShardNames[0]].primary); - -const reshardingPauseDonorBeforeCatalogCacheRefreshFailpoint = - configureFailPoint(donor, "reshardingPauseDonorBeforeCatalogCacheRefresh"); - -// We trigger a refresh to make the catalog cache track the routing info for the temporary -// resharding namespace as unsharded because the collection won't exist yet. -assert.commandWorked(donor.adminCommand({_flushRoutingTableCacheUpdates: reshardingTest.tempNs})); - -reshardingTest.withReshardingInBackground( // - { - newShardKeyPattern: {newKey: 1}, - newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}], - }, - () => { - reshardingPauseDonorBeforeCatalogCacheRefreshFailpoint.wait(); - - assert.commandWorked(sourceCollection.insert({_id: 0, oldKey: 5, newKey: 15})); - reshardingPauseDonorBeforeCatalogCacheRefreshFailpoint.off(); - }); - -reshardingTest.teardown(); -})(); diff --git a/jstests/sharding/resharding_update_tag_zones.js b/jstests/sharding/resharding_update_tag_zones.js deleted file mode 100644 index dd59a97b8a7..00000000000 --- a/jstests/sharding/resharding_update_tag_zones.js +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Testing that config.tags are correctly updated after resharding hashed shard key with zones. - */ - -(function() { -"use strict"; - -const st = new ShardingTest({ - shard: 2, - configOptions: - {setParameter: - {'reshardingCriticalSectionTimeoutMillis': 24 * 60 * 60 * 1000 /* 1 day */}} -}); -const dbName = "testDb"; -const collName = "testColl"; -const ns = dbName + "." + collName; - -// Enable sharding on the test DB and ensure its primary is st.shard0.shardName. -assert.commandWorked(st.s.adminCommand({enablesharding: dbName})); -st.ensurePrimaryShard(dbName, st.shard0.shardName); -assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {oldKey: "hashed"}})); - -const existingZoneName = 'x1'; -assert.commandWorked( - st.s.adminCommand({addShardToZone: st.shard1.shardName, zone: existingZoneName})); - -assert.commandWorked(st.s.adminCommand({ - updateZoneKeyRange: ns, - min: {oldKey: NumberLong("4470791281878691347")}, - max: {oldKey: NumberLong("7766103514953448109")}, - zone: existingZoneName -})); - -assert.commandWorked(st.s.adminCommand({ - reshardCollection: ns, - key: {oldKey: 1}, - unique: false, - collation: {locale: 'simple'}, - zones: [{ - zone: existingZoneName, - min: {oldKey: NumberLong("4470791281878691346")}, - max: {oldKey: NumberLong("7766103514953448108")} - }], - numInitialChunks: 2, -})); - -// Find the tags docs. -var configDB = st.s.getDB("config"); -let tags = configDB.tags.find({}).toArray(); - -// Assert only one tag doc is present and zone ranges are correct. -assert.eq(1, configDB.tags.countDocuments({})); -assert.eq({oldKey: NumberLong("4470791281878691346")}, tags[0].min); -assert.eq({oldKey: NumberLong("7766103514953448108")}, tags[0].max); -assert.eq(existingZoneName, tags[0].tag); - -st.stop(); -})(); diff --git a/jstests/sharding/resharding_update_tag_zones_large.js b/jstests/sharding/resharding_update_tag_zones_large.js deleted file mode 100644 index 03f7960d711..00000000000 --- a/jstests/sharding/resharding_update_tag_zones_large.js +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Testing that the reshardCollection command aborts correctly when the transaction for updating - * the persistent state (e.g. config.collections and config.tags) in the resharding commit phase - * fails with a TransactionTooLargeForCache error. - */ - -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); - -function assertEqualObj(lhs, rhs, keysToIgnore) { - assert.eq(Object.keys(lhs).length, Object.keys(lhs).length, {lhs, rhs}); - for (let key in rhs) { - if (keysToIgnore && keysToIgnore.has(key)) { - continue; - } - - const value = rhs[key]; - if (typeof value === 'object') { - assertEqualObj(lhs[key], rhs[key], keysToIgnore); - } else { - assert.eq(lhs[key], rhs[key], {key, actual: lhs, expected: rhs}); - } - } -} - -const st = new ShardingTest({ - shard: 2, - configOptions: - {setParameter: - {'reshardingCriticalSectionTimeoutMillis': 24 * 60 * 60 * 1000 /* 1 day */}} -}); - -const configRSPrimary = st.configRS.getPrimary(); - -const dbName = "testDb"; -const collName = "testColl"; -const ns = dbName + "." + collName; - -const configDB = st.s.getDB("config"); -const collectionsColl = configDB.getCollection("collections"); -const chunksColl = configDB.getCollection("chunks"); -const tagsColl = configDB.getCollection("tags"); - -assert.commandWorked(st.s.adminCommand({enablesharding: dbName})); -st.ensurePrimaryShard(dbName, st.shard0.shardName); -assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {skey: "hashed"}})); - -const zoneName = "testZone"; -assert.commandWorked(st.s.adminCommand({addShardToZone: st.shard0.shardName, zone: zoneName})); - -const oldZone = { - tag: zoneName, - min: {skey: NumberLong("4470791281878691347")}, - max: {skey: NumberLong("7766103514953448109")} -}; -assert.commandWorked(st.s.adminCommand( - {updateZoneKeyRange: ns, min: oldZone.min, max: oldZone.max, zone: oldZone.tag})); - -const collBefore = collectionsColl.findOne({_id: ns}); -assert.neq(collBefore, null); -const chunksBefore = chunksColl.find({uuid: collBefore.uuid}).sort({lastmod: -1}).toArray(); -assert.gte(chunksBefore.length, 1, chunksBefore); -const tagsBefore = tagsColl.find({ns}).toArray(); -assert.gte(tagsBefore.length, 1, tagsBefore); - -const reshardingFunc = (mongosHost, ns, zoneName) => { - const mongos = new Mongo(mongosHost); - const newZone = { - tag: zoneName, - min: {skey: NumberLong("4470791281878691346")}, - max: {skey: NumberLong("7766103514953448108")} - }; - jsTest.log("Start resharding"); - const reshardingRes = mongos.adminCommand({ - reshardCollection: ns, - key: {skey: 1}, - unique: false, - collation: {locale: 'simple'}, - zones: [{zone: newZone.tag, min: newZone.min, max: newZone.max}], - numInitialChunks: 2, - }); - jsTest.log("Finished resharding"); - return reshardingRes; -}; -let reshardingThread = new Thread(reshardingFunc, st.s.host, ns, zoneName); - -const persistFp = - configureFailPoint(configRSPrimary, "reshardingPauseCoordinatorBeforeDecisionPersisted"); -reshardingThread.start(); -persistFp.wait(); - -const commitFp = configureFailPoint(configRSPrimary, - "failCommand", - { - failCommands: ["commitTransaction"], - failInternalCommands: true, - failLocalClients: true, - errorCode: ErrorCodes.TransactionTooLargeForCache, - }, - {times: 1}); -persistFp.off(); -commitFp.wait(); -commitFp.off(); -const reshardingRes = reshardingThread.returnData(); - -assert.commandFailedWithCode(reshardingRes, ErrorCodes.TransactionTooLargeForCache); - -const collAfter = collectionsColl.findOne({_id: ns}); -assert.neq(collAfter, null); -const chunksAfter = chunksColl.find({uuid: collAfter.uuid}).sort({lastmod: -1}).toArray(); -const tagsAfter = tagsColl.find({ns}).toArray(); - -jsTest.log( - "Verify that the collection metadata remains the same since the resharding operation failed."); - -assertEqualObj(collBefore, collAfter); - -assert.eq(chunksBefore.length, chunksAfter.length, {chunksBefore, chunksAfter}); -for (let i = 0; i < chunksAfter.length; i++) { - // Ignore "lastmod" when verifying the newest chunk because resharding bumps the minor version - // of the newest chunk whenever it goes through a state transition. - assertEqualObj(chunksBefore[i], chunksAfter[i], new Set(i == 0 ? ["lastmod"] : [])); -} - -assert.eq(tagsBefore.length, tagsAfter.length, {tagsBefore, tagsAfter}); -for (let i = 0; i < tagsAfter.length; i++) { - assertEqualObj(tagsBefore[i], tagsAfter[i]); -} - -st.stop(); -})(); diff --git a/jstests/sharding/retryable_write_error_labels.js b/jstests/sharding/retryable_write_error_labels.js index f4cf2db1a8e..23b0ff3c831 100644 --- a/jstests/sharding/retryable_write_error_labels.js +++ b/jstests/sharding/retryable_write_error_labels.js @@ -137,10 +137,6 @@ function testMongodError(errorCode, isWCError) { function testMongosError() { const shard0Primary = st.rs0.getPrimary(); - // Insert initial documents used by the test. - const docs = [{k: 0, x: 0}, {k: 1, x: 1}]; - assert.commandWorked(shard0Primary.getDB(dbName)[collName].insert(docs)); - // Test retryable writes. jsTestLog("Retryable write should return mongos shutdown error with RetryableWriteError label"); @@ -215,7 +211,7 @@ function testMongosError() { const sessionDb = session.getDatabase(dbName); const sessionColl = sessionDb.getCollection(collName); session.startTransaction(); - assert.commandWorked(sessionColl.update({k: 0}, {$inc: {x: 1}})); + assert.commandWorked(sessionColl.update({}, {$inc: {x: 1}})); return sessionDb.adminCommand({ commitTransaction: 1, txnNumber: NumberLong(session.getTxnNumber_forTesting()), @@ -252,7 +248,7 @@ function testMongosError() { const sessionDb = session.getDatabase(dbName); const sessionColl = sessionDb.getCollection(collName); session.startTransaction(); - assert.commandWorked(sessionColl.update({k: 1}, {$inc: {x: 1}})); + assert.commandWorked(sessionColl.update({}, {$inc: {x: 1}})); return sessionDb.adminCommand({ abortTransaction: 1, txnNumber: NumberLong(session.getTxnNumber_forTesting()), diff --git a/jstests/sharding/run_restore.js b/jstests/sharding/run_restore.js index b5dd22e10f9..3dfbd0b5b38 100644 --- a/jstests/sharding/run_restore.js +++ b/jstests/sharding/run_restore.js @@ -12,12 +12,6 @@ load("jstests/libs/feature_flag_util.js"); -// Because we restart nodes in standalone mode, it's possible for fast count, which doesn't -// discriminate between majority committed data and locally committed data, and the true count, -// which only includes majority committed data on standalones, to diverge. Therefore skip -// validating fast count. -TestData.skipEnforceFastCountOnValidate = true; - const s = new ShardingTest({name: "runRestore", shards: 2, mongos: 1, config: 1, other: {chunkSize: 1}}); diff --git a/jstests/sharding/run_restore_unsharded.js b/jstests/sharding/run_restore_unsharded.js deleted file mode 100644 index 30591a0bbb5..00000000000 --- a/jstests/sharding/run_restore_unsharded.js +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Tests that the "_configsvrRunRestore" command restores databases with unsharded collections - * referenced in the "local.system.collections_to_restore" collection. - * - * @tags: [ - * requires_persistence, - * ] - */ -(function() { -"use strict"; - -load("jstests/libs/feature_flag_util.js"); - -// Because we restart nodes in standalone mode, it's possible for fast count, which doesn't -// discriminate between majority committed data and locally committed data, and the true count, -// which only includes majority committed data on standalones, to diverge. Therefore skip -// validating fast count. -TestData.skipEnforceFastCountOnValidate = true; - -const s = - new ShardingTest({name: "runRestore", shards: 2, mongos: 1, config: 1, other: {chunkSize: 1}}); - -let mongos = s.s0; -let db = s.getDB("test"); -if (!FeatureFlagUtil.isEnabled(s.configRS.getPrimary().getDB("test"), "SelectiveBackup")) { - jsTestLog("Skipping as featureFlagSelectiveBackup is not enabled"); - s.stop(); - return; -} - -s.adminCommand({enablesharding: "test"}); -s.ensurePrimaryShard("test", s.shard0.shardName); - -// Create an unsharded collection. -assert.commandWorked(db.createCollection("a")); -const collUUID = - s.shard0.getDB("test").runCommand({listCollections: 1}).cursor.firstBatch[0].info.uuid; - -// Only sharded collections appear in config.collections -assert.eq(0, mongos.getDB("config").getCollection("collections").find({_id: "test.a"}).count()); - -assert.eq(1, mongos.getDB("config").getCollection("locks").find({_id: "test"}).count()); -assert.eq(1, mongos.getDB("config").getCollection("databases").find({_id: "test"}).count()); - -s.stop({noCleanData: true}); - -const configDbPath = s.c0.dbpath; - -// Start the config server in standalone restore mode. -let conn = MongoRunner.runMongod({noCleanData: true, dbpath: configDbPath, restore: ""}); -assert(conn); - -assert.commandWorked(conn.getDB("admin").runCommand({setParameter: 1, logLevel: 1})); - -// Create the "local.system.collections_to_restore" collection and insert "test.a". -assert.commandWorked(conn.getDB("local").createCollection("system.collections_to_restore")); -assert.commandWorked(conn.getDB("local").getCollection("system.collections_to_restore").insert({ - ns: "test.a", - uuid: collUUID -})); - -assert.commandWorked(conn.getDB("admin").runCommand({_configsvrRunRestore: 1})); - -// Only sharded collections appear in config.collections -assert.eq(0, conn.getDB("config").getCollection("collections").find({_id: "test.a"}).count()); - -let locks = conn.getDB("config").getCollection("locks").find({_id: "test"}).toArray(); -assert.eq(1, locks.length); -assert.eq(0, locks[0].state); // State::UNLOCKED -assert.eq(1, conn.getDB("config").getCollection("databases").find({_id: "test"}).count()); - -MongoRunner.stopMongod(conn); -}()); diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js index 6ace4437983..d6ad842361f 100644 --- a/jstests/sharding/safe_secondary_reads_drop_recreate.js +++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js @@ -72,7 +72,6 @@ let testCases = { _configsvrShardCollection: {skip: "primary only"}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS _configsvrUpdateZoneKeyRange: {skip: "primary only"}, - _dropConnectionsToMongot: {skip: "internal command"}, _flushReshardingStateChange: {skip: "does not return user data"}, _flushRoutingTableCacheUpdates: {skip: "does not return user data"}, _flushRoutingTableCacheUpdatesWithWriteConcern: {skip: "does not return user data"}, @@ -83,7 +82,6 @@ let testCases = { _killOperations: {skip: "does not return user data"}, _mergeAuthzCollections: {skip: "primary only"}, _migrateClone: {skip: "primary only"}, - _mongotConnPoolStats: {skip: "internal command"}, _shardsvrCompactStructuredEncryptionData: {skip: "primary only"}, _shardsvrMovePrimary: {skip: "primary only"}, _shardsvrMoveRange: {skip: "primary only"}, @@ -164,7 +162,6 @@ let testCases = { create: {skip: "primary only"}, createIndexes: {skip: "primary only"}, createRole: {skip: "primary only"}, - createSearchIndexes: {skip: "primary only"}, createUser: {skip: "primary only"}, currentOp: {skip: "does not return user data"}, dataSize: {skip: "does not return user data"}, @@ -191,7 +188,6 @@ let testCases = { dropDatabase: {skip: "primary only"}, dropIndexes: {skip: "primary only"}, dropRole: {skip: "primary only"}, - dropSearchIndex: {skip: "primary only"}, dropUser: {skip: "primary only"}, echo: {skip: "does not return user data"}, emptycapped: {skip: "primary only"}, @@ -250,7 +246,6 @@ let testCases = { listCommands: {skip: "does not return user data"}, listDatabases: {skip: "primary only"}, listIndexes: {skip: "primary only"}, - listSearchIndexes: {skip: "primary only"}, listShards: {skip: "does not return user data"}, lockInfo: {skip: "primary only"}, logApplicationMessage: {skip: "primary only"}, @@ -334,7 +329,7 @@ let testCases = { setDefaultRWConcern: {skip: "primary only"}, setIndexCommitQuorum: {skip: "primary only"}, setFeatureCompatibilityVersion: {skip: "primary only"}, - setProfilingFilterGlobally: {skip: "does not return user data"}, + setFreeMonitoring: {skip: "primary only"}, setParameter: {skip: "does not return user data"}, setShardVersion: {skip: "does not return user data"}, setClusterParameter: {skip: "does not return user data"}, @@ -359,7 +354,6 @@ let testCases = { top: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, - updateSearchIndex: {skip: "primary only"}, updateUser: {skip: "primary only"}, updateZoneKeyRange: {skip: "primary only"}, usersInfo: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js index 27c3ca2cb0e..3e01be669a4 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js @@ -82,7 +82,6 @@ let testCases = { _configsvrShardCollection: {skip: "primary only"}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS _configsvrUpdateZoneKeyRange: {skip: "primary only"}, - _dropConnectionsToMongot: {skip: "internal command"}, _flushReshardingStateChange: {skip: "does not return user data"}, _flushRoutingTableCacheUpdates: {skip: "does not return user data"}, _flushRoutingTableCacheUpdatesWithWriteConcern: {skip: "does not return user data"}, @@ -93,7 +92,6 @@ let testCases = { _killOperations: {skip: "does not return user data"}, _mergeAuthzCollections: {skip: "primary only"}, _migrateClone: {skip: "primary only"}, - _mongotConnPoolStats: {skip: "internal command"}, _shardsvrCompactStructuredEncryptionData: {skip: "primary only"}, _shardsvrMovePrimary: {skip: "primary only"}, _shardsvrMoveRange: {skip: "primary only"}, @@ -186,7 +184,6 @@ let testCases = { create: {skip: "primary only"}, createIndexes: {skip: "primary only"}, createRole: {skip: "primary only"}, - createSearchIndexes: {skip: "does not return user data"}, createUser: {skip: "primary only"}, currentOp: {skip: "does not return user data"}, dataSize: {skip: "does not return user data"}, @@ -218,7 +215,6 @@ let testCases = { dropDatabase: {skip: "primary only"}, dropIndexes: {skip: "primary only"}, dropRole: {skip: "primary only"}, - dropSearchIndex: {skip: "does not return user data"}, dropUser: {skip: "primary only"}, echo: {skip: "does not return user data"}, emptycapped: {skip: "primary only"}, @@ -283,7 +279,6 @@ let testCases = { listCommands: {skip: "does not return user data"}, listDatabases: {skip: "primary only"}, listIndexes: {skip: "primary only"}, - listSearchIndexes: {skip: "does not return user data"}, listShards: {skip: "does not return user data"}, lockInfo: {skip: "primary only"}, logApplicationMessage: {skip: "primary only"}, @@ -405,7 +400,7 @@ let testCases = { setDefaultRWConcern: {skip: "primary only"}, setIndexCommitQuorum: {skip: "primary only"}, setFeatureCompatibilityVersion: {skip: "primary only"}, - setProfilingFilterGlobally: {skip: "does not return user data"}, + setFreeMonitoring: {skip: "primary only"}, setParameter: {skip: "does not return user data"}, setShardVersion: {skip: "does not return user data"}, setClusterParameter: {skip: "does not return user data"}, @@ -430,7 +425,6 @@ let testCases = { top: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, - updateSearchIndex: {skip: "does not return user data"}, updateUser: {skip: "primary only"}, updateZoneKeyRange: {skip: "primary only"}, usersInfo: {skip: "primary only"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js index 1199327750f..64ea7271f2e 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js @@ -74,7 +74,6 @@ let testCases = { _configsvrShardCollection: {skip: "primary only"}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS _configsvrUpdateZoneKeyRange: {skip: "primary only"}, - _dropConnectionsToMongot: {skip: "does not return user data"}, _flushReshardingStateChange: {skip: "does not return user data"}, _flushRoutingTableCacheUpdates: {skip: "does not return user data"}, _flushRoutingTableCacheUpdatesWithWriteConcern: {skip: "does not return user data"}, @@ -85,7 +84,6 @@ let testCases = { _killOperations: {skip: "does not return user data"}, _mergeAuthzCollections: {skip: "primary only"}, _migrateClone: {skip: "primary only"}, - _mongotConnPoolStats: {skip: "internal command"}, _shardsvrCompactStructuredEncryptionData: {skip: "primary only"}, _shardsvrMovePrimary: {skip: "primary only"}, _shardsvrMoveRange: {skip: "primary only"}, @@ -168,7 +166,6 @@ let testCases = { create: {skip: "primary only"}, createIndexes: {skip: "primary only"}, createRole: {skip: "primary only"}, - createSearchIndexes: {skip: "does not return user data"}, createUser: {skip: "primary only"}, currentOp: {skip: "does not return user data"}, dataSize: {skip: "does not return user data"}, @@ -195,7 +192,6 @@ let testCases = { dropDatabase: {skip: "primary only"}, dropIndexes: {skip: "primary only"}, dropRole: {skip: "primary only"}, - dropSearchIndex: {skip: "does not return user data"}, dropUser: {skip: "primary only"}, echo: {skip: "does not return user data"}, emptycapped: {skip: "primary only"}, @@ -255,7 +251,6 @@ let testCases = { listCommands: {skip: "does not return user data"}, listDatabases: {skip: "primary only"}, listIndexes: {skip: "primary only"}, - listSearchIndexes: {skip: "does not return user data"}, listShards: {skip: "does not return user data"}, lockInfo: {skip: "primary only"}, logApplicationMessage: {skip: "primary only"}, @@ -341,7 +336,7 @@ let testCases = { setDefaultRWConcern: {skip: "primary only"}, setIndexCommitQuorum: {skip: "primary only"}, setFeatureCompatibilityVersion: {skip: "primary only"}, - setProfilingFilterGlobally: {skip: "does not return user data"}, + setFreeMonitoring: {skip: "primary only"}, setParameter: {skip: "does not return user data"}, setShardVersion: {skip: "does not return user data"}, setClusterParameter: {skip: "does not return user data"}, @@ -366,7 +361,6 @@ let testCases = { top: {skip: "does not return user data"}, update: {skip: "primary only"}, updateRole: {skip: "primary only"}, - updateSearchIndex: {skip: "does not return user data"}, updateUser: {skip: "primary only"}, updateZoneKeyRange: {skip: "primary only"}, usersInfo: {skip: "primary only"}, diff --git a/jstests/sharding/server_status_crud_metrics.js b/jstests/sharding/server_status_crud_metrics.js index 99821f78b4c..4f83d7e4b6e 100644 --- a/jstests/sharding/server_status_crud_metrics.js +++ b/jstests/sharding/server_status_crud_metrics.js @@ -26,14 +26,15 @@ assert.commandWorked(unshardedColl.insert({x: 1, _id: 1})); // Verification for 'updateOneOpStyleBroadcastWithExactIDCount' metric. // Should increment the metric as the update cannot target single shard and are {multi:false}. -assert.commandWorked(testColl.update({_id: "missing"}, {$set: {a: 1}}, {multi: false})); -assert.commandWorked(testColl.update({_id: 1}, {$set: {a: 2}}, {multi: false})); +assert.commandWorked(testDB.coll.update({_id: "missing"}, {$set: {a: 1}}, {multi: false})); +assert.commandWorked(testDB.coll.update({_id: 1}, {$set: {a: 2}}, {multi: false})); // Should increment the metric because we broadcast by _id, even though the update subsequently // fails on the individual shard. -assert.commandFailedWithCode(testColl.update({_id: 1}, {$set: {x: 2}}, {multi: false}), 31025); -assert.commandFailedWithCode(testColl.update({_id: 1}, {$set: {x: 12}, $hello: 1}, {multi: false}), - ErrorCodes.FailedToParse); +assert.commandFailedWithCode(testDB.coll.update({_id: 1}, {$set: {x: 2}}, {multi: false}), 31025); +assert.commandFailedWithCode( + testDB.coll.update({_id: 1}, {$set: {x: 12}, $hello: 1}, {multi: false}), + ErrorCodes.FailedToParse); let mongosServerStatus = testDB.adminCommand({serverStatus: 1}); @@ -41,21 +42,21 @@ let mongosServerStatus = testDB.adminCommand({serverStatus: 1}); assert.eq(4, mongosServerStatus.metrics.query.updateOneOpStyleBroadcastWithExactIDCount); // Shouldn't increment the metric when {multi:true}. -assert.commandWorked(testColl.update({_id: 1}, {$set: {a: 3}}, {multi: true})); -assert.commandWorked(testColl.update({}, {$set: {a: 3}}, {multi: true})); +assert.commandWorked(testDB.coll.update({_id: 1}, {$set: {a: 3}}, {multi: true})); +assert.commandWorked(testDB.coll.update({}, {$set: {a: 3}}, {multi: true})); // Shouldn't increment the metric when update can target single shard. -assert.commandWorked(testColl.update({x: 11}, {$set: {a: 2}}, {multi: false})); -assert.commandWorked(testColl.update({x: 1}, {$set: {a: 2}}, {multi: false})); +assert.commandWorked(testDB.coll.update({x: 11}, {$set: {a: 2}}, {multi: false})); +assert.commandWorked(testDB.coll.update({x: 1}, {$set: {a: 2}}, {multi: false})); // Shouldn't increment the metric for replacement style updates. -assert.commandWorked(testColl.update({_id: 1}, {x: 1, a: 2})); -assert.commandWorked(testColl.update({x: 1}, {x: 1, a: 1})); +assert.commandWorked(testDB.coll.update({_id: 1}, {x: 1, a: 2})); +assert.commandWorked(testDB.coll.update({x: 1}, {x: 1, a: 1})); // Shouldn't increment the metric when routing fails. -assert.commandFailedWithCode(testColl.update({}, {$set: {x: 2}}, {multi: false}), +assert.commandFailedWithCode(testDB.coll.update({}, {$set: {x: 2}}, {multi: false}), ErrorCodes.InvalidOptions); -assert.commandFailedWithCode(testColl.update({_id: 1}, {$set: {x: 2}}, {upsert: true}), +assert.commandFailedWithCode(testDB.coll.update({_id: 1}, {$set: {x: 2}}, {upsert: true}), ErrorCodes.ShardKeyNotFound); // Shouldn't increment the metrics for unsharded collection. @@ -64,7 +65,7 @@ assert.commandWorked(unshardedColl.update({_id: 1}, {$set: {a: 2}}, {multi: fals // Shouldn't incement the metrics when query had invalid operator. assert.commandFailedWithCode( - testColl.update({_id: 1, $invalidOperator: 1}, {$set: {a: 2}}, {multi: false}), + testDB.coll.update({_id: 1, $invalidOperator: 1}, {$set: {a: 2}}, {multi: false}), ErrorCodes.BadValue); mongosServerStatus = testDB.adminCommand({serverStatus: 1}); diff --git a/jstests/sharding/shard_collection_basic.js b/jstests/sharding/shard_collection_basic.js index 9c22eff25b0..7fc136dbc8b 100644 --- a/jstests/sharding/shard_collection_basic.js +++ b/jstests/sharding/shard_collection_basic.js @@ -1,7 +1,3 @@ -/** - * Disabled in multiversion, see SERVER-69290 for more details. - * @tags: [multiversion_incompatible] - */ (function() { 'use strict'; diff --git a/jstests/sharding/shard_drain_works_with_chunks_of_any_size.js b/jstests/sharding/shard_drain_works_with_chunks_of_any_size.js deleted file mode 100644 index b9f8aff9460..00000000000 --- a/jstests/sharding/shard_drain_works_with_chunks_of_any_size.js +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Shard a collection with documents spread on 2 shards and then call `removeShard` checking that: - * - Huge non-jumbo chunks are split during draining (moveRange moves off pieces of `chunkSize` MB) - * - Jumbo chunks are moved off (without splitting, since it's not possible) - * - * Regression test for SERVER-76550. - * - * @tags: [ requires_fcv_60 ] - */ - -(function() { -"use strict"; -load("jstests/sharding/libs/find_chunks_util.js"); - -function removeShard(st, shardName, timeout) { - if (timeout == undefined) { - timeout = 5 * 60 * 1000; // 5 minutes - } - - assert.soon(function() { - const res = st.s.adminCommand({removeShard: shardName}); - if (!res.ok && res.code === ErrorCodes.ShardNotFound) { - // If the config server primary steps down right after removing the config.shards doc - // for the shard but before responding with "state": "completed", the mongos would retry - // the _configsvrRemoveShard command against the new config server primary, which would - // not find the removed shard in its ShardRegistry if it has done a ShardRegistry reload - // after the config.shards doc for the shard was removed. This would cause the command - // to fail with ShardNotFound. - return true; - } - assert.commandWorked(res); - return res.state == 'completed'; - }, "failed to remove shard " + shardName + " within " + timeout + "ms", timeout); -} - -const st = new ShardingTest({other: {enableBalancer: false, chunkSize: 1}}); -const mongos = st.s0; -const configDB = st.getDB('config'); - -st.forEachConfigServer((conn) => { - assert.commandWorked(conn.adminCommand({ - configureFailPoint: 'overrideBalanceRoundInterval', - mode: 'alwaysOn', - data: {intervalMs: 100} - })); -}); - -const dbName = 'test'; -const collName = 'collToDrain'; -const ns = dbName + '.' + collName; -const db = st.getDB(dbName); -const coll = db.getCollection(collName); - -// Shard collection with shard0 as db primary -assert.commandWorked( - mongos.adminCommand({enablesharding: dbName, primaryShard: st.shard0.shardName})); -assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {x: 1}})); - -// shard0 owns docs with shard key [MinKey, 0), shard1 owns docs with shard key [0, MaxKey) -assert.commandWorked(st.s.adminCommand( - {moveRange: ns, min: {x: 0}, max: {x: MaxKey}, toShard: st.shard1.shardName})); - -// Insert ~20MB of docs with different shard keys (10MB on shard0 and 10MB on shard1) -// and ~10MB of docs with the same shard key (jumbo chunk) -const big = 'X'.repeat(1024 * 1024); // 1MB -const jumboKey = 100; -var bulk = coll.initializeUnorderedBulkOp(); -for (var i = -10; i < 10; i++) { - bulk.insert({x: i, big: big}); - bulk.insert({x: jumboKey, big: big}); -} -assert.commandWorked(bulk.execute()); - -// Check that there are only 2 big chunks before starting draining -const chunksBeforeDrain = findChunksUtil.findChunksByNs(configDB, ns).toArray(); -assert.eq(2, chunksBeforeDrain.length); - -st.startBalancer(); - -// Remove shard 1 and wait for all chunks to be moved off from it -removeShard(st, st.shard1.shardName); - -// Check that after draining there are 12 chunks on shard0: -// - [MinKey, 0) original chunk on shard 1 -// - [0, 1), [1, 2), ... [8, 9) 1 MB chunks -// - [9, MaxKey) 10MB jumbo chunk -const chunksAfterDrain = - findChunksUtil.findChunksByNs(configDB, ns, {shard: st.shard0.shardName}).toArray(); -assert.eq(12, chunksAfterDrain.length); - -st.stop(); -})(); diff --git a/jstests/sharding/shard_existing.js b/jstests/sharding/shard_existing.js new file mode 100644 index 00000000000..5430b5577fc --- /dev/null +++ b/jstests/sharding/shard_existing.js @@ -0,0 +1,45 @@ +/* + * @tags: [ + * requires_fcv_51, + * ] + */ + +(function() { +'use strict'; + +load("jstests/sharding/libs/find_chunks_util.js"); + +var s = new ShardingTest({name: "shard_existing", shards: 2, mongos: 1, other: {chunkSize: 1}}); +var db = s.getDB("test"); + +var stringSize = 10000; +var numDocs = 2000; + +// we want a lot of data, so lets make a string to cheat :) +var bigString = new Array(stringSize).toString(); +var docSize = Object.bsonsize({_id: numDocs, s: bigString}); +var totalSize = docSize * numDocs; +print("NumDocs: " + numDocs + " DocSize: " + docSize + " TotalSize: " + totalSize); + +var bulk = db.data.initializeUnorderedBulkOp(); +for (var i = 0; i < numDocs; i++) { + bulk.insert({_id: i, s: bigString}); +} +assert.commandWorked(bulk.execute()); + +var avgObjSize = db.data.stats().avgObjSize; +var dataSize = db.data.stats().size; +assert.lte(totalSize, dataSize); + +s.adminCommand({enablesharding: "test"}); +s.ensurePrimaryShard('test', s.shard1.shardName); +var res = s.adminCommand({shardcollection: "test.data", key: {_id: 1}}); +printjson(res); + +// number of chunks should be approx equal to the total data size / chunk size +var numChunks = findChunksUtil.findChunksByNs(s.config, 'test.data').itcount(); +var guess = Math.ceil(dataSize / (1024 * 1024 + avgObjSize)); +assert.lte(Math.abs(numChunks - guess), 2, "not right number of chunks"); + +s.stop(); +})(); diff --git a/jstests/sharding/shard_existing_coll_chunk_count.js b/jstests/sharding/shard_existing_coll_chunk_count.js new file mode 100644 index 00000000000..c53dca8fdf9 --- /dev/null +++ b/jstests/sharding/shard_existing_coll_chunk_count.js @@ -0,0 +1,177 @@ +/** + * This test confirms that after sharding a collection with some pre-existing data, + * the resulting chunks aren't auto-split too aggressively. + * + * @tags: [ + * requires_fcv_51, + * requires_persistence + * ] + */ +(function() { +'use strict'; +load('jstests/sharding/autosplit_include.js'); +load("jstests/sharding/libs/find_chunks_util.js"); + +var s = new ShardingTest({ + name: "shard_existing_coll_chunk_count", + shards: 1, + mongos: 1, + other: {enableAutoSplit: true}, +}); + +assert.commandWorked(s.s.adminCommand({enablesharding: "test"})); + +var collNum = 0; +var overhead = Object.bsonsize({_id: ObjectId(), i: 1, pad: ""}); + +var getNumberChunks = function(ns) { + return findChunksUtil.countChunksForNs(s.getDB("config"), ns); +}; + +var runCase = function(opts) { + // Expected options. + assert.gte(opts.docSize, 0); + assert.gte(opts.stages.length, 2); + + // Compute padding. + if (opts.docSize < overhead) { + var pad = ""; + } else { + var pad = (new Array(opts.docSize - overhead + 1)).join(' '); + } + + collNum++; + var db = s.getDB("test"); + var collName = "coll" + collNum; + var coll = db.getCollection(collName); + var i = 0; + var limit = 0; + var stageNum = 0; + var stage = opts.stages[stageNum]; + + // Insert initial docs. + var bulk = coll.initializeUnorderedBulkOp(); + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + bulk.insert({i, pad}); + } + assert.commandWorked(bulk.execute()); + + // Create shard key index. + assert.commandWorked(coll.createIndex({i: 1})); + + // Shard collection. + assert.commandWorked(s.s.adminCommand({shardcollection: coll.getFullName(), key: {i: 1}})); + + // Confirm initial number of chunks. + var numChunks = getNumberChunks(coll.getFullName()); + assert.eq(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' initial chunks, but found ' + numChunks + '\nopts: ' + tojson(opts) + + '\nchunks:\n' + s.getChunksString(coll.getFullName())); + + // Do the rest of the stages. + for (stageNum = 1; stageNum < opts.stages.length; stageNum++) { + stage = opts.stages[stageNum]; + + // Insert the later docs (one at a time, to maximise the autosplit effects). + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + coll.insert({i, pad}); + + waitForOngoingChunkSplits(s); + } + + // Confirm number of chunks for this stage. + var numChunks = getNumberChunks(coll.getFullName()); + assert.lte(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' chunks for stage ' + stageNum + ', but found ' + numChunks + '\nopts: ' + + tojson(opts) + '\nchunks:\n' + s.getChunksString(coll.getFullName())); + } +}; + +// Original problematic case. +runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 20000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], +}); + +// Original problematic case (worse). +runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 90000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], +}); + +// Pathological case #1. +runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8191, expectedNumChunks: 1}, + {numDocsToInsert: 2, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], +}); + +// Pathological case #2. +runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8192, expectedNumChunks: 1}, + {numDocsToInsert: 8192, expectedNumChunks: 1}, + ], +}); + +// Lower chunksize to 1MB, and restart the mongod for it to take. We also +// need to restart mongos for the case of the last-lts suite where the +// shard is also last-lts. +assert.commandWorked( + s.getDB("config").getCollection("settings").update({_id: "chunksize"}, {$set: {value: 1}}, { + upsert: true + })); + +s.restartMongos(0); +s.restartShardRS(0); + +// Original problematic case, scaled down to smaller chunksize. +runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 10000, expectedNumChunks: 1}, + {numDocsToInsert: 10, expectedNumChunks: 1}, + {numDocsToInsert: 20, expectedNumChunks: 1}, + {numDocsToInsert: 40, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], +}); + +// Docs just smaller than half chunk size. +runCase({ + docSize: 510 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 6}, + {numDocsToInsert: 10, expectedNumChunks: 12}, + ], +}); + +// Docs just larger than half chunk size. +runCase({ + docSize: 514 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 10}, + {numDocsToInsert: 10, expectedNumChunks: 20}, + ], +}); + +s.stop(); +})(); diff --git a/jstests/sharding/shard_key_index_must_exist.js b/jstests/sharding/shard_key_index_must_exist.js index 32c998bb51b..d9c6ff90723 100644 --- a/jstests/sharding/shard_key_index_must_exist.js +++ b/jstests/sharding/shard_key_index_must_exist.js @@ -1,8 +1,6 @@ (function() { 'use strict'; -load('jstests/libs/feature_flag_util.js'); - let st = new ShardingTest({shards: 1}); assert.commandWorked(st.s.adminCommand({enableSharding: 'test'})); @@ -10,14 +8,18 @@ assert.commandWorked(st.s.adminCommand({enableSharding: 'test'})); let testDB = st.getDB('test'); let checkIndex = function(collName, expectedIndexNames) { - const indexes = testDB.getCollection(collName).getIndexes(); - indexes.forEach(index => { - assert(expectedIndexNames.includes(index.name), 'index should not exist: ' + tojson(index)); + let indexNotSeen = expectedIndexNames; + + testDB.getCollection(collName).getIndexes().forEach((index) => { + assert(expectedIndexNames.includes(index.name), + 'index should not expected to exist: ' + tojson(index)); - expectedIndexNames = expectedIndexNames.filter(name => name !== index.name); + indexNotSeen = indexNotSeen.filter((name) => { + return name == index.name; + }); }); - assert.eq([], expectedIndexNames); + assert.eq([], indexNotSeen); }; (() => { @@ -47,95 +49,23 @@ let checkIndex = function(collName, expectedIndexNames) { checkIndex('user', ['_id_', 'xy']); - // Check dropping indexes in a non-empty collection. - assert.commandWorked(testDB.runCommand({insert: 'user', documents: [{x: 1}]})); - assert.commandFailedWithCode(testDB.runCommand({dropIndexes: 'user', index: {x: 1, y: 1}}), - ErrorCodes.CannotDropShardKeyIndex); - assert.commandFailedWithCode(testDB.runCommand({dropIndexes: 'user', index: 'xy'}), - ErrorCodes.CannotDropShardKeyIndex); - - checkIndex('user', ['_id_', 'xy']); - - if (jsTestOptions().mongosBinVersion != "last-lts") { - // Check that making the shard key compatible index hidden fails. - assert.commandFailedWithCode( - testDB.runCommand({collMod: 'user', index: {name: 'xy', hidden: true}}), - ErrorCodes.InvalidOptions); - } - assert.commandWorked(testDB.runCommand({drop: 'user'})); })(); (() => { - const feautureFlagDropHashedShardKeyIndexes = - FeatureFlagUtil.isEnabled(st.shard0.getDB('admin'), "ShardKeyIndexOptionalHashedSharding"); - if (feautureFlagDropHashedShardKeyIndexes) { - // Users are allowed to drop hashed shard key indexes. This includes any compound index - // that is prefixed by the hashed shard key. - assert.commandWorked( - st.s.adminCommand({shardCollection: 'test.hashed', key: {x: 'hashed'}})); - - assert.commandWorked(testDB.runCommand( - {createIndexes: 'hashed', indexes: [{key: {x: 1, y: 1}, name: 'xy'}]})); - // This will also drop the hashed shard key index. - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: '*'})); - checkIndex('hashed', ['_id_']); - - assert.commandWorked(testDB.runCommand( - {createIndexes: 'hashed', indexes: [{key: {x: 'hashed'}, name: 'x_hashed'}]})); - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: 'x_hashed'})); - checkIndex('hashed', ['_id_']); - - assert.commandWorked(testDB.runCommand({ - createIndexes: 'hashed', - indexes: [{key: {x: 'hashed', y: 1}, name: 'x_hashed_y_1'}] - })); - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: '*'})); - checkIndex('hashed', ['_id_']); - - assert.commandWorked(testDB.runCommand({ - createIndexes: 'hashed', - indexes: [{key: {x: 'hashed', y: 1}, name: 'x_hashed_y_1'}] - })); - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: 'x_hashed_y_1'})); - checkIndex('hashed', ['_id_']); - - // Check dropping indexes in a non-empty collection. - assert.commandWorked(testDB.runCommand( - {createIndexes: 'hashed', indexes: [{key: {x: 'hashed'}, name: 'x_hashed'}]})); - assert.commandWorked(testDB.runCommand({insert: 'hashed', documents: [{x: 1}]})); - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: 'x_hashed'})); - checkIndex('hashed', ['_id_']); - - assert.commandWorked(testDB.runCommand({ - createIndexes: 'hashed', - indexes: [{key: {x: 'hashed', y: 1}, name: 'x_hashed_y_1'}] - })); - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: '*'})); - checkIndex('hashed', ['_id_']); - - // Check that making a hashed shard key compatible index hidden succeeds. - assert.commandWorked(testDB.runCommand( - {createIndexes: 'hashed', indexes: [{key: {x: 'hashed'}, name: 'x_hashed'}]})); - assert.commandWorked( - testDB.runCommand({collMod: 'hashed', index: {name: 'x_hashed', hidden: true}})); - - } else { - assert.commandWorked( - st.s.adminCommand({shardCollection: 'test.hashed', key: {x: 'hashed'}})); - - assert.commandWorked(testDB.runCommand( - {createIndexes: 'hashed', indexes: [{key: {x: 1, y: 1}, name: 'xy'}]})); - - assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: '*'})); - - checkIndex('hashed', ['_id_', 'x_hashed']); - - assert.commandFailedWithCode(testDB.runCommand({dropIndexes: 'hashed', index: 'x_hashed'}), - ErrorCodes.CannotDropShardKeyIndex); - - checkIndex('hashed', ['_id_', 'x_hashed']); - } + assert.commandWorked(st.s.adminCommand({shardCollection: 'test.hashed', key: {x: 'hashed'}})); + + assert.commandWorked( + testDB.runCommand({createIndexes: 'hashed', indexes: [{key: {x: 1, y: 1}, name: 'xy'}]})); + + assert.commandWorked(testDB.runCommand({dropIndexes: 'hashed', index: '*'})); + + checkIndex('hashed', ['_id_', 'x_hashed']); + + assert.commandFailedWithCode(testDB.runCommand({dropIndexes: 'hashed', index: 'x_hashed'}), + ErrorCodes.CannotDropShardKeyIndex); + + checkIndex('hashed', ['_id_', 'x_hashed']); assert.commandWorked(testDB.runCommand({drop: 'hashed'})); })(); diff --git a/jstests/sharding/shard_keys_with_dollar_sign.js b/jstests/sharding/shard_keys_with_dollar_sign.js deleted file mode 100644 index c7fbbc73be5..00000000000 --- a/jstests/sharding/shard_keys_with_dollar_sign.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Tests that the shardCollection command and reshardCollection command correctly reject a shard key - * that has a field name that starts with '$' or contains parts that start with '$' unless the part - * is a DBRef (i.e. is equal to '$id', '$db' or '$ref'). - */ -(function() { -"use strict"; - -const criticalSectionTimeoutMS = 24 * 60 * 60 * 1000; // 1 day -const st = new ShardingTest({ - shards: 1, - other: { - // Avoid spurious failures with small 'ReshardingCriticalSectionTimeout' values being set. - configOptions: - {setParameter: {reshardingCriticalSectionTimeoutMillis: criticalSectionTimeoutMS}} - } -}); - -const dbName = "testDb"; -const ns0 = dbName + ".testColl0"; -const ns1 = dbName + ".testColl1"; -const ns2 = dbName + ".testColl2"; -const db = st.s.getDB(dbName); - -function testValidation(key, {isValidIndexKey, isValidShardKey}) { - jsTest.log(`Testing ${tojson({key, isValidIndexKey, isValidShardKey})}`); - assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); - st.ensurePrimaryShard(dbName, st.shard0.name); - - const createIndexRes = db.getCollection(ns0).createIndex(key); - if (isValidIndexKey) { - assert.commandWorked(createIndexRes); - } else { - assert.commandFailedWithCode(createIndexRes, ErrorCodes.CannotCreateIndex); - } - - const shardCollectionRes = st.s.adminCommand({shardCollection: ns1, key}); - if (isValidShardKey) { - assert.commandWorked(shardCollectionRes); - } else { - assert.commandFailedWithCode(shardCollectionRes, ErrorCodes.BadValue); - } - - assert.commandWorked(st.s.adminCommand({shardCollection: ns2, key: {_id: 1}})); - const reshardCollectionRes = st.s.adminCommand({reshardCollection: ns2, key}); - if (isValidShardKey) { - assert.commandWorked(reshardCollectionRes); - } else { - assert.commandFailedWithCode(reshardCollectionRes, ErrorCodes.BadValue); - } - - assert.commandWorked(db.dropDatabase()); -} - -testValidation({"$x": 1}, {isValidIndexKey: false, isValidShardKey: false}); -testValidation({"x.$y": 1}, {isValidIndexKey: false, isValidShardKey: false}); -testValidation({"$**": 1}, {isValidIndexKey: true, isValidShardKey: false}); -testValidation({"x.$**": 1}, {isValidIndexKey: true, isValidShardKey: false}); -testValidation({"$": 1}, {isValidIndexKey: false, isValidShardKey: false}); - -testValidation({"x$": 1}, {isValidIndexKey: true, isValidShardKey: true}); -testValidation({"x$.y": 1}, {isValidIndexKey: true, isValidShardKey: true}); -testValidation({"x.y$": 1}, {isValidIndexKey: true, isValidShardKey: true}); - -// Verify that a shard key can have a field that contains a DBRef as long as the field itself -// does not start with '$'. -testValidation({"$id": 1}, {isValidIndexKey: false, isValidShardKey: false}); -testValidation({"$db": 1}, {isValidIndexKey: false, isValidShardKey: false}); -testValidation({"$ref": 1}, {isValidIndexKey: false, isValidShardKey: false}); -testValidation({"x.$id": 1}, {isValidIndexKey: true, isValidShardKey: true}); -testValidation({"x.$db": 1}, {isValidIndexKey: true, isValidShardKey: true}); -testValidation({"x.$ref": 1}, {isValidIndexKey: true, isValidShardKey: true}); - -st.stop(); -})(); diff --git a/jstests/sharding/sharded_data_distribution.js b/jstests/sharding/sharded_data_distribution.js deleted file mode 100644 index 8e12f291c65..00000000000 --- a/jstests/sharding/sharded_data_distribution.js +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Test to validate the $shardedDataDistribution stage. - * - * @tags: [ - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; - -function testShardedDataAggregationStage() { - // Get all expected results in obj format - const fooResults = fooColl.aggregate([{$collStats: {storageStats: {}}}]).toArray(); - assert.neq(null, fooResults); - const bazResults = bazColl.aggregate([{$collStats: {storageStats: {}}}]).toArray(); - assert.neq(null, bazResults); - - const objFooResults = {}; - for (let fooRes of fooResults) { - objFooResults[fooRes.shard] = fooRes; - } - - const objBazResults = {}; - for (const bazRes of bazResults) { - objBazResults[bazRes.shard] = bazRes; - } - - const expectedResults = {[ns1]: objFooResults, [ns2]: objBazResults}; - - // Get data to validate - const outputData = adminDb.aggregate([{$shardedDataDistribution: {}}]).toArray(); - - assert.gte(outputData.length, 2); - - // Test the data obtained by $shardedDataDistribution stage - for (const data of outputData) { - const ns = data.ns; - - // Check only for namespaces test.foo and bar.baz - if (expectedResults.hasOwnProperty(ns)) { - // Check for length - assert.eq(data.shards.length, Object.keys(expectedResults[ns]).length); - - // Check for data - for (const shard of data.shards) { - const outputShardName = shard.shardName; - const outputOwnedSizeBytes = shard.ownedSizeBytes; - const outputOrphanedSizeBytes = shard.orphanedSizeBytes; - const outputNumOwnedDocuments = shard.numOwnedDocuments; - const outputNumOrphanedDocs = shard.numOrphanedDocs; - - assert.eq(true, expectedResults[ns].hasOwnProperty(outputShardName)); - - const avgObjSize = expectedResults[ns][outputShardName].storageStats.avgObjSize; - const numOrphanDocs = - expectedResults[ns][outputShardName].storageStats.numOrphanDocs; - const storageStatsCount = expectedResults[ns][outputShardName].storageStats.count; - - const expectedOwnedSizeBytes = (storageStatsCount - numOrphanDocs) * avgObjSize; - const expectedOrphanedSizeBytes = numOrphanDocs * avgObjSize; - const expectedNumOwnedDocuments = storageStatsCount - numOrphanDocs; - const expectedNumOrphanedDocs = numOrphanDocs; - - assert.eq(outputOwnedSizeBytes, expectedOwnedSizeBytes); - assert.eq(outputOrphanedSizeBytes, expectedOrphanedSizeBytes); - assert.eq(outputNumOwnedDocuments, expectedNumOwnedDocuments); - assert.eq(outputNumOrphanedDocs, expectedNumOrphanedDocs); - } - } - } -} - -// Configure initial sharding cluster -const st = new ShardingTest({shards: 2}); -const mongos = st.s; - -const ns1 = "test.foo"; -const ns2 = "bar.baz"; - -const adminDb = mongos.getDB("admin"); -const testDb = mongos.getDB("test"); -const barDb = mongos.getDB("bar"); -const fooColl = testDb.getCollection("foo"); -const bazColl = barDb.getCollection("baz"); - -st.adminCommand({enablesharding: testDb.getName(), primaryShard: st.shard1.shardName}); -st.adminCommand({shardcollection: ns1, key: {skey: 1}}); -st.adminCommand({enablesharding: barDb.getName(), primaryShard: st.shard1.shardName}); -st.adminCommand({shardcollection: ns2, key: {skey: 1}}); - -// Insert data to validate the aggregation stage -for (let i = 0; i < 6; i++) { - assert.commandWorked(fooColl.insert({skey: i})); - assert.commandWorked(bazColl.insert({skey: (i + 5)})); -} - -// Test before chunk migration -testShardedDataAggregationStage(); - -st.adminCommand({split: ns1, middle: {skey: 2}}); -st.adminCommand({moveChunk: ns1, find: {skey: 2}, to: st.shard0.name, _waitForDelete: true}); -st.adminCommand({split: ns2, middle: {skey: 7}}); -st.adminCommand({moveChunk: ns2, find: {skey: 7}, to: st.shard0.name, _waitForDelete: true}); - -// Test after chunk migration -testShardedDataAggregationStage(); - -// Test invalid queries/values. -assert.commandFailedWithCode( - adminDb.runCommand({aggregate: 1, pipeline: [{$shardedDataDistribution: 3}], cursor: {}}), - 6789100); - -const response = assert.commandFailedWithCode( - testDb.runCommand({aggregate: "foo", pipeline: [{$shardedDataDistribution: {}}], cursor: {}}), - 6789102); -assert.neq(-1, response.errmsg.indexOf("$shardedDataDistribution"), response.errmsg); -assert.neq(-1, response.errmsg.indexOf("admin database"), response.errmsg); - -// Test $shardedDataDistribution followed by a $match stage on the 'ns'. -assert.eq(1, adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {ns: ns1}}]).itcount()); -assert.eq(2, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {ns: {$in: [ns1, ns2]}}}]) - .itcount()); -assert.eq(0, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {ns: 'test.IDoNotExist'}}]) - .itcount()); - -// Test $shardedDataDistribution followed by a $match stage on the 'ns' and something else. -assert.eq( - 1, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {ns: ns1, shards: {$size: 2}}}]) - .itcount()); -assert.eq( - 0, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {ns: ns1, shards: {$size: 50}}}]) - .itcount()); - -// Test $shardedDataDistribution followed by a $match stage on the 'ns' and other match stages. -assert.eq( - 1, - adminDb - .aggregate( - [{$shardedDataDistribution: {}}, {$match: {ns: ns1}}, {$match: {shards: {$size: 2}}}]) - .itcount()); -assert.eq( - 0, - adminDb - .aggregate( - [{$shardedDataDistribution: {}}, {$match: {ns: ns1}}, {$match: {shards: {$size: 50}}}]) - .itcount()); -assert.eq(1, - adminDb - .aggregate([ - {$shardedDataDistribution: {}}, - {$match: {ns: /^test/}}, - {$match: {shards: {$size: 2}}}, - {$match: {ns: /foo$/}}, - ]) - .itcount()); - -// Test $shardedDataDistribution followed by a $match stage unrelated to 'ns'. -assert.eq( - 0, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {shards: {$size: 50}}}]).itcount()); - -assert.neq( - 0, - adminDb.aggregate([{$shardedDataDistribution: {}}, {$match: {shards: {$size: 2}}}]).itcount()); - -st.stop(); - -// Test that verifies the behavior in unsharded deployments -const rsTest = new ReplSetTest({name: 'replicaSetTest', nodes: 2}); -rsTest.startSet(); -rsTest.initiate(); - -const primary = rsTest.getPrimary(); -const admin = primary.getDB('admin'); - -const response2 = assert.commandFailedWithCode( - admin.runCommand({aggregate: 1, pipeline: [{$shardedDataDistribution: {}}], cursor: {}}), - 6789101); -assert.neq( - -1, response2.errmsg.indexOf("The $shardedDataDistribution stage can only be run on mongoS")); - -rsTest.stopSet(); -})(); diff --git a/jstests/sharding/sharded_data_distribution_auth.js b/jstests/sharding/sharded_data_distribution_auth.js deleted file mode 100644 index 7e63cf19533..00000000000 --- a/jstests/sharding/sharded_data_distribution_auth.js +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Test to validate the privileges of using $shardedDataDistribution stage. - * - * @tags: [ - * requires_fcv_60, - * ] - */ - -(function() { -'use strict'; - -if (!TestData.auth) { - jsTestLog("Skipping testing authorization since auth is not enabled"); - return; -} - -// Test privileges -function testPrivileges() { - // Create new role with the exact privileges to execute $shardedDataDistribution - assert.commandWorked(adminDb.runCommand({ - createRole: "role_ok_priv", - roles: [], - privileges: [{resource: {cluster: true}, actions: ["shardedDataDistribution"]}] - })); - - // Creates users with privileges and no privileges - assert.commandWorked(adminDb.runCommand({createUser: "user_no_priv", pwd: "pwd", roles: []})); - - assert.commandWorked(adminDb.runCommand( - {createUser: "user_priv1", pwd: "pwd", roles: [{role: "role_ok_priv", db: 'admin'}]})); - - assert.commandWorked(adminDb.runCommand( - {createUser: "user_priv2", pwd: "pwd", roles: [{role: "clusterMonitor", db: 'admin'}]})); - - assert(adminDb.logout()); - - // User is in a role with privileges to execute the stage - assert(adminDb.auth("user_priv1", "pwd")); - assert.commandWorked( - adminDb.runCommand({aggregate: 1, pipeline: [{$shardedDataDistribution: {}}], cursor: {}})); - assert(adminDb.logout()); - - // User is in a role with privileges to execute the stage - assert(adminDb.auth("user_priv2", "pwd")); - assert.commandWorked( - adminDb.runCommand({aggregate: 1, pipeline: [{$shardedDataDistribution: {}}], cursor: {}})); - assert(adminDb.logout()); - - // User has no privileges to execute the stage - assert(adminDb.auth("user_no_priv", "pwd")); - assert.commandFailedWithCode( - adminDb.runCommand({aggregate: 1, pipeline: [{$shardedDataDistribution: {}}], cursor: {}}), - ErrorCodes.Unauthorized, - "user should no longer have privileges to execute $shardedDataDistribution stage."); - assert(adminDb.logout()); -} - -// Configure initial sharding cluster -const st = new ShardingTest({shards: 1}); -const mongos = st.s; - -const ns1 = "test.foo"; -const adminDb = mongos.getDB("admin"); -const testDb = mongos.getDB("test"); - -// Create a super user with __system role. -assert.commandWorked(adminDb.runCommand({createUser: "super", pwd: "super", roles: ["__system"]})); -assert(adminDb.logout()); -assert(adminDb.auth("super", "super")); - -st.adminCommand({shardcollection: ns1, key: {skey: 1}}); - -// Insert data to validate the aggregation stage -for (let i = 0; i < 6; i++) { - assert.commandWorked(testDb.getCollection("foo").insert({skey: i})); -} - -testPrivileges(); - -st.stop(); -})(); diff --git a/jstests/sharding/sharding_balance1.js b/jstests/sharding/sharding_balance1.js new file mode 100644 index 00000000000..d61d677f1ae --- /dev/null +++ b/jstests/sharding/sharding_balance1.js @@ -0,0 +1,49 @@ +/* + * @tags: [ + * requires_fcv_51, + * ] + */ + +(function() { +'use strict'; + +load("jstests/sharding/libs/find_chunks_util.js"); + +var st = new ShardingTest({shards: 2, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); + +const dbName = 'ShardingBalanceTest'; +const collName = 'foo'; +const coll = st.getDB(dbName).getCollection(collName); +const minChunkNum = 20; + +assert.commandWorked(st.s.adminCommand({enablesharding: dbName})); +st.ensurePrimaryShard(dbName, st.shard1.shardName); + +const bigStringSize = 10000; +const bigString = "X=".repeat(bigStringSize / 2); + +jsTest.log("Inserting documents that will account for at least 20MB"); +var insertedChars = 0; +var num = 0; +var bulk = coll.initializeUnorderedBulkOp(); +while (insertedChars < (minChunkNum * 1024 * 1024)) { + bulk.insert({_id: num++, s: bigString}); + insertedChars += bigStringSize; +} +assert.commandWorked(bulk.execute()); + +assert.commandWorked(st.s.adminCommand({shardcollection: coll.getFullName(), key: {_id: 1}})); +jsTest.log("Checking initial chunk distribution: " + st.chunkCounts(collName, dbName)); +assert.lte(minChunkNum, + findChunksUtil.countChunksForNs(st.config, coll.getFullName()), + "Number of initial chunks is less then expected"); +assert.lte(minChunkNum, + st.chunkDiff(collName, dbName), + "The initial chunks difference between the shards is less then expected"); + +jsTest.log("Await for the balancer to reduce the chunk imbalance between the shards"); +// Make sure there's enough time here, since balancing can sleep for 15s or so between balances. +st.awaitBalance(collName, dbName, 1000 * 60 * 5 /* 5 min */); + +st.stop(); +})(); diff --git a/jstests/sharding/sharding_balance2.js b/jstests/sharding/sharding_balance2.js index bfb57e1dec6..4aa528ff6f7 100644 --- a/jstests/sharding/sharding_balance2.js +++ b/jstests/sharding/sharding_balance2.js @@ -5,7 +5,6 @@ 'use strict'; load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); var MaxSizeMB = 1; @@ -18,14 +17,6 @@ var s = new ShardingTest({ {setParameter: {internalQueryMaxBlockingSortMemoryUsageBytes: 32 * 1024 * 1024}} } }); - -// TODO SERVER-66754 review tests disabled because expecting initial chunks split -if (FeatureFlagUtil.isEnabled(s.configRS.getPrimary().getDB('admin'), 'NoMoreAutoSplitter')) { - jsTestLog("Skipping as featureFlagNoMoreAutoSplitter is enabled"); - s.stop(); - return; -} - var db = s.getDB("test"); var names = s.getConnNames(); diff --git a/jstests/sharding/sharding_balance3.js b/jstests/sharding/sharding_balance3.js index b13751bd537..c40699d0f82 100644 --- a/jstests/sharding/sharding_balance3.js +++ b/jstests/sharding/sharding_balance3.js @@ -3,7 +3,6 @@ (function() { load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); var s = new ShardingTest({ name: "slow_sharding_balance3", @@ -17,13 +16,6 @@ var s = new ShardingTest({ } }); -// TODO SERVER-66754 review tests disabled because expecting initial chunks split -if (FeatureFlagUtil.isEnabled(s.configRS.getPrimary().getDB('admin'), 'NoMoreAutoSplitter')) { - jsTestLog("Skipping as featureFlagNoMoreAutoSplitter is enabled"); - s.stop(); - return; -} - s.adminCommand({enablesharding: "test"}); s.ensurePrimaryShard('test', s.shard1.shardName); diff --git a/jstests/sharding/sharding_balance4.js b/jstests/sharding/sharding_balance4.js new file mode 100644 index 00000000000..de54e9460cd --- /dev/null +++ b/jstests/sharding/sharding_balance4.js @@ -0,0 +1,180 @@ +/** + * Check that doing updates done during a migrate all go to the right place + * + * This test is labeled resource intensive because its total io_write is 36MB compared to a median + * of 5MB across all sharding tests in wiredTiger. + * @tags: [resource_intensive] + */ +(function() { +load('jstests/sharding/autosplit_include.js'); +load("jstests/sharding/libs/find_chunks_util.js"); + +var s = new ShardingTest({shards: 2, mongos: 1, other: {chunkSize: 1, enableAutoSplit: true}}); + +// Double the balancer interval to produce fewer migrations per unit time so that the test does not +// run out of stale shard version retries. +s.forEachConfigServer((conn) => { + conn.adminCommand({ + configureFailPoint: 'overrideBalanceRoundInterval', + mode: 'alwaysOn', + data: {intervalMs: 2000} + }); +}); + +assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); +s.ensurePrimaryShard('test', s.shard1.shardName); +assert.commandWorked(s.s0.adminCommand({shardcollection: "test.foo", key: {_id: 1}})); + +s.config.settings.find().forEach(printjson); + +db = s.getDB("test"); + +bigString = ""; +while (bigString.length < 10000) + bigString += "asdasdasdasdadasdasdasdasdasdasdasdasda"; + +N = 3000; + +num = 0; + +var counts = {}; + +// +// TODO: Rewrite to make much clearer. +// +// The core behavior of this test is to add a bunch of documents to a sharded collection, then +// incrementally update each document and make sure the counts in the document match our update +// counts while balancing occurs (doUpdate()). Every once in a while we also check (check()) +// our counts via a query. +// +// If during a chunk migration an update is missed, we trigger an assertion and fail. +// + +function doUpdate(bulk, includeString, optionalId) { + var up = {$inc: {x: 1}}; + if (includeString) { + up["$set"] = {s: bigString}; + } + var myid = optionalId == undefined ? Random.randInt(N) : optionalId; + bulk.find({_id: myid}).upsert().update(up); + + counts[myid] = (counts[myid] ? counts[myid] : 0) + 1; + return myid; +} + +Random.setRandomSeed(); + +// Initially update all documents from 1 to N, otherwise later checks can fail because no +// document previously existed +for (i = 0; i < N; i++) { + let bulk = db.foo.initializeUnorderedBulkOp(); + doUpdate(bulk, true, i); + assert.commandWorked(bulk.execute()); + waitForOngoingChunkSplits(s); +} + +for (i = 0; i < N * 9; i++) { + let bulk = db.foo.initializeUnorderedBulkOp(); + doUpdate(bulk, false); + assert.commandWorked(bulk.execute()); + waitForOngoingChunkSplits(s); +} + +for (var i = 0; i < 50; i++) { + s.printChunks("test.foo"); + if (check("initial:" + i, true)) + break; + sleep(5000); +} +check("initial at end"); + +assert.lt(20, findChunksUtil.countChunksForNs(s.config, "test.foo"), "setup2"); + +function check(msg, dontAssert) { + for (var x in counts) { + var e = counts[x]; + var z = db.foo.findOne({_id: parseInt(x)}); + + if (z && z.x == e) + continue; + + if (dontAssert) { + if (z) + delete z.s; + print("not asserting for key failure: " + x + " want: " + e + " got: " + tojson(z)); + return false; + } + + s.s.getDB("admin").runCommand({setParameter: 1, logLevel: 2}); + + printjson(db.foo.findOne({_id: parseInt(x)})); + + var y = db.foo.findOne({_id: parseInt(x)}); + + if (y) { + delete y.s; + } + + s.printChunks("test.foo"); + + assert(z, "couldn't find : " + x + " y:" + tojson(y) + " e: " + e + " " + msg); + assert.eq(e, z.x, "count for : " + x + " y:" + tojson(y) + " " + msg); + } + + return true; +} + +var consecutiveNoProgressMadeErrors = 0; + +function diff1() { + jsTest.log("Running diff1..."); + + var bulk = db.foo.initializeUnorderedBulkOp(); + var myid = doUpdate(bulk, false); + var res = bulk.execute(); + + assert(res instanceof BulkWriteResult, + 'Result from bulk.execute should be of type BulkWriteResult'); + if (res.hasWriteErrors()) { + res.writeErrors.forEach(function(err) { + // Ignore up to 3 consecutive NoProgressMade errors for the cases where migration + // might be going faster than the writes are executing + if (err.code == ErrorCodes.NoProgressMade) { + consecutiveNoProgressMadeErrors++; + if (consecutiveNoProgressMadeErrors < 3) { + return; + } + } + + assert.commandWorked(res); + }); + } else { + consecutiveNoProgressMadeErrors = 0; + + assert.eq(1, + res.nModified, + "diff myid: " + myid + " 2: " + res.toString() + "\n" + + " correct count is: " + counts[myid] + + " db says count is: " + tojson(db.foo.findOne({_id: myid}))); + } + + var x = s.chunkCounts("foo"); + if (Math.random() > .999) + printjson(x); + + return Math.max(x[s.shard0.shardName], x[s.shard1.shardName]) - + Math.min(x[s.shard0.shardName], x[s.shard1.shardName]); +} + +assert.lt(20, diff1(), "initial load"); +print(diff1()); + +s.startBalancer(); + +assert.soon(function() { + var d = diff1(); + return d < 5; +}, "balance didn't happen", 1000 * 60 * 20, 1); + +s.stop(); +})(); diff --git a/jstests/sharding/sharding_migrate_cursor1.js b/jstests/sharding/sharding_migrate_cursor1.js index c0ef8d391fd..3bac25b5e34 100644 --- a/jstests/sharding/sharding_migrate_cursor1.js +++ b/jstests/sharding/sharding_migrate_cursor1.js @@ -7,18 +7,10 @@ */ (function() { -load("jstests/libs/feature_flag_util.js"); - var chunkSize = 25; var s = new ShardingTest( {name: "migrate_cursor1", shards: 2, mongos: 1, other: {chunkSize: chunkSize}}); -// TODO SERVER-66754 review tests disabled because expecting initial chunks split -if (FeatureFlagUtil.isEnabled(s.configRS.getPrimary().getDB('admin'), 'NoMoreAutoSplitter')) { - jsTestLog("Skipping as featureFlagNoMoreAutoSplitter is enabled"); - s.stop(); - return; -} s.adminCommand({enablesharding: "test"}); db = s.getDB("test"); diff --git a/jstests/sharding/sharding_statistics_server_status.js b/jstests/sharding/sharding_statistics_server_status.js index e7f087d6a68..67e22039403 100644 --- a/jstests/sharding/sharding_statistics_server_status.js +++ b/jstests/sharding/sharding_statistics_server_status.js @@ -13,34 +13,24 @@ load("jstests/libs/chunk_manipulation_util.js"); load("jstests/libs/parallelTester.js"); load("jstests/libs/wait_for_command.js"); -// Documents inserted in this test are in the shape {_id: int} so the size is 18 bytes -const docSizeInBytes = 18; - function ShardStat() { this.countDonorMoveChunkStarted = 0; this.countRecipientMoveChunkStarted = 0; this.countDocsClonedOnRecipient = 0; this.countDocsClonedOnDonor = 0; - this.countDocsDeletedByRangeDeleter = 0; - this.countBytesDeletedByRangeDeleter = 0; + this.countDocsDeletedOnDonor = 0; } -function incrementStatsAndCheckServerShardStats(db, donor, recipient, numDocs) { +function incrementStatsAndCheckServerShardStats(donor, recipient, numDocs) { ++donor.countDonorMoveChunkStarted; donor.countDocsClonedOnDonor += numDocs; ++recipient.countRecipientMoveChunkStarted; recipient.countDocsClonedOnRecipient += numDocs; - donor.countDocsDeletedByRangeDeleter += numDocs; - // The size of each document inserted in this test is 1 byte, so the number of bytes - // deleted must be exactly `numDocs` - donor.countBytesDeletedByRangeDeleter += numDocs * docSizeInBytes; + donor.countDocsDeletedOnDonor += numDocs; const statsFromServerStatus = shardArr.map(function(shardVal) { return shardVal.getDB('admin').runCommand({serverStatus: 1}).shardingStatistics; }); for (let i = 0; i < shardArr.length; ++i) { - let countDocsDeleted = statsFromServerStatus[i].hasOwnProperty('countDocsDeletedOnDonor') - ? statsFromServerStatus[i].countDocsDeletedOnDonor - : statsFromServerStatus[i].countDocsDeletedByRangeDeleter; assert(statsFromServerStatus[i]); assert(statsFromServerStatus[i].countStaleConfigErrors); assert(statsFromServerStatus[i].totalCriticalSectionCommitTimeMillis); @@ -53,13 +43,8 @@ function incrementStatsAndCheckServerShardStats(db, donor, recipient, numDocs) { assert.eq(stats[i].countDocsClonedOnRecipient, statsFromServerStatus[i].countDocsClonedOnRecipient); assert.eq(stats[i].countDocsClonedOnDonor, statsFromServerStatus[i].countDocsClonedOnDonor); - assert.eq(stats[i].countDocsDeletedByRangeDeleter, countDocsDeleted); - const fcvDoc = db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}); - if (MongoRunner.compareBinVersions(fcvDoc.featureCompatibilityVersion.version, '6.0') >= - 0) { - assert.eq(stats[i].countBytesDeletedByRangeDeleter, - statsFromServerStatus[i].countBytesDeletedByRangeDeleter); - } + assert.eq(stats[i].countDocsDeletedOnDonor, + statsFromServerStatus[i].countDocsDeletedOnDonor); assert.eq(stats[i].countRecipientMoveChunkStarted, statsFromServerStatus[i].countRecipientMoveChunkStarted); } @@ -79,13 +64,6 @@ function checkServerStatusAbortedMigrationCount(shardConn, count) { assert.eq(count, shardStats.countDonorMoveChunkAbortConflictingIndexOperation); } -function checkServerStatusNumShardedCollections(conn, count) { - const shardStats = - assert.commandWorked(conn.adminCommand({serverStatus: 1})).shardingStatistics; - assert(shardStats.hasOwnProperty("numShardedCollections")); - assert.eq(count, shardStats.numShardedCollections); -} - function runConcurrentMoveChunk(host, ns, toShard) { const mongos = new Mongo(host); // Helper function to run moveChunk, retrying on ConflictingOperationInProgress. We need to @@ -154,21 +132,10 @@ st.ensurePrimaryShard(coll.getDB() + "", st.shard0.shardName); assert.commandWorked(admin.runCommand({shardCollection: coll + "", key: {_id: 1}})); assert.commandWorked(admin.runCommand({split: coll + "", middle: {_id: 0}})); -// Check the number of sharded collections. -const testDB = st.rs0.getPrimary().getDB(dbName); -const fcvDoc = testDB.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}); -if (MongoRunner.compareBinVersions(fcvDoc.featureCompatibilityVersion.version, '6.0') >= 0) { - st.shardColl(dbName + ".coll2", {_id: 1}, false); - st.shardColl(dbName + ".coll3", {_id: 1}, false); - const configCollections = mongos.getCollection("config.collections"); - checkServerStatusNumShardedCollections(st.configRS.getPrimary(), - configCollections.countDocuments({})); -} - // Move chunk from shard0 to shard1 without docs. assert.commandWorked( mongos.adminCommand({moveChunk: coll + '', find: {_id: 1}, to: st.shard1.shardName})); -incrementStatsAndCheckServerShardStats(testDB, stats[0], stats[1], numDocsInserted); +incrementStatsAndCheckServerShardStats(stats[0], stats[1], numDocsInserted); // Insert docs and then move chunk again from shard1 to shard0. for (let i = 0; i < numDocsToInsert; ++i) { @@ -177,17 +144,17 @@ for (let i = 0; i < numDocsToInsert; ++i) { } assert.commandWorked(mongos.adminCommand( {moveChunk: coll + '', find: {_id: 1}, to: st.shard0.shardName, _waitForDelete: true})); -incrementStatsAndCheckServerShardStats(testDB, stats[1], stats[0], numDocsInserted); +incrementStatsAndCheckServerShardStats(stats[1], stats[0], numDocsInserted); // Check that numbers are indeed cumulative. Move chunk from shard0 to shard1. assert.commandWorked(mongos.adminCommand( {moveChunk: coll + '', find: {_id: 1}, to: st.shard1.shardName, _waitForDelete: true})); -incrementStatsAndCheckServerShardStats(testDB, stats[0], stats[1], numDocsInserted); +incrementStatsAndCheckServerShardStats(stats[0], stats[1], numDocsInserted); // Move chunk from shard1 to shard0. assert.commandWorked(mongos.adminCommand( {moveChunk: coll + '', find: {_id: 1}, to: st.shard0.shardName, _waitForDelete: true})); -incrementStatsAndCheckServerShardStats(testDB, stats[1], stats[0], numDocsInserted); +incrementStatsAndCheckServerShardStats(stats[1], stats[0], numDocsInserted); // // Tests for the count of migrations aborting from lock timeouts. diff --git a/jstests/sharding/stale_mongos_updates_and_removes.js b/jstests/sharding/stale_mongos_updates_and_removes.js index ca3617cd24a..06878fe9177 100644 --- a/jstests/sharding/stale_mongos_updates_and_removes.js +++ b/jstests/sharding/stale_mongos_updates_and_removes.js @@ -132,16 +132,20 @@ function checkAllRemoveQueries(makeMongosStaleFunc) { assert.writeError(res); } - doRemove(emptyQuery, single, makeMongosStaleFunc); + // Not possible because single remove requires equality match on shard key. + checkRemoveIsInvalid(emptyQuery, single, makeMongosStaleFunc); doRemove(emptyQuery, multi, makeMongosStaleFunc); doRemove(pointQuery, single, makeMongosStaleFunc); doRemove(pointQuery, multi, makeMongosStaleFunc); - doRemove(rangeQuery, single, makeMongosStaleFunc); + // Not possible because can't do range query on a single remove. + checkRemoveIsInvalid(rangeQuery, single, makeMongosStaleFunc); doRemove(rangeQuery, multi, makeMongosStaleFunc); - doRemove(multiPointQuery, single, makeMongosStaleFunc); + // Not possible because single remove must contain _id or shard key at top level + // (not within $or). + checkRemoveIsInvalid(multiPointQuery, single, makeMongosStaleFunc); doRemove(multiPointQuery, multi, makeMongosStaleFunc); } diff --git a/jstests/sharding/standalone_in_queryable_backup_mode.js b/jstests/sharding/standalone_in_queryable_backup_mode.js deleted file mode 100644 index e68bd41dbb0..00000000000 --- a/jstests/sharding/standalone_in_queryable_backup_mode.js +++ /dev/null @@ -1,99 +0,0 @@ -/** - * The goal of this test is to apply some oplog entries during the startup of a mongod process - * configured with --shardsvr and --queryableBackupMode. - * - * @tags: [ - * # In-memory storage engine does not support queryable backups. - * requires_persistence, - * # Config shards do not support queryable backups. - * config_shard_incompatible - * ] - */ - -// This test shuts down a shard. -TestData.skipCheckingUUIDsConsistentAcrossCluster = true; -TestData.skipCheckingIndexesConsistentAcrossCluster = true; -TestData.skipCheckDBHashes = true; -TestData.skipCheckOrphans = true; -TestData.skipCheckShardFilteringMetadata = true; -TestData.skipCheckMetadataConsistency = true; - -(function() { -'use strict'; - -const st = new ShardingTest({ - mongos: 1, - shards: 1, - rs: {nodes: 2}, -}); - -jsTest.log("Going to set up the environment"); -var kDbName = 'testDb'; -var kShardedCollName = 'testShardedColl'; -var kUnshardedCollName = 'testUnshardedColl'; - -assert.commandWorked(st.s.adminCommand({enableSharding: kDbName})); -assert.commandWorked( - st.s.adminCommand({shardCollection: kDbName + '.' + kShardedCollName, key: {_id: 1}})); - -const recoveryTimestamp = - assert.commandWorked(st.rs0.getPrimary().getDB(kDbName).runCommand({ping: 1})).operationTime; - -jsTest.log("Going to hold the stable timestamp of the secondary node at " + - tojson(recoveryTimestamp)); -// Hold back the recovery timestamp before doing another write so we have some oplog entries to -// apply when restart in queryableBackupMode with recoverToOplogTimestamp. -assert.commandWorked(st.rs0.getSecondary().getDB('admin').adminCommand({ - "configureFailPoint": 'holdStableTimestampAtSpecificTimestamp', - "mode": 'alwaysOn', - "data": {"timestamp": recoveryTimestamp} -})); - -jsTest.log("Going to apply some CRUD operations over sharded and unsharded collections"); -function applyCRUDOnColl(coll) { - coll.insert({age: 42}); - coll.update({age: 42}, {$set: {name: "john"}}); - coll.deleteMany({}); -} -applyCRUDOnColl(st.s.getDB(kDbName)[kShardedCollName]); -applyCRUDOnColl(st.s.getDB(kDbName)[kUnshardedCollName]); -st.rs0.awaitReplication(); - -jsTest.log("Going to stop the secondary node of the shard"); -const operationTime = - assert.commandWorked(st.rs0.getPrimary().getDB(kDbName).runCommand({ping: 1})).operationTime; -const secondaryPort = st.rs0.getSecondary().port; -const secondaryDbPath = st.rs0.getSecondary().dbpath; -MongoRunner.stopMongod(st.rs0.getSecondary()); - -jsTest.log( - "Going to start a mongod process with --shardsvr, --queryableBackupMode and recoverToOplogTimestamp"); -const shardIdentity = st.rs0.getPrimary().getDB("admin").getCollection("system.version").findOne({ - _id: "shardIdentity" -}); -let configFileStr = - "sharding:\n _overrideShardIdentity: '" + tojson(shardIdentity).replace(/\s+/g, ' ') + "'"; -let delim = _isWindows() ? '\\' : '/'; -let configFilePath = secondaryDbPath + delim + "config-for-read-only-mongod.yml"; -writeFile(configFilePath, configFileStr); - -const newMongoD = MongoRunner.runMongod({ - config: configFilePath, - dbpath: secondaryDbPath, - port: secondaryPort, - noCleanData: true, - setParameter: {recoverToOplogTimestamp: tojson({timestamp: operationTime})}, - queryableBackupMode: "", - shardsvr: "", -}); - -jsTest.log("Going to verify the number of documents of both collections"); -assert.eq(newMongoD.getDB(kDbName)[kShardedCollName].find({}).itcount(), 0); -assert.eq(newMongoD.getDB(kDbName)[kUnshardedCollName].find({}).itcount(), 0); - -jsTest.log("Going to stop the shardsvr queryable backup mode mongod process"); -MongoRunner.stopMongod(newMongoD); - -jsTest.log("Going to stop the sharding test"); -st.stop(); -})(); diff --git a/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js b/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js deleted file mode 100644 index 5fa737560ef..00000000000 --- a/jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js +++ /dev/null @@ -1,62 +0,0 @@ -// Tests that a deleteMany operation on a cluster doesn't return a UUID mismatch in case it targets -// a shard with no chunks. -// -// @tags: [ -// requires_fcv_81, -// ] -// -(function() { -"use strict"; -const st = new ShardingTest({shards: 3}); - -const db = st.s.getDB(jsTestName()); -assert.commandWorked( - st.s.adminCommand({enableSharding: db.getName(), primaryShard: st.shard0.shardName})); - -const shardedColl1 = db.sharded_1; - -assert.commandWorked(shardedColl1.insert({_id: 0})); -assert.commandWorked(shardedColl1.insert({_id: 10})); -assert.commandWorked(shardedColl1.insert({_id: 50})); -assert.commandWorked(shardedColl1.insert({_id: 100})); - -const splitChunk = function(coll, splitPointKeyValue) { - assert.commandWorked( - st.s.adminCommand({split: coll.getFullName(), middle: {_id: splitPointKeyValue}})); -}; - -const uuid = function(coll) { - return assert.commandWorked(db.runCommand({listCollections: 1})) - .cursor.firstBatch.find(c => c.name === coll.getName()) - .info.uuid; -}; - -// shard0: [inf, 50), shard1: [50, inf), shard2 has no chunks -assert.commandWorked( - st.s.adminCommand({shardCollection: shardedColl1.getFullName(), key: {_id: 1}})); -splitChunk(shardedColl1, 50); -assert.commandWorked(st.s.adminCommand( - {moveChunk: shardedColl1.getFullName(), find: {_id: 50}, to: st.shard1.shardName})); - -const cmdObj = { - delete: shardedColl1.getName(), - collectionUUID: uuid(shardedColl1), - deletes: [{ - q: { - $and: - [{$expr: {$gte: ["$_id", {$literal: 0}]}}, {$expr: {$lt: ["$_id", {$literal: 99}]}}] - }, - limit: 0, - hint: {_id: 1} - }], -}; - -// Run a multi-write which should only target shards 0 & 1. This could target shard 2 but we -// shouldn't see any UUID mismatch errors since shard2 has no chunks and a delete is correctly -// defined in this case. -let res = db.runCommand(cmdObj); -assert.commandWorked(res); -// Only 3 documents fulfill the criteria so verify we've deleted them. -assert.eq(3, res.n); -st.stop(); -})(); diff --git a/jstests/sharding/timeseries_cluster_collstats.js b/jstests/sharding/timeseries_cluster_collstats.js index 8d5b196d8e7..b672c67a3fe 100644 --- a/jstests/sharding/timeseries_cluster_collstats.js +++ b/jstests/sharding/timeseries_cluster_collstats.js @@ -1,7 +1,6 @@ /** * Tests that the cluster collStats command returns timeseries statistics in the expected format. * - * For legacy collStats command: * { * ...., * "ns" : ..., @@ -21,43 +20,15 @@ * .... * } * - * For aggregate $collStats stage: - * [ - * { - * ...., - * "ns" : ..., - * "shard" : ..., - * "latencyStats" : { - * .... - * }, - * "storageStats" : { - * ..., - * "timeseries" : { - * ..., - * }, - * }, - * "count" : { - * .... - * }, - * "queryExecStats" : { - * .... - * }, - * }, - * { - * .... (Other shard's result) - * }, - * ... - * ] - * * @tags: [ - * requires_fcv_60, + * requires_fcv_51 * ] */ (function() { load("jstests/core/timeseries/libs/timeseries.js"); -const numShards = 2; -const st = new ShardingTest({shards: numShards}); + +const st = new ShardingTest({shards: 2}); if (!TimeseriesTest.shardedtimeseriesCollectionsEnabled(st.shard0)) { jsTestLog("Skipping test because the sharded time-series collection feature flag is disabled"); @@ -112,14 +83,14 @@ assert.commandWorked(st.s.adminCommand({ key: {[metaField]: 1}, })); -// Force splitting numShards chunks. +// Force splitting two chunks. const splitPoint = { - meta: numberDoc / numShards + meta: numberDoc / 2 }; assert.commandWorked(st.s.adminCommand({split: bucketNs, middle: splitPoint})); // Ensure that currently both chunks reside on the primary shard. let counts = st.chunkCounts(`system.buckets.${collName}`, dbName); -assert.eq(numShards, counts[primaryShard.shardName]); +assert.eq(2, counts[primaryShard.shardName]); // Move one of the chunks into the second shard. assert.commandWorked(st.s.adminCommand( {movechunk: bucketNs, find: splitPoint, to: otherShard.name, _waitForDelete: true})); @@ -134,13 +105,20 @@ for (let i = 0; i < numberDoc; i++) { } assert.eq(mongosColl.find().itcount(), numberDoc * 2); -function checkAllFieldsAreInResult(result) { - assert(result.hasOwnProperty("latencyStats"), result); - assert(result.hasOwnProperty("storageStats"), result); - assert(result.hasOwnProperty("count"), result); - assert(result.hasOwnProperty("queryExecStats"), result); -} +clusterCollStatsResult = assert.commandWorked(mongosDB.runCommand({collStats: collName})); +jsTestLog("Sharded cluster collStats command result: " + tojson(clusterCollStatsResult)); +// Check that the top-level 'timeseries' fields match the sum of two shard's, that the stats were +// correctly aggregated. +assert(clusterCollStatsResult.shards[primaryShard.shardName].timeseries, + "Expected a shard 'timeseries' field on shard " + primaryShard.shardName + + " but didn't find one: " + tojson(clusterCollStatsResult)); +assert(clusterCollStatsResult.shards[otherShard.shardName].timeseries, + "Expected a shard 'timeseries' field on shard " + otherShard.shardName + + " but didn't find one: " + tojson(clusterCollStatsResult)); +assert(clusterCollStatsResult.timeseries, + "Expected an aggregated 'timeseries' field but didn't find one: " + + tojson(clusterCollStatsResult)); function assertTimeseriesAggregationCorrectness(total, shards) { assert(shards.every(x => x.bucketNs === total.bucketNs)); assert.eq(total.bucketCount, @@ -188,69 +166,10 @@ function assertTimeseriesAggregationCorrectness(total, shards) { assert(total.numCommits > 0); assert(total.numMeasurementsCommitted > 0); } - -function verifyClusterCollStatsResult( - clusterCollStatsResult, sumTimeseriesStatsAcrossShards, isAggregation) { - if (isAggregation) { - // $collStats should output one document per shard. - assert.eq(clusterCollStatsResult.length, - numShards, - "Expected " + numShards + - "documents to be returned: " + tojson(clusterCollStatsResult)); - - checkAllFieldsAreInResult(clusterCollStatsResult[0]); - checkAllFieldsAreInResult(clusterCollStatsResult[1]); - } - - assert(sumTimeseriesStatsAcrossShards, - "Expected an aggregated 'timeseries' field but didn't find one: " + - tojson(clusterCollStatsResult)); - - const primaryShardStats = isAggregation - ? clusterCollStatsResult[0].storageStats.timeseries - : clusterCollStatsResult.shards[primaryShard.shardName].timeseries; - - const otherShardStats = isAggregation - ? clusterCollStatsResult[1].storageStats.timeseries - : clusterCollStatsResult.shards[otherShard.shardName].timeseries; - - // Check that the top-level 'timeseries' fields match the sum of two shard's, that the stats - // were correctly aggregated. - assert(primaryShardStats, - "Expected a shard 'timeseries' field on shard " + primaryShard.shardName + - " but didn't find one: " + tojson(clusterCollStatsResult)); - assert(otherShardStats, - "Expected a shard 'timeseries' field on shard " + otherShard.shardName + - " but didn't find one: " + tojson(clusterCollStatsResult)); - - assertTimeseriesAggregationCorrectness(sumTimeseriesStatsAcrossShards, - [primaryShardStats, otherShardStats]); -} - -// Tests that the output of the collStats command returns results from both the shards and -// includes all the expected fields. -clusterCollStatsResult = assert.commandWorked(mongosDB.runCommand({collStats: collName})); -jsTestLog("Sharded cluster collStats command result: " + tojson(clusterCollStatsResult)); -const sumTimeseriesStatsAcrossShards = clusterCollStatsResult.timeseries; -verifyClusterCollStatsResult( - clusterCollStatsResult, sumTimeseriesStatsAcrossShards, false // isAggregation -); - -// Tests that the output of the $collStats stage returns results from both the shards and includes -// all the expected fields. -clusterCollStatsResult = - mongosColl - .aggregate( - [{$collStats: {latencyStats: {}, storageStats: {}, count: {}, queryExecStats: {}}}]) - .toArray(); -jsTestLog("Sharded cluster collStats aggregation result: " + tojson(clusterCollStatsResult)); - -// Use the same sumTimeseriesStatsAcrossShards value as the collStats command since -// aggregation does not sum up timeseries stats results. This will also verify that the results -// output by collStats in find and aggregation are the same. -verifyClusterCollStatsResult( - clusterCollStatsResult, sumTimeseriesStatsAcrossShards, true // isAggregation -); +assertTimeseriesAggregationCorrectness(clusterCollStatsResult.timeseries, [ + clusterCollStatsResult.shards[primaryShard.shardName].timeseries, + clusterCollStatsResult.shards[otherShard.shardName].timeseries +]); st.stop(); })(); diff --git a/jstests/sharding/timeseries_cluster_indexstats.js b/jstests/sharding/timeseries_cluster_indexstats.js index 5c480cdf65f..f734d8b03b4 100644 --- a/jstests/sharding/timeseries_cluster_indexstats.js +++ b/jstests/sharding/timeseries_cluster_indexstats.js @@ -59,7 +59,13 @@ function checkIndexStats(coll, keys, sharded) { keys.length, `There should be ${keys.length} indices on the collection.\n${tojson(indices)}`); indices.forEach((index, i) => { - assert(index.hasOwnProperty('shard'), tojson(index)); + assert.eq(index.hasOwnProperty('shard'), + sharded, + sharded + ? `Index stats 'shard' field should exist on a sharded collection.\n${ + tojson(index)}` + : `Index stats 'shard' field should not exist on a non-sharded collection.\n${ + tojson(index)}`); assert.docEq( index.key, keys[i], `Index should have key spec ${tojson(keys[i])}.\n${tojson(index)}`); }); diff --git a/jstests/sharding/timeseries_coll_mod.js b/jstests/sharding/timeseries_coll_mod.js index 36b33cb3e04..2627fa21cf4 100644 --- a/jstests/sharding/timeseries_coll_mod.js +++ b/jstests/sharding/timeseries_coll_mod.js @@ -68,10 +68,11 @@ function runBasicTest(failPoint) { key: {[metaField]: 1}, })); - // Check that collMod commands works for the sharded time-series collection. - assert.commandWorked(db[collName].createIndex({'a': 1})); - assert.commandWorked(db.runCommand({collMod: collName, index: {name: 'a_1', hidden: true}})); - assert.commandWorked(db.runCommand({collMod: collName, index: {name: 'a_1', hidden: false}})); + // Normal collMod commands works for the sharded time-series collection. + assert.commandWorked( + db.runCommand({collMod: collName, index: {name: indexName, hidden: true}})); + assert.commandWorked( + db.runCommand({collMod: collName, index: {name: indexName, hidden: false}})); if (failPoint) { // Granularity update disabled for sharded time-series collection, when we're using primary diff --git a/jstests/sharding/timeseries_insert_targeting_normalize_metadata.js b/jstests/sharding/timeseries_insert_targeting_normalize_metadata.js deleted file mode 100644 index 2aceb57475f..00000000000 --- a/jstests/sharding/timeseries_insert_targeting_normalize_metadata.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Tests that when inserting into a sharded time-series collection, the targeting takes into account - * metadata normalization. - */ - -(function() { -"use strict"; - -const st = new ShardingTest({shards: 2}); - -const db = st.s0.getDB(jsTestName()); -const coll = db.coll; -const bucketsColl = db.system.buckets.coll; - -assert.commandWorked( - db.createCollection(coll.getName(), {timeseries: {timeField: "t", metaField: "m"}})); -assert.commandWorked(db.adminCommand({shardCollection: coll.getFullName(), key: {m: 1}})); -assert.commandWorked(st.splitAt(bucketsColl.getFullName(), {meta: {a: 5, b: 5}})); -assert.commandWorked( - st.moveChunk(bucketsColl.getFullName(), {meta: {a: 0, b: 0}}, st.shard0.shardName)); -assert.commandWorked( - st.moveChunk(bucketsColl.getFullName(), {meta: {a: 10, b: 10}}, st.shard1.shardName)); - -assert.commandWorked( - coll.insert({_id: 0, t: ISODate("2023-01-01T12:00:00.000Z"), m: {a: 1, b: 1}})); -assert.commandWorked( - coll.insert({_id: 1, t: ISODate("2023-01-01T12:00:01.000Z"), m: {b: 1, a: 1}})); - -assert.eq(coll.find().itcount(), 2); - -st.stop(); -})(); diff --git a/jstests/sharding/timeseries_multiple_mongos.js b/jstests/sharding/timeseries_multiple_mongos.js index 6b6aee4987c..dbf88f6fa56 100644 --- a/jstests/sharding/timeseries_multiple_mongos.js +++ b/jstests/sharding/timeseries_multiple_mongos.js @@ -298,6 +298,19 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { update: collName, updates: [{ + q: {}, + u: {$inc: {[metaField + ".b"]: 1}}, + multi: true, + }] + }, + numProfilerEntries: {sharded: 2, unsharded: 1}, + }); + + runTest({ + shardKey: {[metaField + ".a"]: 1}, + cmdObj: { + update: collName, + updates: [{ q: {[metaField + ".a"]: 1}, u: {$inc: {[metaField + ".b"]: -1}}, multi: true, @@ -311,6 +324,19 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { update: bucketsCollName, updates: [{ + q: {}, + u: {$inc: {["meta.b"]: 1}}, + multi: true, + }] + }, + numProfilerEntries: {sharded: 2, unsharded: 1}, + }); + + runTest({ + shardKey: {[metaField + ".a"]: 1}, + cmdObj: { + update: bucketsCollName, + updates: [{ q: {["meta.a"]: 1}, u: {$inc: {["meta.b"]: -1}}, multi: true, @@ -325,6 +351,18 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { delete: collName, deletes: [{ + q: {}, + limit: 0, + }], + }, + numProfilerEntries: {sharded: 2, unsharded: 1}, + }); + + runTest({ + shardKey: {[metaField]: 1}, + cmdObj: { + delete: collName, + deletes: [{ q: {[metaField]: 0}, limit: 0, }], @@ -337,6 +375,18 @@ if (TimeseriesTest.timeseriesUpdatesAndDeletesEnabled(st.shard0) && cmdObj: { delete: bucketsCollName, deletes: [{ + q: {}, + limit: 0, + }], + }, + numProfilerEntries: {sharded: 2, unsharded: 1}, + }); + + runTest({ + shardKey: {[metaField]: 1}, + cmdObj: { + delete: bucketsCollName, + deletes: [{ q: {meta: 0}, limit: 0, }], diff --git a/jstests/sharding/timeseries_query.js b/jstests/sharding/timeseries_query.js index 7a8b835c876..4cf3b1c28c5 100644 --- a/jstests/sharding/timeseries_query.js +++ b/jstests/sharding/timeseries_query.js @@ -122,7 +122,7 @@ function runQuery( if (expectCollScan) { assert(isCollscan(sDB, winningPlan)); } else { - assert(isIxscan(sDB, winningPlan) || isClusteredIxscan(sDB, winningPlan)); + assert(isIxscan(sDB, winningPlan)); } }); }); diff --git a/jstests/sharding/timeseries_sharding_admin_commands.js b/jstests/sharding/timeseries_sharding_admin_commands.js index bfd8d3ef853..696fa86ee69 100644 --- a/jstests/sharding/timeseries_sharding_admin_commands.js +++ b/jstests/sharding/timeseries_sharding_admin_commands.js @@ -274,5 +274,17 @@ function assertRangeMatch(savedRange, paramRange) { dropTimeSeriesColl(); })(); +// Check renameCollection command cannot modify name through the view namespace. +(function checkRenameCollectionCommand() { + createTimeSeriesColl( + {index: {[metaField]: 1, [timeField]: 1}, shardKey: {[metaField]: 1, [timeField]: 1}}); + const newCollName = `${collName}New`; + const newViewNss = `${dbName}.${newCollName}`; + // Rename collection is not supported through view namespace. + assert.commandFailedWithCode(mongo.s.adminCommand({renameCollection: viewNss, to: newViewNss}), + ErrorCodes.NamespaceNotFound); + dropTimeSeriesColl(); +})(); + mongo.stop(); })(); diff --git a/jstests/sharding/timeseries_update.js b/jstests/sharding/timeseries_update.js index 0ec84933dab..96531a09c56 100644 --- a/jstests/sharding/timeseries_update.js +++ b/jstests/sharding/timeseries_update.js @@ -23,8 +23,7 @@ const dbName = 'testDB'; const collName = 'coll'; const timeField = "time"; const metaField = "tag"; -const dateTime1 = ISODate("2021-07-12T16:00:00Z"); -const dateTime2 = ISODate("2021-07-13T16:00:00Z"); +const dateTime = ISODate("2021-07-12T16:00:00Z"); // // Checks for feature flags. @@ -67,29 +66,29 @@ if (!TimeseriesTest.shardedTimeseriesUpdatesAndDeletesEnabled(st.shard0)) { const doc1 = { _id: 1, - [timeField]: dateTime1, + [timeField]: dateTime, [metaField]: {a: "A", b: "B"} }; const doc2 = { _id: 2, - [timeField]: dateTime2, + [timeField]: dateTime, [metaField]: {c: "C", d: 2}, f: [{"k": "K", "v": "V"}] }; const doc3 = { _id: 3, - [timeField]: dateTime1, + [timeField]: dateTime, f: "F" }; const doc4 = { _id: 4, - [timeField]: dateTime1, + [timeField]: dateTime, [metaField]: {a: "A", b: "B"}, f: "F" }; const doc5 = { _id: 5, - [timeField]: dateTime1, + [timeField]: dateTime, [metaField]: {a: "A", b: "B", c: "C"} }; @@ -408,7 +407,7 @@ function testCaseBatchUpdates({testUpdate}) { ], resultDocList: [{ _id: 2, - [timeField]: dateTime1, + [timeField]: dateTime, [metaField]: 3, f: [{"k": "K", "v": "V"}], }], @@ -456,7 +455,7 @@ function testCaseBatchUpdates({testUpdate}) { ], resultDocList: [{ _id: 2, - [timeField]: dateTime1, + [timeField]: dateTime, [metaField]: {c: "C", d: 8}, f: [{"k": "K", "v": "V"}], }], @@ -507,7 +506,7 @@ function testCaseBatchUpdates({testUpdate}) { ], resultDocList: [{ _id: 2, - [timeField]: dateTime2, + [timeField]: dateTime, [metaField]: {c: "C", d: 15}, f: [{"k": "K", "v": "V"}], }], @@ -527,7 +526,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$rename: {[metaField + ".a"]: metaField + ".z"}}, multi: true, }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: {z: "A", b: "B"}}, doc2], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: {z: "A", b: "B"}}, doc2], n: 1, pathToMetaFieldBeingUpdated: "a", }); @@ -540,7 +539,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: {c: "C"}}}, multi: true, }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: {c: "C"}}], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: {c: "C"}}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -555,7 +554,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { }], resultDocList: [ doc1, - {_id: 2, [timeField]: dateTime2, [metaField]: {c: 1, d: 2}, f: [{"k": "K", "v": "V"}]}, + {_id: 2, [timeField]: dateTime, [metaField]: {c: 1, d: 2}, f: [{"k": "K", "v": "V"}]}, doc4, doc5 ], @@ -573,15 +572,10 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { multi: true, }], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: {b: "B"}}, - { - _id: 2, - [timeField]: dateTime1, - [metaField]: {c: "C", d: 2}, - f: [{"k": "K", "v": "V"}] - }, - {_id: 4, [timeField]: dateTime1, [metaField]: {b: "B"}, f: "F"}, - {_id: 5, [timeField]: dateTime1, [metaField]: {b: "B", c: "C"}} + {_id: 1, [timeField]: dateTime, [metaField]: {b: "B"}}, + {_id: 2, [timeField]: dateTime, [metaField]: {c: "C", d: 2}, f: [{"k": "K", "v": "V"}]}, + {_id: 4, [timeField]: dateTime, [metaField]: {b: "B"}, f: "F"}, + {_id: 5, [timeField]: dateTime, [metaField]: {b: "B", c: "C"}} ], ordered: false, n: 3, @@ -596,7 +590,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: {c: "C"}}}, multi: true, }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: {c: "C"}}], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: {c: "C"}}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -609,7 +603,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: {c: "C"}}}, multi: true, }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: {c: "C"}}], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: {c: "C"}}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -622,12 +616,9 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$inc: {[metaField + ".d"]: 10}}, multi: true, }], - resultDocList: [{ - _id: 2, - [timeField]: dateTime2, - [metaField]: {c: "C", d: 12}, - f: [{"k": "K", "v": "V"}] - }], + resultDocList: [ + {_id: 2, [timeField]: dateTime, [metaField]: {c: "C", d: 12}, f: [{"k": "K", "v": "V"}]} + ], n: 1, pathToMetaFieldBeingUpdated: "d", }); @@ -641,8 +632,8 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { multi: true, }], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: {z: "Z"}}, - {_id: 2, [timeField]: dateTime2, [metaField]: {z: "Z"}, f: [{"k": "K", "v": "V"}]} + {_id: 1, [timeField]: dateTime, [metaField]: {z: "Z"}}, + {_id: 2, [timeField]: dateTime, [metaField]: {z: "Z"}, f: [{"k": "K", "v": "V"}]} ], n: 2, pathToMetaFieldBeingUpdated: "", @@ -653,7 +644,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { initialDocList: [doc1], updates: [{q: {[metaField]: {a: "A", b: "B"}}, u: {$unset: {[metaField]: ""}}, multi: true}], - resultDocList: [{_id: 1, [timeField]: dateTime1}], + resultDocList: [{_id: 1, [timeField]: dateTime}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -669,9 +660,9 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { }, ], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: {a: "A", b: "B", c: "C"}}, - {_id: 4, [timeField]: dateTime1, [metaField]: {a: "A", b: "B", c: "C"}, f: "F"}, - {_id: 5, [timeField]: dateTime1, [metaField]: {a: "A", b: "B", c: "C"}} + {_id: 1, [timeField]: dateTime, [metaField]: {a: "A", b: "B", c: "C"}}, + {_id: 4, [timeField]: dateTime, [metaField]: {a: "A", b: "B", c: "C"}, f: "F"}, + {_id: 5, [timeField]: dateTime, [metaField]: {a: "A", b: "B", c: "C"}} ], n: 3, nModified: 2, @@ -687,8 +678,8 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { multi: true }], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: "a"}, - {_id: 2, [timeField]: dateTime2, [metaField]: "a", f: [{"k": "K", "v": "V"}]}, + {_id: 1, [timeField]: dateTime, [metaField]: "a"}, + {_id: 2, [timeField]: dateTime, [metaField]: "a", f: [{"k": "K", "v": "V"}]}, doc3 ], n: 2, @@ -703,7 +694,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: "a"}}, multi: true }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: "a"}, doc2, doc3], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: "a"}, doc2, doc3], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -718,7 +709,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { }, expectFailedUpdate([doc1, doc2, doc3])); - const nestedMetaObj = {_id: 6, [timeField]: dateTime1, [metaField]: {[metaField]: "A", a: 1}}; + const nestedMetaObj = {_id: 6, [timeField]: dateTime, [metaField]: {[metaField]: "A", a: 1}}; // Query for documents using $jsonSchema with the metaField required and a required subfield of // the metaField with the same name as the metaField. @@ -734,7 +725,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: "a"}}, multi: true }], - resultDocList: [doc1, {_id: 6, [timeField]: dateTime1, [metaField]: "a", a: 1}], + resultDocList: [doc1, {_id: 6, [timeField]: dateTime, [metaField]: "a", a: 1}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -768,7 +759,7 @@ function testCaseValidMetaFieldUpdates({testUpdate}) { u: {$set: {[metaField]: "a"}}, multi: true }], - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: "a"}, doc2, doc3], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: "a"}, doc2, doc3], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -810,9 +801,9 @@ function testCaseUpdateWithLetDoc({testUpdate}) { }], letDoc: {oldVal: "A"}, resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: "aaa"}, - {_id: 4, [timeField]: dateTime1, [metaField]: "aaa", f: "F"}, - {_id: 5, [timeField]: dateTime1, [metaField]: "aaa"} + {_id: 1, [timeField]: dateTime, [metaField]: "aaa"}, + {_id: 4, [timeField]: dateTime, [metaField]: "aaa", f: "F"}, + {_id: 5, [timeField]: dateTime, [metaField]: "aaa"} ], n: 3, pathToMetaFieldBeingUpdated: "", @@ -829,7 +820,7 @@ function testCaseUpdateWithLetDoc({testUpdate}) { multi: true, }], letDoc: {myVar: "aaa"}, - resultDocList: [{_id: 1, [timeField]: dateTime1, [metaField]: "$$myVar"}], + resultDocList: [{_id: 1, [timeField]: dateTime, [metaField]: "$$myVar"}], n: 1, pathToMetaFieldBeingUpdated: "", }); @@ -851,9 +842,9 @@ function testCaseUpdateWithLetDoc({testUpdate}) { ], letDoc: {val1: "A", val2: "aaa"}, resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: "bbb"}, - {_id: 4, [timeField]: dateTime1, [metaField]: "bbb", f: "F"}, - {_id: 5, [timeField]: dateTime1, [metaField]: "bbb"} + {_id: 1, [timeField]: dateTime, [metaField]: "bbb"}, + {_id: 4, [timeField]: dateTime, [metaField]: "bbb", f: "F"}, + {_id: 5, [timeField]: dateTime, [metaField]: "bbb"} ], n: 6, pathToMetaFieldBeingUpdated: "", @@ -861,9 +852,9 @@ function testCaseUpdateWithLetDoc({testUpdate}) { } function testCaseCollationUpdates({testUpdate}) { - const collationDoc1 = {_id: 1, [timeField]: dateTime1, [metaField]: "café"}; - const collationDoc2 = {_id: 2, [timeField]: dateTime1, [metaField]: "cafe"}; - const collationDoc3 = {_id: 3, [timeField]: dateTime1, [metaField]: "cafE"}; + const collationDoc1 = {_id: 1, [timeField]: dateTime, [metaField]: "café"}; + const collationDoc2 = {_id: 2, [timeField]: dateTime, [metaField]: "cafe"}; + const collationDoc3 = {_id: 3, [timeField]: dateTime, [metaField]: "cafE"}; const initialDocList = [collationDoc1, collationDoc2, collationDoc3]; // Query on the metaField and modify the metaField using collation with strength level 1. @@ -876,9 +867,9 @@ function testCaseCollationUpdates({testUpdate}) { collation: {locale: "fr", strength: 1}, }], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: "Updated"}, - {_id: 2, [timeField]: dateTime1, [metaField]: "Updated"}, - {_id: 3, [timeField]: dateTime1, [metaField]: "Updated"} + {_id: 1, [timeField]: dateTime, [metaField]: "Updated"}, + {_id: 2, [timeField]: dateTime, [metaField]: "Updated"}, + {_id: 3, [timeField]: dateTime, [metaField]: "Updated"} ], n: 3, pathToMetaFieldBeingUpdated: "", @@ -896,7 +887,7 @@ function testCaseCollationUpdates({testUpdate}) { }], resultDocList: [ collationDoc1, - {_id: 2, [timeField]: dateTime1, [metaField]: "Updated"}, + {_id: 2, [timeField]: dateTime, [metaField]: "Updated"}, collationDoc3, ], n: 1, @@ -906,9 +897,9 @@ function testCaseCollationUpdates({testUpdate}) { function testCaseNullUpdates({testUpdate}) { // Assumes shard key is meta.a. - const nullDoc = {_id: 1, [timeField]: dateTime1, [metaField]: {a: null, b: 1}}; - const missingDoc1 = {_id: 2, [timeField]: dateTime1, [metaField]: {b: 1}}; - const missingDoc2 = {_id: 3, [timeField]: dateTime1, [metaField]: "foo"}; + const nullDoc = {_id: 1, [timeField]: dateTime, [metaField]: {a: null, b: 1}}; + const missingDoc1 = {_id: 2, [timeField]: dateTime, [metaField]: {b: 1}}; + const missingDoc2 = {_id: 3, [timeField]: dateTime, [metaField]: "foo"}; const initialDocList = [nullDoc, missingDoc1, missingDoc2]; // Query on the metaField and modify the metaField using collation with strength level 1. @@ -920,9 +911,9 @@ function testCaseNullUpdates({testUpdate}) { multi: true, }], resultDocList: [ - {_id: 1, [timeField]: dateTime1, [metaField]: "Updated"}, - {_id: 2, [timeField]: dateTime1, [metaField]: "Updated"}, - {_id: 3, [timeField]: dateTime1, [metaField]: "Updated"}, + {_id: 1, [timeField]: dateTime, [metaField]: "Updated"}, + {_id: 2, [timeField]: dateTime, [metaField]: "Updated"}, + {_id: 3, [timeField]: dateTime, [metaField]: "Updated"}, ], n: 3, }); diff --git a/jstests/sharding/top_chunk_autosplit.js b/jstests/sharding/top_chunk_autosplit.js index f189db2e644..7d969aed0c8 100644 --- a/jstests/sharding/top_chunk_autosplit.js +++ b/jstests/sharding/top_chunk_autosplit.js @@ -3,11 +3,8 @@ * of 5MB across all sharding tests in wiredTiger. * @tags: [resource_intensive] */ -(function() { -'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled function shardSetup(shardConfig, dbName, collName) { var st = new ShardingTest(shardConfig); @@ -67,14 +64,14 @@ function runTest(test) { // Add tags to each shard var tags = test.shards[i].tags || []; for (j = 0; j < tags.length; j++) { - st.addShardTag(test.shards[i].name, tags[j]); + sh.addShardTag(test.shards[i].name, tags[j]); } } // Add tag ranges associated to a tag var tagRanges = test.tagRanges || []; for (var j = 0; j < tagRanges.length; j++) { - st.addTagRange(db + "." + collName, + sh.addTagRange(db + "." + collName, {x: tagRanges[j].range.min}, {x: tagRanges[j].range.max}, tagRanges[j].tag); @@ -108,7 +105,7 @@ function runTest(test) { for (var i = 0; i < test.shards.length; i++) { var tags = test.shards[i].tags || []; for (j = 0; j < tags.length; j++) { - st.removeShardTag(test.shards[i].name, tags[j]); + sh.removeShardTag(test.shards[i].name, tags[j]); } } @@ -140,16 +137,6 @@ var collName = "coll"; var st = shardSetup( {name: "topchunk", shards: 4, chunkSize: 1, other: {enableAutoSplit: true}}, dbName, collName); - -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - st.stop(); - return; -} - var db = st.getDB(dbName); var coll = db[collName]; var configDB = st.s.getDB('config'); @@ -393,4 +380,3 @@ if (unsupported.indexOf(st.rs0.getPrimary().adminCommand({serverStatus: 1}).stor } st.stop(); -})(); diff --git a/jstests/sharding/transfer_mods_large_batches.js b/jstests/sharding/transfer_mods_large_batches.js deleted file mode 100644 index 966285024ed..00000000000 --- a/jstests/sharding/transfer_mods_large_batches.js +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Verify the recipient shard continues to run the _transferMods command against the donor shard - * primary until it receives an empty _transferMods batch after the kCommitStart recipient state was - * reached. In particular, a batch of changes unrelated to the chunk migration must not cause the - * recipient shard to stop running the _transferMods command. - * - * @tags: [uses_transactions] - */ -(function() { -"use strict"; - -load('jstests/libs/chunk_manipulation_util.js'); -load('jstests/libs/fail_point_util.js'); -load('jstests/sharding/libs/create_sharded_collection_util.js'); -load('jstests/sharding/libs/sharded_transactions_helpers.js'); - -const staticMongod = MongoRunner.runMongod({}); // Mongod used for startParallelOps(). -const st = new ShardingTest({shards: {rs0: {nodes: 1}, rs1: {nodes: 1}}}); - -const dbName = "test"; -const collName = "transfer_mods_large_batches"; -const collection = st.s.getDB(dbName).getCollection(collName); - -CreateShardedCollectionUtil.shardCollectionWithChunks(collection, {x: 1}, [ - {min: {x: MinKey}, max: {x: 0}, shard: st.shard0.shardName}, - {min: {x: 0}, max: {x: 1000}, shard: st.shard0.shardName}, - {min: {x: 1000}, max: {x: MaxKey}, shard: st.shard1.shardName}, -]); - -function insertLargeDocsInTransaction(collection, docIds, shardKey) { - const lsid = {id: UUID()}; - const txnNumber = 0; - const largeStr = "x".repeat(9 * 1024 * 1024); - - for (let i = 0; i < docIds.length; ++i) { - const docToInsert = {_id: docIds[i]._id}; - Object.assign(docToInsert, shardKey); - docToInsert.note = "large document to force separate _transferMods call"; - docToInsert.padding = largeStr; - - const commandObj = { - documents: [docToInsert], - lsid: lsid, - txnNumber: NumberLong(txnNumber), - autocommit: false - }; - - if (i === 0) { - commandObj.startTransaction = true; - } - - assert.commandWorked(collection.runCommand("insert", commandObj)); - } - - assert.commandWorked(collection.getDB().adminCommand( - {commitTransaction: 1, lsid: lsid, txnNumber: NumberLong(txnNumber), autocommit: false})); -} - -assert.commandWorked(collection.insert([ - {_id: 1, x: -2, note: "keep out of chunk range being migrated"}, - {_id: 2, x: 100, note: "keep in chunk range being migrated"}, -])); - -pauseMoveChunkAtStep(st.shard0, moveChunkStepNames.reachedSteadyState); -const fp = configureFailPoint(st.shard1.rs.getPrimary(), "migrateThreadHangAfterSteadyTransition"); - -const joinMoveChunk = moveChunkParallel( - staticMongod, st.s.host, {x: 1}, undefined, collection.getFullName(), st.shard1.shardName); - -waitForMoveChunkStep(st.shard0, moveChunkStepNames.reachedSteadyState); -insertLargeDocsInTransaction(collection, [{_id: 3}, {_id: 4}], {x: -1000}); -assert.commandWorked( - collection.insert({_id: 5, x: 1, note: "inserted into range after large _transferMods"})); - -unpauseMoveChunkAtStep(st.shard0, moveChunkStepNames.reachedSteadyState); - -// The kCommitStart state isn't a separate "step" of the chunk migration procedure on the -// recipient shard. We therefore cannot use the waitForMigrateStep() helper to wait for the -// _recvChunkCommit command to have been received by the recipient shard. The problematic -// behavior of the recipient shard finishing its catch up too early only manifests after the -// _recvChunkCommit command has been received by the recipient shard. -assert.soon(() => { - const res = assert.commandWorked(fp.conn.adminCommand({_recvChunkStatus: 1})); - return res.state !== "steady"; -}); - -fp.off(); -joinMoveChunk(); - -class ArrayCursor { - constructor(arr) { - this.i = 0; - this.arr = arr; - } - - hasNext() { - return this.i < this.arr.length; - } - - next() { - return this.arr[this.i++]; - } -} - -const expected = new ArrayCursor([ - {_id: 1, x: -2, note: "keep out of chunk range being migrated"}, - {_id: 2, x: 100, note: "keep in chunk range being migrated"}, - {_id: 3, x: -1000, note: "large document to force separate _transferMods call"}, - {_id: 4, x: -1000, note: "large document to force separate _transferMods call"}, - {_id: 5, x: 1, note: "inserted into range after large _transferMods"}, -]); - -const diff = ((diff) => { - return { - docsWithDifferentContents: diff.docsWithDifferentContents.map( - ({first, second}) => ({expected: first, actual: second})), - docsExtraAfterMigration: diff.docsMissingOnFirst, - docsMissingAfterMigration: diff.docsMissingOnSecond, - }; -})( - DataConsistencyChecker.getDiff( - expected, collection.find({}, {_id: 1, x: 1, note: 1}).sort({_id: 1, x: 1}))); - -assert.eq(diff, { - docsWithDifferentContents: [], - docsExtraAfterMigration: [], - docsMissingAfterMigration: [], -}); - -st.stop(); -MongoRunner.stopMongod(staticMongod); -})(); diff --git a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js index 10ec9c91e31..9fb36baeecc 100644 --- a/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js +++ b/jstests/sharding/ttl_deletes_not_targeting_orphaned_documents.js @@ -25,10 +25,10 @@ assert.commandWorked( st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName})); assert.commandWorked(st.s.adminCommand({shardCollection: collName, key: {_id: 1}})); -// Initialize TTL index: delete documents with field `a: <current date>` after 20 seconds -assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 20})); +// Initialize TTL index: delete documents with field `a: <current date>` after 10 seconds +assert.commandWorked(coll.createIndex({a: 1}, {expireAfterSeconds: 10})); -// Insert documents that are going to be deleted in 20 seconds +// Insert documents that are going to be deleted in 10 seconds const currTime = new Date(); var bulk = coll.initializeUnorderedBulkOp(); const nDocs = 100; @@ -41,10 +41,11 @@ assert.commandWorked(bulk.execute()); assert.commandWorked( st.s.adminCommand({moveChunk: collName, find: {_id: 0}, to: st.shard1.shardName})); +// Optimistically wait 15 seconds (10 seconds TTL index delay followed by 5 rounds of TTL monitor) +sleep(15000); + // Verify that TTL index worked properly on owned documents -assert.soon(function() { - return coll.countDocuments({}) == 0; -}, "Failed to move all documents", 60000 /* 60 seconds */, 5000 /* 5 seconds */); +assert.eq(coll.countDocuments({}), 0); // Verify that TTL index did not delete orphaned documents assert.eq(nDocs, st.rs0.getPrimary().getCollection(collName).countDocuments({})); diff --git a/jstests/sharding/txn_recover_decision_using_recovery_router.js b/jstests/sharding/txn_recover_decision_using_recovery_router.js index 62818ab0bb2..09afd9a8a04 100644 --- a/jstests/sharding/txn_recover_decision_using_recovery_router.js +++ b/jstests/sharding/txn_recover_decision_using_recovery_router.js @@ -185,19 +185,8 @@ const waitForCommitTransactionToComplete = function(coordinatorRs, lsid, txnNumb }); }; -let st = new ShardingTest({ - shards: 2, - rs: {nodes: 2}, - rsOptions: { - setParameter: { - // Set this to match the value of transactionLifetimeLimitSeconds to avoid transition - // failure due to not being able to acquire the lock quickly enough. - maxTransactionLockRequestTimeoutMillis: TestData.transactionLifetimeLimitSeconds * 1000 - } - }, - mongos: 2, - other: {mongosOptions: {verbose: 3}} -}); +let st = + new ShardingTest({shards: 2, rs: {nodes: 2}, mongos: 2, other: {mongosOptions: {verbose: 3}}}); // The default WC is majority and this test can't satisfy majority writes. assert.commandWorked(st.s.adminCommand( diff --git a/jstests/sharding/txn_with_several_routers.js b/jstests/sharding/txn_with_several_routers.js index 25af47bbbb9..4dededd0cb0 100644 --- a/jstests/sharding/txn_with_several_routers.js +++ b/jstests/sharding/txn_with_several_routers.js @@ -58,10 +58,6 @@ runTest(() => { autocommit: false, })); - // Refresh the routing table on router1 before running other commands. Since the transaction - // hasn't committed yet there are no documents seen by router1. - assert.eq(0, router1.getDB(dbName)[collName].find().itcount()); - // Try to start a new transaction with the same transaction number on router 1 by inserting // a document onto each shard. assert.commandFailedWithCode(router1.getDB(dbName).runCommand({ diff --git a/jstests/sharding/update_delete_many_metrics.js b/jstests/sharding/update_delete_many_metrics.js deleted file mode 100644 index 3987f9ab156..00000000000 --- a/jstests/sharding/update_delete_many_metrics.js +++ /dev/null @@ -1,163 +0,0 @@ -/** - * Tests for the 'metrics.query' section of the mongos and mongod serverStatus response verifying - * counters for updateMany and deleteMany - * @tags: [multiversion_incompatible] - */ - -(function() { -"use strict"; - -{ - const st = new ShardingTest({shards: 2, rs: {nodes: 2}}); - const mongodConns = []; - st.rs0.nodes.forEach(node => mongodConns.push(node)); - st.rs1.nodes.forEach(node => mongodConns.push(node)); - - const testDB = st.s.getDB("test"); - const shardedColl = testDB.shardedColl; - const unshardedColl = testDB.unshardedColl; - - assert.commandWorked(st.s0.adminCommand({enableSharding: testDB.getName()})); - st.ensurePrimaryShard(testDB.getName(), st.shard0.shardName); - - // Shard shardedColl on {x:1}, split it at {x:0}, and move chunk {x:1} to shard1. - st.shardColl(shardedColl, {x: 1}, {x: 0}, {x: 1}); - - // Insert one document on each shard. - assert.commandWorked(shardedColl.insert({x: 1, _id: 1})); - assert.commandWorked(shardedColl.insert({x: 2, _id: 2})); - assert.commandWorked(shardedColl.insert({x: -1, _id: -1})); - assert.commandWorked(shardedColl.insert({x: -2, _id: -2})); - assert.eq(4, shardedColl.find().itcount()); - - assert.commandWorked(unshardedColl.insert({x: 1, _id: 1})); - assert.commandWorked(unshardedColl.insert({x: -11, _id: 0})); - assert.eq(2, unshardedColl.find().itcount()); - - let mongosServerStatus = testDB.adminCommand({serverStatus: 1}); - - // Verification for initial values. - assert.eq(0, mongosServerStatus.metrics.query.updateManyCount); - assert.eq(0, mongosServerStatus.metrics.query.deleteManyCount); - - assert.commandWorked(unshardedColl.update({_id: 1}, {$set: {a: 2}}, {multi: false})); - assert.eq(1, unshardedColl.find({a: 2}).count()); - - assert.commandWorked(shardedColl.update({_id: 1}, {$set: {a: 2}}, {multi: false})); - assert.eq(1, shardedColl.find({a: 2}).count()); - - // 3 update with multi:true calls. - assert.commandWorked(unshardedColl.update({}, {$set: {a: 3}}, {multi: true})); - assert.eq(2, unshardedColl.find({a: 3}).count()); - assert.commandWorked(shardedColl.update({}, {$set: {a: 3}}, {multi: true})); - assert.eq(0, shardedColl.find({a: 2}).count()); - assert.eq(4, shardedColl.find({a: 3}).count()); - assert.commandWorked(shardedColl.update({}, {$set: {a: 4}}, {multi: true})); - assert.eq(4, shardedColl.find({a: 4}).count()); - - // 2 updateMany calls. - assert.commandWorked(shardedColl.updateMany({}, {$set: {array: 'string', doc: 'string'}})); - assert.commandWorked(unshardedColl.updateMany({}, {$set: {array: 'string', doc: 'string'}})); - - // batch update: 2 more, so 7 updates in total - var request = { - update: shardedColl.getName(), - updates: [{q: {}, u: {$set: {c: 3}}, multi: true}, {q: {}, u: {$set: {a: 5}}, multi: true}], - writeConcern: {w: 1}, - ordered: false - }; - shardedColl.runCommand(request); - assert.eq(4, shardedColl.find({a: 5}).count()); - - // Use deleteMany to delete one of the documents. - const result = shardedColl.deleteMany({_id: 1}); - assert.commandWorked(result); - assert.eq(1, result.deletedCount); - assert.eq(3, shardedColl.find().itcount()); - // Next call will not increase count. - assert.commandWorked(shardedColl.deleteOne({_id: 1})); - - // Use deleteMany to delete one document in the unsharded collection. - assert.commandWorked(unshardedColl.deleteMany({_id: 1})); - // Next call will not increase count. - assert.commandWorked(unshardedColl.deleteOne({_id: 1})); - - mongosServerStatus = testDB.adminCommand({serverStatus: 1}); - // Verification for metrics. - assert.eq(7, mongosServerStatus.metrics.query.updateManyCount); - assert.eq(2, mongosServerStatus.metrics.query.deleteManyCount); - - st.stop(); -} - -{ - const rst = new ReplSetTest({nodes: 2}); - rst.startSet(); - rst.initiate(); - - const primary = rst.getPrimary(); - const testDB = primary.getDB("test"); - const testColl = testDB.coll; - - // Insert documents - assert.commandWorked(testColl.insert({x: 1, _id: 1})); - assert.commandWorked(testColl.insert({x: 2, _id: 2})); - assert.commandWorked(testColl.insert({x: -1, _id: -1})); - assert.commandWorked(testColl.insert({x: -2, _id: -2})); - assert.eq(4, testColl.find().itcount()); - - let mongosServerStatus = testDB.adminCommand({serverStatus: 1}); - - // Verification for initial values. - assert.eq(0, mongosServerStatus.metrics.query.updateManyCount); - assert.eq(0, mongosServerStatus.metrics.query.deleteManyCount); - assert.eq(0, mongosServerStatus.metrics.query.updateDeleteManyDocumentsMaxCount); - assert.eq(0, mongosServerStatus.metrics.query.updateDeleteManyDurationMaxMs); - assert.eq(0, mongosServerStatus.metrics.query.updateDeleteManyDocumentsTotalCount); - assert.eq(0, mongosServerStatus.metrics.query.updateDeleteManyDurationTotalMs); - - assert.commandWorked(testColl.update({_id: 1}, {$set: {a: 2}}, {multi: false})); - assert.eq(1, testColl.find({a: 2}).count()); - // 2 update with multi:true calls. - assert.commandWorked(testColl.update({}, {$set: {a: 3}}, {multi: true})); - assert.eq(0, testColl.find({a: 2}).count()); - assert.eq(4, testColl.find({a: 3}).count()); - assert.commandWorked(testColl.update({}, {$set: {a: 4}}, {multi: true})); - assert.eq(4, testColl.find({a: 4}).count()); - - // 1 updateMany call. - assert.commandWorked(testColl.updateMany({}, {$set: {array: 'string', doc: 'string'}})); - - // batch update: 2 more, so 5 updates in total - var request = { - update: testColl.getName(), - updates: [{q: {}, u: {$set: {c: 3}}, multi: true}, {q: {}, u: {$set: {a: 5}}, multi: true}], - writeConcern: {w: 1}, - ordered: false - }; - testColl.runCommand(request); - assert.eq(4, testColl.find({a: 5}).count()); - - // Use deleteMany to delete two of the documents. - const result = testColl.deleteMany({_id: {$lt: 0}}); - assert.commandWorked(result); - assert.eq(2, result.deletedCount); - assert.eq(2, testColl.find().itcount()); - // Next call will not increase count. - assert.commandWorked(testColl.deleteOne({_id: 1})); - - mongosServerStatus = testDB.adminCommand({serverStatus: 1}); - - // Verification for final metric values. - assert.eq(5, mongosServerStatus.metrics.query.updateManyCount); - assert.eq(1, mongosServerStatus.metrics.query.deleteManyCount); - assert.eq(4, mongosServerStatus.metrics.query.updateDeleteManyDocumentsMaxCount); - assert(mongosServerStatus.metrics.query.hasOwnProperty("updateDeleteManyDurationMaxMs")); - assert.lte(0, mongosServerStatus.metrics.query.updateDeleteManyDurationMaxMs); - assert.eq(22, mongosServerStatus.metrics.query.updateDeleteManyDocumentsTotalCount); - assert(mongosServerStatus.metrics.query.hasOwnProperty("updateDeleteManyDurationTotalMs")); - assert.lte(0, mongosServerStatus.metrics.query.updateDeleteManyDurationTotalMs); - - rst.stopSet(); -} -})(); diff --git a/jstests/sharding/verify_reshard_collection_opmessage.js b/jstests/sharding/verify_reshard_collection_opmessage.js deleted file mode 100644 index 305a75b4643..00000000000 --- a/jstests/sharding/verify_reshard_collection_opmessage.js +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Test that an opMessage is created during reshardCollection command. - * - * @tags: [requires_fcv_60] - */ -(function() { -"use strict"; -load("jstests/libs/uuid_util.js"); -load("jstests/sharding/libs/resharding_test_fixture.js"); - -const sourceNs = "reshardingDb.coll"; - -const reshardingTest = new ReshardingTest({ - numDonors: 2, - numRecipients: 2, - reshardInPlace: true, -}); -reshardingTest.setup(); - -const donorShardNames = reshardingTest.donorShardNames; -const recipientShardNames = reshardingTest.recipientShardNames; - -const primaryShardName = donorShardNames[0]; - -const inputCollection = reshardingTest.createShardedCollection({ - ns: sourceNs, - shardKeyPattern: {oldKey: 1}, - primaryShardName: primaryShardName, - chunks: [ - {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, - {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]}, - ], -}); - -const mongos = inputCollection.getMongo(); -const collectionUUID = getUUIDFromConfigCollections(mongos, sourceNs); - -reshardingTest.withReshardingInBackground({ - newShardKeyPattern: {newKey: 1}, - newChunks: [ - {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]}, - {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]}, - ], -}); - -const oplog = reshardingTest.getReplSetForShard(primaryShardName) - .getPrimary() - .getCollection("local.oplog.rs"); -const logEntry = oplog.findOne({ns: sourceNs, op: 'n', "o2.reshardCollection": sourceNs}); -assert(logEntry != null); -const reshardedUUID = getUUIDFromConfigCollections(mongos, sourceNs); -assert.eq(reshardedUUID, logEntry.o2.reshardUUID, logEntry); -assert.eq(logEntry.ui, collectionUUID, logEntry); -assert.eq(bsonWoCompare(logEntry.o2.oldShardKey, {oldKey: 1}), 0, logEntry); -assert.eq(bsonWoCompare(logEntry.o2.shardKey, {newKey: 1}), 0, logEntry); - -reshardingTest.teardown(); -})(); diff --git a/jstests/sharding/write_cmd_auto_split.js b/jstests/sharding/write_cmd_auto_split.js index c98103a468e..9d7d55a5729 100644 --- a/jstests/sharding/write_cmd_auto_split.js +++ b/jstests/sharding/write_cmd_auto_split.js @@ -9,19 +9,9 @@ 'use strict'; load('jstests/sharding/autosplit_include.js'); load("jstests/sharding/libs/find_chunks_util.js"); -load("jstests/libs/feature_flag_util.js"); // for FeatureFlagUtil.isEnabled var st = new ShardingTest({shards: 1, other: {chunkSize: 1, enableAutoSplit: true}}); -// TODO SERVER-66652 remove this test after 7.0 branches out -const noMoreAutoSplitterFeatureFlag = - FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'), "NoMoreAutoSplitter"); -if (noMoreAutoSplitterFeatureFlag) { - jsTestLog("Skipping as featureFlagNoMoreAutosplitter is enabled"); - st.stop(); - return; -} - var configDB = st.s.getDB('config'); var doc1k = (new Array(1024)).join('x'); diff --git a/jstests/sharding/zone_changes_hashed.js b/jstests/sharding/zone_changes_hashed.js index 5d9323c6f5d..83265fac92f 100644 --- a/jstests/sharding/zone_changes_hashed.js +++ b/jstests/sharding/zone_changes_hashed.js @@ -1,14 +1,9 @@ /** * Test that chunks and documents are moved correctly after zone changes. - * - * * @tags: [ - * requires_fcv_60, - * ] */ (function() { 'use strict'; -load("jstests/libs/feature_flag_util.js"); load("jstests/sharding/libs/zone_changes_util.js"); load("jstests/sharding/libs/find_chunks_util.js"); @@ -52,7 +47,7 @@ function findHighestChunkBounds(chunkBounds) { return highestBounds; } -const st = new ShardingTest({shards: 3, other: {chunkSize: 1, enableAutoSplitter: false}}); +let st = new ShardingTest({shards: 3}); let primaryShard = st.shard0; let dbName = "test"; let testDB = st.s.getDB(dbName); @@ -61,8 +56,8 @@ let coll = testDB.hashed; let ns = coll.getFullName(); let shardKey = {x: "hashed"}; -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: primaryShard.shardName})); +assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); +st.ensurePrimaryShard(dbName, primaryShard.shardName); jsTest.log( "Shard the collection. The command creates two chunks on each of the shards by default."); @@ -71,15 +66,7 @@ let chunkDocs = findChunksUtil.findChunksByNs(configDB, ns).sort({min: 1}).toArr let shardChunkBounds = chunkBoundsUtil.findShardChunkBounds(chunkDocs); jsTest.log("Insert docs (one for each chunk) and check that they end up on the right shards."); -const bigString = 'X'.repeat(1024 * 1024); // 1MB -let docs = [ - {x: -25, s: bigString}, - {x: -18, s: bigString}, - {x: -5, s: bigString}, - {x: -1, s: bigString}, - {x: 5, s: bigString}, - {x: 10, s: bigString} -]; +let docs = [{x: -25}, {x: -18}, {x: -5}, {x: -1}, {x: 5}, {x: 10}]; assert.commandWorked(coll.insert(docs)); let docChunkBounds = []; @@ -139,10 +126,7 @@ shardTags = { }; assertShardTags(configDB, shardTags); -const balanceAccordingToDataSize = FeatureFlagUtil.isEnabled( - st.configRS.getPrimary().getDB('admin'), "BalanceAccordingToDataSize"); -let numChunksToMove = balanceAccordingToDataSize ? zoneChunkBounds["zoneB"].length - 1 - : zoneChunkBounds["zoneB"].length / 2; +let numChunksToMove = zoneChunkBounds["zoneB"].length / 2; runBalancer(st, numChunksToMove); shardChunkBounds = { [st.shard0.shardName]: zoneChunkBounds["zoneB"].slice(0, numChunksToMove), diff --git a/jstests/sharding/zone_changes_range.js b/jstests/sharding/zone_changes_range.js index 78ec7c40dad..2f2963da220 100644 --- a/jstests/sharding/zone_changes_range.js +++ b/jstests/sharding/zone_changes_range.js @@ -7,7 +7,7 @@ load("jstests/sharding/libs/zone_changes_util.js"); load("jstests/sharding/libs/find_chunks_util.js"); -const st = new ShardingTest({shards: 3, other: {chunkSize: 1, enableAutoSplitter: false}}); +let st = new ShardingTest({shards: 3}); let primaryShard = st.shard0; let dbName = "test"; let testDB = st.s.getDB(dbName); @@ -16,8 +16,8 @@ let coll = testDB.range; let ns = coll.getFullName(); let shardKey = {x: 1}; -assert.commandWorked( - st.s.adminCommand({enableSharding: dbName, primaryShard: primaryShard.shardName})); +assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); +st.ensurePrimaryShard(dbName, primaryShard.shardName); jsTest.log("Shard the collection and create chunks."); assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: shardKey})); @@ -26,15 +26,8 @@ assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 0}})); assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 10}})); assert.commandWorked(st.s.adminCommand({split: ns, middle: {x: 20}})); -const bigString = 'X'.repeat(1024 * 1024); // 1MB jsTest.log("Insert docs (one for each chunk) and check that they end up on the primary shard."); -let docs = [ - {x: -15, s: bigString}, - {x: -5, s: bigString}, - {x: 5, s: bigString}, - {x: 15, s: bigString}, - {x: 25, s: bigString} -]; +let docs = [{x: -15}, {x: -5}, {x: 5}, {x: 15}, {x: 25}]; assert.eq(docs.length, findChunksUtil.countChunksForNs(configDB, ns)); assert.commandWorked(coll.insert(docs)); assert.eq(docs.length, primaryShard.getCollection(ns).count()); |