diff options
Diffstat (limited to 'jstests/replsets')
79 files changed, 2310 insertions, 3687 deletions
diff --git a/jstests/replsets/apply_prepare_txn_write_conflict_robustness.js b/jstests/replsets/apply_prepare_txn_write_conflict_robustness.js index 330ebfed751..62e98879d48 100644 --- a/jstests/replsets/apply_prepare_txn_write_conflict_robustness.js +++ b/jstests/replsets/apply_prepare_txn_write_conflict_robustness.js @@ -22,7 +22,8 @@ const primaryDB = primary.getDB(dbName); const primaryColl = primaryDB[collName]; jsTestLog("Do a document write"); -assert.commandWorked(primaryColl.insert({_id: 0}, {"writeConcern": {"w": "majority"}})); +assert.commandWorked( + primaryColl.insert({_id: 0}, {"writeConcern": {"w": "majority"}})); // Enable fail point on secondary to cause apply prepare transaction oplog entry's ops to fail // with write conflict error at least once. @@ -36,21 +37,15 @@ const sessionColl = sessionDB.getCollection(collName); session.startTransaction({writeConcern: {w: "majority"}}); assert.commandWorked(sessionColl.insert({_id: 1})); -const status1 = secondary.adminCommand({serverStatus: 1}); - // PrepareTransaction cmd will be successful only if secondary is able to retry applying // prepareTransaction oplog entry on WT_ROLLBACK (WriteConflictException) error. jsTestLog("Prepare transaction"); let prepareTimestamp = PrepareHelpers.prepareTransaction(session); -// Verify that the writeConflicts metrics in serverStatus is incremented on secondary. -const status2 = secondary.adminCommand({serverStatus: 1}); -assert.gt(status2.metrics.operation.writeConflicts, status1.metrics.operation.writeConflicts); - jsTestLog("Commit transaction"); assert.commandWorked(PrepareHelpers.commitTransaction(session, prepareTimestamp)); -// Verify that the committed transaction data is present on secondary. +// Verify that the committed transaction data is present on secondary. assert.eq(secondary.getDB(dbName)[collName].findOne({_id: 1}), {_id: 1}); // verify that secondaries are not holding any transactional lock resources. diff --git a/jstests/replsets/backwards_compatible_timeseries_catalog_options.js b/jstests/replsets/backwards_compatible_timeseries_catalog_options.js deleted file mode 100644 index af65d49ae8a..00000000000 --- a/jstests/replsets/backwards_compatible_timeseries_catalog_options.js +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Test that timeseriesBucketsMayHaveMixedSchemaData and timeseriesBucketingParametersHaveChanged - * collection options are correctly applied by secondaries and cloned on initial sync according to - * the format introduced under SERVER-91195 (relying on storageEngine.wiredTiger.configString). - * - * @tags: [multiversion_incompatible] - */ -load("jstests/libs/feature_flag_util.js"); // For "FeatureFlagUtil" - -const dbName = "testdb"; -const collName = "testcoll"; -const bucketCollName = 'system.buckets.' + collName; -const bucketNs = dbName + '.' + bucketCollName; - -const rst = new ReplSetTest({name: 'rs', nodes: 2}); -rst.startSet(); -rst.initiateWithHighElectionTimeout(); - -// Create collection on the primary node -const primary = rst.getPrimary(); -const primaryDb = primary.getDB(dbName); -const primaryColl = primaryDb.getCollection(collName); - -assert.commandWorked(primaryDb.createCollection(collName, {timeseries: {timeField: "t"}})); - -// Execute collMod to change the timeseries catalog options under testing -assert.commandWorked(primaryDb.runCommand({ - collMod: collName, - timeseriesBucketsMayHaveMixedSchemaData: true, -})); - -// Double check that the option has been correctly applied on the primary node -const expectedAppMetadata = "app_metadata=(timeseriesBucketsMayHaveMixedSchemaData=true)"; - -const configStringAfterCollMod = - primaryDb.runCommand({listCollections: 1, filter: {name: bucketCollName}}) - .cursor.firstBatch[0] - .options.storageEngine.wiredTiger.configString; - -assert.eq(configStringAfterCollMod, expectedAppMetadata); - -// Add a new node and wait for it to complete initial sync -rst.add({rsConfig: {priority: 0}}); - -rst.reInitiate(); -rst.awaitSecondaryNodes(); -rst.awaitReplication(); - -function assertSameOutputFromDifferentNodes(func) { - let outputs = []; - rst.nodes.forEach(function(node) { - outputs.push(func(node)); - }); - assert.eq(outputs[0], outputs[1]); - assert.eq(outputs[1], outputs[2]); -} - -// Assert that collection options for the view and for the buckets namespace -// are the same on primary, secondary and initial-synced secondary. -assertSameOutputFromDifferentNodes(node => { - return node.getDB(dbName) - .runCommand({listCollections: 1, filter: {name: collName}}) - .cursor.firstBatch[0]; -}); - -assertSameOutputFromDifferentNodes(node => { - return node.getDB(dbName) - .runCommand({listCollections: 1, filter: {name: bucketCollName}}) - .cursor.firstBatch[0]; -}); - -rst.stopSet(); diff --git a/jstests/replsets/catchup_ignores_old_heartbeats.js b/jstests/replsets/catchup_ignores_old_heartbeats.js deleted file mode 100644 index 9f72adbc86a..00000000000 --- a/jstests/replsets/catchup_ignores_old_heartbeats.js +++ /dev/null @@ -1,70 +0,0 @@ -// Makes sure an old heartbeat that is being processed when primary catchup starts does not cause -// primary catchup to think we're already caught up. - -(function() { - -load('jstests/libs/fail_point_util.js'); -load('jstests/replsets/rslib.js'); - -var name = TestData.testName; -var rst = new ReplSetTest({ - name: name, - nodes: 3, - // We're not testing catchup takeover in this test, and in the case where primary catchup fails, - // catchup takeover may cause an additional election and muddle the results. Setting - // catchUpTakeoverDelayMillis to -1 disables catchup takeover. - settings: {chainingAllowed: true, catchUpTakeoverDelayMillis: -1}, - nodeOptions: { - "setParameter": { - "logComponentVerbosity": tojsononeline({"replication": {"verbosity": 2}}), - }, - }, - useBridge: true, - waitForKeys: true -}); - -rst.startSet(); -rst.initiate(); -rst.awaitSecondaryNodes(); - -var primary = rst.getPrimary(); -var primaryColl = primary.getDB("test").coll; - -// The default WC is majority and this test can't test catchup properly if it used majority writes. -assert.commandWorked(primary.adminCommand( - {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}})); - -assert(primary.host == rst.nodes[0].host); -// Make us chain node 1 (the node which will become the new primary) from node 2. Don't allow -// node 1 to switch back. -const forceSyncSource = configureFailPoint( - rst.nodes[1], "forceSyncSourceCandidate", {"hostAndPort": rst.nodes[2].host}); -syncFrom(rst.nodes[2], rst.nodes[0], rst); -syncFrom(rst.nodes[1], rst.nodes[2], rst); -const RBIDBeforeStepUp = assert.commandWorked(primary.adminCommand({replSetGetRBID: 1})); - -// Disconnect the primary from the node syncing from it. -primary.disconnect(rst.nodes[2]); -// Get a heartbeat from the original primary "stuck" in the new primary. -const newPrimary = rst.nodes[1]; -let hbfp = - configureFailPoint(newPrimary, "pauseInHandleHeartbeatResponse", {"target": primary.host}); -hbfp.wait(); -// Put the original primary ahead of the secondaries. -assert.commandWorked(primaryColl.insert({_id: 1})); -jsTestLog("Stepping up new primary"); -assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1})); -// Allow the "stuck" heartbeat to proceed. -hbfp.off(); -// The step-up command waits for the election to complete, but not catch-up. Reconnect the old -// primary to the new primary's sync source to allow replication. -primary.reconnect(rst.nodes[2]); -rst.awaitReplication(); -// The new primary should still be primary. -assert.eq(newPrimary.host, rst.getPrimary().host); -// No rollbacks should have happened. -const RBIDAfterStepUp = assert.commandWorked(primary.adminCommand({replSetGetRBID: 1})); -assert.eq(RBIDBeforeStepUp.rbid, RBIDAfterStepUp.rbid); -forceSyncSource.off(); -rst.stopSet(); -})(); diff --git a/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js b/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js index b0e1f887b60..014dd8fe568 100644 --- a/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js +++ b/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js @@ -47,8 +47,7 @@ assert.commandWorked(coll.insert({_id: 1, v: 1}, {writeConcern: {w: 2}})); // data situation during oplog application. const initialSyncNode = replTest.add({ rsConfig: {priority: 0}, - setParameter: {'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'})}, - oplogSize: oplogSizeMB + setParameter: {'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'})} }); // Wait until the new node starts and pauses on the fail point. @@ -91,10 +90,6 @@ function oplogIsRolledOver() { } while (!oplogIsRolledOver()) { - // Reducing the number of documents inserted to prevent using too much memory. - // The oplog rollover depends on timestamp, sleeping between inserts can significantly reduce - // the total number of documents. - sleep(20); // Insert a large document with a write concern that ensures that before proceeding the // operation gets replicated to all 3 nodes in the replica set, since, otherwise, the node that // is being initial synced may not be able to catchup due to a small size of the oplog. diff --git a/jstests/replsets/commit_point_propagation_with_oplog_exhaust_cursor.js b/jstests/replsets/commit_point_propagation_with_oplog_exhaust_cursor.js deleted file mode 100644 index c90fb94cb82..00000000000 --- a/jstests/replsets/commit_point_propagation_with_oplog_exhaust_cursor.js +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Tests commit point propagation behavior for oplog exhaust cursors. Specifically, we make sure - * commit point propagation is not affected by lastCommittedOpTime temporarily being null (on either - * the sync source node or the syncing node). - * - * @tags: [ - * requires_persistence, - * ] - */ - -(function() { -"use strict"; - -const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); -rst.startSet(); -rst.initiate(); - -const dbName = jsTest.name(); -const collName = "foo"; - -let primary = rst.getPrimary(); -let secondary = rst.getSecondary(); -let primaryColl = primary.getDB(dbName)[collName]; - -// Do an initial write and wait for it to be committed on both nodes to advance the commit point and -// the stable timestamp to reflect the user write. This simulates a healthy replica set. -jsTestLog("Do a document write and wait for the commit point to advance on both nodes"); -assert.commandWorked(primaryColl.insert({_id: -1}, {"writeConcern": {"w": "majority"}})); -rst.awaitLastOpCommitted(); - -let startupParams = {}; -startupParams["logComponentVerbosity"] = tojson({replication: 2}); -startupParams["failpoint.pauseJournalFlusherThread"] = tojson({mode: "alwaysOn"}); - -// Restart both nodes at the same time and pause the JournalFlusher. The lastCommittedOpTime for -// both nodes should be uninitialized after the restart until the first journal flush. -jsTestLog("Shutting down the replica set for restart"); -rst.stopSet(null /* signal */, true /*forRestart */); - -jsTestLog("Restarting the replica set with JournalFlusher thread paused"); -const nodes = rst.startSet({restart: true, setParameter: startupParams}); - -// Step up the first node to speed up the test instead of waiting out the election timeout. -rst.stepUp(nodes[0], {awaitReplicationBeforeStepUp: false}); - -primary = rst.getPrimary(); -secondary = rst.getSecondary(); -primaryColl = primary.getDB(dbName)[collName]; - -// Wait for the restarted secondary to establish an exhaust oplog cursor while both nodes' -// lastCommittedOpTime is null. -jsTestLog("Waiting for the restarted secondary to select a sync source and run an oplog getMore"); -assert.soon( - () => assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})).syncSourceId === 0 && - assert.commandWorked(secondary.adminCommand({serverStatus: 1})) - .metrics.repl.network.getmores.num >= 1, - "Timed out waiting for restarted secondary to fetch oplog"); - -jsTestLog("Resume JournalFlusher thread on both nodes"); -assert.commandWorked( - primary.adminCommand({configureFailPoint: "pauseJournalFlusherThread", mode: "off"})); -assert.commandWorked( - secondary.adminCommand({configureFailPoint: "pauseJournalFlusherThread", mode: "off"})); - -// Record the numEmptyBatches now before the test. -const numEmptyBatchesBefore = - secondary.adminCommand({serverStatus: 1}).metrics.repl.network.getmores.numEmptyBatches; -jsTestLog("numEmptyBatches[Before] on secondary is: " + numEmptyBatchesBefore); - -// Do some more writes, which will advance the commit point on the primary (sync source) for a -// couple of times. -jsTestLog("Do more writes"); -primaryColl = primary.getCollection(primaryColl.getFullName()); -for (let i = 0; i < 10; i++) { - assert.commandWorked(primaryColl.insert({_id: i}, {"writeConcern": {"w": "majority"}})); - sleep(100); -} - -// Test that numEmptyBatches increases after the test as the commit point advances. -const numEmptyBatchesAfter = - secondary.adminCommand({serverStatus: 1}).metrics.repl.network.getmores.numEmptyBatches; -jsTestLog("numEmptyBatches[After] on secondary now is: " + numEmptyBatchesAfter); - -assert(numEmptyBatchesAfter > numEmptyBatchesBefore, - "Expected empty oplog batches for commit point propagation but got none"); - -rst.stopSet(); -})(); diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js index f98dcf0ac2e..02fc2486ca4 100644 --- a/jstests/replsets/db_reads_while_recovering_all_commands.js +++ b/jstests/replsets/db_reads_while_recovering_all_commands.js @@ -38,7 +38,6 @@ const allCommands = { _configsvrCommitChunksMerge: {skip: isPrimaryOnly}, _configsvrCommitChunkMigration: {skip: isPrimaryOnly}, _configsvrCommitChunkSplit: {skip: isPrimaryOnly}, - _configsvrCommitMovePrimary: {skip: isPrimaryOnly}, _configsvrCommitReshardCollection: {skip: isPrimaryOnly}, _configsvrConfigureCollectionBalancing: {skip: isPrimaryOnly}, _configsvrCreateDatabase: {skip: isPrimaryOnly}, @@ -58,7 +57,6 @@ const allCommands = { _configsvrSetClusterParameter: {skip: isPrimaryOnly}, _configsvrSetUserWriteBlockMode: {skip: isPrimaryOnly}, _configsvrUpdateZoneKeyRange: {skip: isPrimaryOnly}, - _dropConnectionsToMongot: {skip: isAnInternalCommand}, _flushDatabaseCacheUpdates: {skip: isPrimaryOnly}, _flushDatabaseCacheUpdatesWithWriteConcern: {skip: isPrimaryOnly}, _flushReshardingStateChange: {skip: isPrimaryOnly}, @@ -72,7 +70,6 @@ const allCommands = { _killOperations: {skip: isNotAUserDataRead}, _mergeAuthzCollections: {skip: isPrimaryOnly}, _migrateClone: {skip: isPrimaryOnly}, - _mongotConnPoolStats: {skip: isAnInternalCommand}, _recvChunkAbort: {skip: isPrimaryOnly}, _recvChunkCommit: {skip: isPrimaryOnly}, _recvChunkReleaseCritSec: {skip: isPrimaryOnly}, @@ -162,7 +159,6 @@ const allCommands = { create: {skip: isPrimaryOnly}, createIndexes: {skip: isPrimaryOnly}, createRole: {skip: isPrimaryOnly}, - createSearchIndexes: {skip: isNotAUserDataRead}, createUser: {skip: isPrimaryOnly}, currentOp: {skip: isNotAUserDataRead}, dataSize: { @@ -200,7 +196,6 @@ const allCommands = { dropDatabase: {skip: isPrimaryOnly}, dropIndexes: {skip: isPrimaryOnly}, dropRole: {skip: isPrimaryOnly}, - dropSearchIndex: {skip: isNotAUserDataRead}, dropUser: {skip: isPrimaryOnly}, echo: {skip: isNotAUserDataRead}, emptycapped: {skip: isPrimaryOnly}, @@ -228,6 +223,7 @@ const allCommands = { getDatabaseVersion: {skip: isNotAUserDataRead}, getDefaultRWConcern: {skip: isNotAUserDataRead}, getDiagnosticData: {skip: isNotAUserDataRead}, + getFreeMonitoringStatus: {skip: isNotAUserDataRead}, getLastError: {skip: isPrimaryOnly}, getLog: {skip: isNotAUserDataRead}, getMore: { @@ -274,7 +270,6 @@ const allCommands = { expectFailure: true, expectedErrorCode: ErrorCodes.NotPrimaryOrSecondary }, - listSearchIndexes: {skip: isNotAUserDataRead}, lockInfo: {skip: isPrimaryOnly}, logApplicationMessage: {skip: isNotAUserDataRead}, logMessage: {skip: isNotAUserDataRead}, @@ -310,6 +305,7 @@ const allCommands = { refreshSessions: {skip: isNotAUserDataRead}, reIndex: {skip: isNotAUserDataRead}, renameCollection: {skip: isPrimaryOnly}, + repairDatabase: {skip: isNotAUserDataRead}, repairShardedCollectionChunksHistory: {skip: isPrimaryOnly}, replSetAbortPrimaryCatchUp: {skip: isNotAUserDataRead}, replSetFreeze: {skip: isNotAUserDataRead}, @@ -342,7 +338,7 @@ const allCommands = { setDefaultRWConcern: {skip: isPrimaryOnly}, setIndexCommitQuorum: {skip: isPrimaryOnly}, setFeatureCompatibilityVersion: {skip: isPrimaryOnly}, - setProfilingFilterGlobally: {skip: isNotAUserDataRead}, + setFreeMonitoring: {skip: isPrimaryOnly}, setParameter: {skip: isNotAUserDataRead}, setShardVersion: {skip: isNotAUserDataRead}, setClusterParameter: {skip: isNotAUserDataRead}, @@ -366,7 +362,6 @@ const allCommands = { top: {skip: isNotAUserDataRead}, update: {skip: isPrimaryOnly}, updateRole: {skip: isPrimaryOnly}, - updateSearchIndex: {skip: isNotAUserDataRead}, updateUser: {skip: isPrimaryOnly}, usersInfo: {skip: isPrimaryOnly}, validate: {skip: isNotAUserDataRead}, diff --git a/jstests/replsets/dbcheck_fails_on_write_concern_error.js b/jstests/replsets/dbcheck_fails_on_write_concern_error.js deleted file mode 100644 index 4e76e9ef1c6..00000000000 --- a/jstests/replsets/dbcheck_fails_on_write_concern_error.js +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Tests dbCheck fails on write concern error. - * - */ -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/write_concern_util.js"); -load("jstests/replsets/libs/dbcheck_utils.js"); - -const dbName = jsTestName(); -const collName = jsTestName(); - -// There should be multiple batches in the dbcheck run, but dbcheck should stop after the first -// batch fails when waiting for write concern. -const nDocs = 100; -const batchSize = 10; - -function runTest(validateMode, writeConcern) { - const rst = new ReplSetTest({ - name: jsTestName(), - nodes: 2, - nodeOptions: { - setParameter: - {logComponentVerbosity: tojson({command: 3}), dbCheckHealthLogEveryNBatches: 1}, - } - }); - rst.startSet(); - rst.initiateWithHighElectionTimeout(); - const primary = rst.getPrimary(); - const secondary = rst.getSecondary(); - const primaryHealthLog = primary.getDB("local").system.healthlog; - const secondaryHealthLog = secondary.getDB("local").system.healthlog; - const primaryDB = primary.getDB(dbName); - const secondaryDB = secondary.getDB(dbName); - - resetAndInsert(rst, primaryDB, collName, nDocs); - assert.commandWorked(primaryDB.runCommand({ - createIndexes: collName, - indexes: [{key: {a: 1}, name: 'a_1'}], - })); - rst.awaitReplication(); - - assert.eq(primaryDB.getCollection(collName).find({}).count(), nDocs); - assert.eq(secondaryDB.getCollection(collName).find({}).count(), nDocs); - clearHealthLog(rst); - - const hangBeforeProcessingDbCheckRunFp = - configureFailPoint(primary, "hangBeforeProcessingDbCheckRun"); - - const hangBeforeAddingDBCheckBatchToOplogFp = - configureFailPoint(primary, "hangBeforeAddingDBCheckBatchToOplog"); - - if (validateMode == "dataConsistencyAndMissingIndexKeysCheck") { - jsTestLog("Running dbCheck dataConsistencyAndMissingIndexKeysCheck"); - runDbCheck(rst, primary.getDB(dbName), collName, { - // validateMode: "dataConsistencyAndMissingIndexKeysCheck", - maxDocsPerBatch: batchSize, - batchWriteConcern: writeConcern, - }); - } - - // TODO SERVER-89817: uncomment when write concern erroring is backported for extra index keys - // check. else if (validateMode == "extraIndexKeysCheck") { - // jsTestLog("Running dbCheck extraIndexKeysCheck"); - // runDbCheck(rst, primary.getDB(dbName), collName, { - // validateMode: "extraIndexKeysCheck", - // secondaryIndex: "a_1", - // maxDocsPerBatch: batchSize, - // batchWriteConcern: writeConcern, - // }); - // } - - hangBeforeProcessingDbCheckRunFp.wait(); - stopReplicationOnSecondaries(rst); - - hangBeforeProcessingDbCheckRunFp.off(); - hangBeforeAddingDBCheckBatchToOplogFp.wait(); - hangBeforeAddingDBCheckBatchToOplogFp.off(); - - // Verify that dbCheck stopped after write concern error. - checkHealthLog(primaryHealthLog, logQueries.writeConcernErrorQuery, 1); - checkHealthLog(primaryHealthLog, logQueries.startStopQuery, 2); - // 1 for start, 1 for the batch, 1 for write concern error, 1 for stop - checkHealthLog(primaryHealthLog, {}, 4); - checkHealthLog(secondaryHealthLog, logQueries.startStopQuery, 1); - checkHealthLog(secondaryHealthLog, {}, 1); - - restartReplicationOnSecondaries(rst); - rst.stopSet(); -} - -["dataConsistencyAndMissingIndexKeysCheck", - // TODO SERVER-89817: uncomment when write concern erroring is backported for extra index keys - // check. "extraIndexKeysCheck" -].forEach((failpointName) => { - runTest(failpointName, { - w: 'majority', - wtimeout: 100, - }); - runTest(failpointName, { - w: 3, - wtimeout: 100, - }); -}); diff --git a/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js b/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js deleted file mode 100644 index e74611ffd1e..00000000000 --- a/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Tests the skipApplyingDbCheckBatchOnSecondary parameter. - */ -load("jstests/libs/feature_flag_util.js"); -load("jstests/replsets/libs/dbcheck_utils.js"); - -const dbName = jsTestName(); -const collName = jsTestName(); - -const nDocs = 20; -const batchSize = 4; - -function runTest(validateMode) { - const rst = new ReplSetTest({ - name: jsTestName(), - nodes: 2, - nodeOptions: { - setParameter: - {logComponentVerbosity: tojson({command: 3}), dbCheckHealthLogEveryNBatches: 1}, - } - }); - rst.startSet(); - rst.initiateWithHighElectionTimeout(); - const primary = rst.getPrimary(); - - if (validateMode == "extraIndexKeysCheck" && - !FeatureFlagUtil.isEnabled(primary.getDB("admin"), "SecondaryIndexChecksInDbCheck")) { - rst.stopSet(); - return; - } - const secondary = rst.getSecondary(); - const primaryHealthLog = primary.getDB("local").system.healthlog; - const secondaryHealthLog = secondary.getDB("local").system.healthlog; - const primaryDB = primary.getDB(dbName); - const secondaryDB = secondary.getDB(dbName); - - assert.commandWorked( - secondary.adminCommand({"setParameter": 1, "skipApplyingDbCheckBatchOnSecondary": true})); - const writeConcern = {w: 'majority'}; - - resetAndInsert(rst, primaryDB, collName, nDocs); - assert.commandWorked(primaryDB.runCommand({ - createIndexes: collName, - indexes: [{key: {a: 1}, name: 'a_1'}], - })); - rst.awaitReplication(); - - assert.eq(primaryDB.getCollection(collName).find({}).count(), nDocs); - assert.eq(secondaryDB.getCollection(collName).find({}).count(), nDocs); - clearHealthLog(rst); - - if (validateMode == "dataConsistencyCheck") { - jsTestLog("Running dbCheck dataConsistencyCheck"); - runDbCheck(rst, - primary.getDB(dbName), - collName, - { - maxDocsPerBatch: batchSize, - batchWriteConcern: writeConcern, - }, - true /*awaitCompletion*/); - } else if (validateMode == "extraIndexKeysCheck") { - jsTestLog("Running dbCheck extraIndexKeysCheck"); - runDbCheck(rst, - primary.getDB(dbName), - collName, - { - validateMode: "extraIndexKeysCheck", - secondaryIndex: "a_1", - maxDocsPerBatch: batchSize, - batchWriteConcern: writeConcern, - }, - true /*awaitCompletion*/); - } - - checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, 0); - checkHealthLog(primaryHealthLog, logQueries.infoBatchQuery, nDocs / batchSize); - - checkHealthLog(secondaryHealthLog, logQueries.startStopQuery, 2); - checkHealthLog( - secondaryHealthLog, logQueries.skipApplyingBatchOnSecondaryQuery, nDocs / batchSize); - checkHealthLog(secondaryHealthLog, logQueries.allErrorsOrWarningsQuery, nDocs / batchSize); - checkHealthLog(secondaryHealthLog, logQueries.infoBatchQuery, 0); - rst.stopSet(); -} - -["extraIndexKeysCheck", - "dataConsistencyCheck", -].forEach((validateMode) => { - runTest(validateMode); -}); diff --git a/jstests/replsets/dbcheck_write_concern.js b/jstests/replsets/dbcheck_write_concern.js index 6e102d7bd17..a87d0bb43d1 100644 --- a/jstests/replsets/dbcheck_write_concern.js +++ b/jstests/replsets/dbcheck_write_concern.js @@ -16,7 +16,7 @@ const replSet = new ReplSetTest({ nodeOptions: {setParameter: {dbCheckHealthLogEveryNBatches: 1}}, settings: { // Prevent the primary from stepping down when we temporarily shut down the secondary. - electionTimeoutMillis: 120000 + electionTimeoutMillis: 60000 } }); replSet.startSet(); @@ -99,5 +99,87 @@ const healthlog = db.getSiblingDB('local').system.healthlog; assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); })(); +// Validate that dbCheck completes with w:majority even when the secondary is down and a wtimeout is +// specified. +(function testWMajorityUnavailable() { + clearLog(); + coll.drop(); + + // Insert 1000 docs and run a few small batches to ensure we wait for write concern between + // each one. + const nDocs = 1000; + const maxDocsPerBatch = 100; + assert.commandWorked(coll.insertMany([...Array(nDocs).keys()].map(x => ({a: x})))); + replSet.awaitReplication(); + + // Stop the secondary and expect that the dbCheck batches still complete on the primary. + const secondaryConn = replSet.getSecondary(); + const secondaryNodeId = replSet.getNodeId(secondaryConn); + replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */}); + + assert.commandWorked(db.runCommand({ + dbCheck: coll.getName(), + maxDocsPerBatch: maxDocsPerBatch, + batchWriteConcern: {w: 'majority', wtimeout: 10}, + })); + + // Confirm dbCheck logs the expected number of batches. + assert.soon(function() { + return (healthlog.find({operation: "dbCheckBatch", severity: "info"}).itcount() == + nDocs / maxDocsPerBatch); + }, "dbCheck doesn't seem to complete", 60 * 1000); + + // Confirm dbCheck logs a warning for every batch. + assert.soon(function() { + return (healthlog.find({operation: "dbCheckBatch", severity: "warning"}).itcount() == + nDocs / maxDocsPerBatch); + }, "dbCheck did not log writeConcern warnings", 60 * 1000); + // There should be no errors. + assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); + + replSet.start(secondaryNodeId, {}, true /*restart*/); + replSet.awaitNodesAgreeOnPrimaryNoAuth(); + replSet.awaitReplication(); +})(); + +// Validate that an invalid 'w' setting still allows dbCheck to succeed when presented with a +// wtimeout. +(function testW3Unavailable() { + clearLog(); + coll.drop(); + + // Insert 1000 docs and run a few small batches to ensure we wait for write concern between + // each one. + const nDocs = 1000; + const maxDocsPerBatch = 100; + assert.commandWorked(coll.insertMany([...Array(nDocs).keys()].map(x => ({a: x})))); + replSet.awaitReplication(); + + // Stop the secondary and expect that the dbCheck batches still complete on the primary. + const secondaryConn = replSet.getSecondary(); + const secondaryNodeId = replSet.getNodeId(secondaryConn); + replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */}); + + assert.commandWorked(db.runCommand({ + dbCheck: coll.getName(), + maxDocsPerBatch: maxDocsPerBatch, + batchWriteConcern: {w: 3, wtimeout: 10}, + })); + + // Confirm dbCheck logs the expected number of batches. + assert.soon(function() { + return (healthlog.find({operation: "dbCheckBatch", severity: "info"}).itcount() == + nDocs / maxDocsPerBatch); + }, "dbCheck doesn't seem to complete", 60 * 1000); + + // Confirm dbCheck logs a warning for every batch. + assert.soon(function() { + return (healthlog.find({operation: "dbCheckBatch", severity: "warning"}).itcount() == + nDocs / maxDocsPerBatch); + }, "dbCheck did not log writeConcern warnings", 60 * 1000); + // There should be no errors. + assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); +})(); + replSet.stopSet(); })(); diff --git a/jstests/replsets/empty_ts_repl.js b/jstests/replsets/empty_ts_repl.js deleted file mode 100644 index 3a2052beb45..00000000000 --- a/jstests/replsets/empty_ts_repl.js +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Tests how replication handles inserts and updates with "Timestamp(0,0)" values. - * - * @tags: [ - * multiversion_incompatible, - * ] - */ -(function() { -"use strict"; - -const dbName = "test"; -const collName = "empty_ts_repl"; - -const rst = new ReplSetTest({ - name: jsTestName(), - nodes: 2, -}); -rst.startSet(); -rst.initiate(); - -const primaryColl = rst.getPrimary().getDB(dbName).getCollection(collName); -const secondaryColl = rst.getSecondary().getDB(dbName).getCollection(collName); -const emptyTs = Timestamp(0, 0); - -// Insert several documents. For the first document inserted (_id=101), the empty timestamp value -// in field "a" should get replaced with the current timestamp. -assert.commandWorked(primaryColl.insert({_id: 101, a: emptyTs})); -assert.commandWorked(primaryColl.insert({_id: 102, a: 1})); -assert.commandWorked(primaryColl.insert({_id: 103, a: 2})); -assert.commandWorked(primaryColl.insert({_id: 106, a: 3})); -assert.commandWorked(primaryColl.insert({_id: 107, a: 4})); -assert.commandWorked(primaryColl.insert({_id: 108, a: 5})); -assert.commandWorked(primaryColl.insert({_id: 109, a: 6})); -assert.commandWorked(primaryColl.insert({_id: 110, a: 7})); -assert.commandWorked(primaryColl.insert({_id: 111, a: 8})); - -// Wait for all the inserted documents to replicate to the secondaries. -rst.awaitReplication(); - -// Use a replacement-style update to update _id=102. This should result in field "a" being set to -// the current timestamp. -assert.commandWorked(primaryColl.update({_id: 102}, {a: emptyTs})); - -// Use a replacement-style findAndModify to update _id=103. This should result in field "a" being -// set to the current timestamp. -let findAndModifyResult = primaryColl.findAndModify({query: {_id: 103}, update: {a: emptyTs}}); -assert.eq(findAndModifyResult, {_id: 103, a: 2}); - -// Do a replacement-style update to add a new document with _id=104. This should result in field "a" -// being set to the current timestamp. -assert.commandWorked(primaryColl.update({_id: 104}, {a: emptyTs}, {upsert: true})); - -// Do a replacement-style findAndModify to add a new document with _id=105. This should result in -// field "a" being set to the current timestamp. -findAndModifyResult = - primaryColl.findAndModify({query: {_id: 105}, update: {a: emptyTs}, upsert: true}); -assert.eq(findAndModifyResult, null); - -// For the rest of the commands below, the empty timestamp values stored in field "a" should be -// preserved as-is. - -// Do an update-operator-style update to update _id=106. -assert.commandWorked(primaryColl.update({_id: 106}, {$set: {a: emptyTs}})); - -// Do an update-operator-style findAndModify to update _id=107. -findAndModifyResult = primaryColl.findAndModify({query: {_id: 107}, update: {$set: {a: emptyTs}}}); -assert.eq(findAndModifyResult, {_id: 107, a: 4}); - -// Do a pipeline-style update to update _id=108. -assert.commandWorked(primaryColl.update({_id: 108}, [{$addFields: {a: emptyTs}}])); - -// Do a pipeline-style findAndModify to update _id=109. -findAndModifyResult = - primaryColl.findAndModify({query: {_id: 109}, update: [{$addFields: {a: emptyTs}}]}); -assert.eq(findAndModifyResult, {_id: 109, a: 6}); - -// Do a pipeline-style update with $internalApplyOplogUpdate to update _id=110. -assert.commandWorked(primaryColl.update( - {_id: 110}, [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}])); - -// Do a pipeline-style findAndModify with $internalApplyOplogUpdate to update _id=111. -findAndModifyResult = primaryColl.findAndModify({ - query: {_id: 111}, - update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}] -}); -assert.eq(findAndModifyResult, {_id: 111, a: 8}); - -// Do an update-operator-style update to add a new document with _id=112. -assert.commandWorked(primaryColl.update({_id: 112}, {$set: {a: emptyTs}}, {upsert: true})); - -// Do an update-operator-style findAndModify to add a new document with _id=113. -findAndModifyResult = - primaryColl.findAndModify({query: {_id: 113}, update: {$set: {a: emptyTs}}, upsert: true}); -assert.eq(findAndModifyResult, null); - -// Do a pipeline-style update to add a new document with _id=114. -assert.commandWorked(primaryColl.update({_id: 114}, [{$addFields: {a: emptyTs}}], {upsert: true})); - -// Do a pipeline-style findAndModify to add a new document with _id=115. -findAndModifyResult = primaryColl.findAndModify( - {query: {_id: 115}, update: [{$addFields: {a: emptyTs}}], upsert: true}); -assert.eq(findAndModifyResult, null); - -// Do a pipeline-style update with $internalApplyOplogUpdate to add a new document _id=116. -assert.commandWorked(primaryColl.update( - {_id: 116}, - [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], - {upsert: true})); - -// Do a pipeline-style findAndModify with $internalApplyOplogUpdate to add a new document _id=117. -findAndModifyResult = primaryColl.findAndModify({ - query: {_id: 117}, - update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], - upsert: true -}); -assert.eq(findAndModifyResult, null); - -rst.awaitReplication(); - -// Verify that all the insert, update, and findAndModify commands behaved the way we expect and that -// they all were replicated correctly to the secondaries. -for (let i = 101; i <= 117; ++i) { - let result = primaryColl.findOne({_id: i}); - let secondaryResult = secondaryColl.findOne({_id: i}); - - assert.eq(tojson(result), tojson(secondaryResult), "_id=" + i); - - if (i >= 106) { - assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + i); - } else { - assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + i); - } -} - -// Insert a document with _id=Timestamp(0,0). -assert.commandWorked(primaryColl.insert({_id: emptyTs, a: 9})); - -// Verify the document we just inserted can be retrieved using the filter "{_id: Timestamp(0,0)}". -let result = primaryColl.findOne({_id: emptyTs}); -assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); -assert.eq(tojson(result.a), tojson(9), "_id=" + tojson(emptyTs)); - -// Do a replacement-style update on the document. -assert.commandWorked(primaryColl.update({_id: emptyTs}, {_id: emptyTs, a: emptyTs})); - -// Verify the document we just updated can still be retrieved using "{_id: Timestamp(0,0)}" and -// verify that field "a" was set to the current timestamp. -result = primaryColl.findOne({_id: emptyTs}); -assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); -assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + tojson(emptyTs)); - -rst.awaitReplication(); - -// Verify that the document was replicated correctly to the secondaries. -let secondaryResult = secondaryColl.findOne({_id: emptyTs}); -assert.eq(tojson(result), tojson(secondaryResult), "_id=" + tojson(emptyTs)); - -rst.stopSet(); -}()); diff --git a/jstests/replsets/ignore_dbcheck_in_initial_sync.js b/jstests/replsets/ignore_dbcheck_in_initial_sync.js deleted file mode 100644 index 8efc8dcabf4..00000000000 --- a/jstests/replsets/ignore_dbcheck_in_initial_sync.js +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Test that dbcheck will be ignored during initial sync. - * - * @tags: [ - * featureFlagSecondaryIndexChecksInDbCheck - * ] - */ - -load("jstests/replsets/libs/dbcheck_utils.js"); -load("jstests/libs/fail_point_util.js"); - -// Skipping data consistency checks because data is inserted into primary and secondary separately. -TestData.skipCollectionAndIndexValidation = true; -TestData.skipCheckDBHashes = true; - -const dbName = "ignore_dbcheck_in_intial_sync"; -const collName = "ignore_dbcheck_in_intial_sync-collection"; - -const doc1 = { - a: 1 -}; - -const replSet = new ReplSetTest({ - name: jsTestName(), - nodes: [ - {}, - { - rsConfig: - // disallow elections on secondary - {priority: 0} - } - ], - nodeOptions: {setParameter: {dbCheckHealthLogEveryNBatches: 1}} -}); -replSet.startSet(); -replSet.initiateWithHighElectionTimeout(); - -const primary = replSet.getPrimary(); -const secondary = replSet.getSecondary(); -const primaryHealthlog = primary.getDB("local").system.healthlog; -const primaryDb = primary.getDB(dbName); -const maxDocsPerBatch = 100; - -jsTestLog("Testing that dbcheck command will be ignored during initial sync."); - -const nDocs = 10; -insertDocsWithMissingIndexKeys(replSet, dbName, collName, doc1, nDocs); - -replSet.awaitReplication(); - -const initialSyncNode = replSet.add({rsConfig: {priority: 0}}); - -const initialSyncHangBeforeSplittingControlFlowFailPoint = - configureFailPoint(initialSyncNode, "initialSyncHangBeforeSplittingControlFlow"); - -replSet.reInitiate(); - -initialSyncHangBeforeSplittingControlFlowFailPoint.wait(); - -// TODO SERVER-89921: Uncomment validateMode once the relevant tickets are backported. -runDbCheck(replSet, - primaryDb, - collName, - { - maxDocsPerBatch: - maxDocsPerBatch //, validateMode: "dataConsistencyAndMissingIndexKeysCheck" - }); - -// Wait until the primary finished dbcheck before turning off the initial sync failpoint so that the -// initial sync node would need to apply (ignore) dbCheckStop in initial sync. -checkHealthLog(primaryHealthlog, {"operation": "dbCheckStop"}, 1); - -assert.commandWorked(initialSyncNode.adminCommand( - {configureFailPoint: 'initialSyncHangBeforeSplittingControlFlow', mode: 'off'})); - -replSet.waitForState(initialSyncNode, ReplSetTest.State.SECONDARY); - -// TODO SERVER-89921: Uncomment all of the following checkHealthLog once the relevant tickets are -// backported. -// Check that the primary logged an error health log entry for each document with missing index key. -// checkHealthLog(primaryHealthlog, logQueries.missingIndexKeysQuery, nDocs); -// Check that the primary does not have other error/warning entries. -// checkHealthLog(primaryHealthlog, logQueries.allErrorsOrWarningsQuery, nDocs); -// Check that the start, batch, and stop entries are warning logs since the node skips dbcheck -// during initial sync. -const initialSyncNodeHealthLog = initialSyncNode.getDB("local").system.healthlog; -checkHealthLog(initialSyncNodeHealthLog, logQueries.duringInitialSyncQuery, 3); -// Check that the initial sync node does not have other info or error/warning entries. -checkHealthLog(initialSyncNodeHealthLog, logQueries.infoBatchQuery, 0); -checkHealthLog(initialSyncNodeHealthLog, logQueries.allErrorsOrWarningsQuery, 3); - -replSet.stopSet(); diff --git a/jstests/replsets/ignore_dbcheck_in_rollback.js b/jstests/replsets/ignore_dbcheck_in_rollback.js deleted file mode 100644 index 239bf0bfc94..00000000000 --- a/jstests/replsets/ignore_dbcheck_in_rollback.js +++ /dev/null @@ -1,104 +0,0 @@ -/* - * This test makes sure the 'dbcheck' command is ignored during rollback and a warning health log - * entry is logged. - * @tags: [ - * featureFlagSecondaryIndexChecksInDbCheck - * ] - */ - -load("jstests/libs/fail_point_util.js"); -load("jstests/replsets/libs/rollback_test.js"); -load("jstests/replsets/libs/dbcheck_utils.js"); - -// This test injects inconsistencies between replica set members; do not fail because of expected -// dbHash differences. -TestData.skipCollectionAndIndexValidation = true; -TestData.skipCheckDBHashes = true; - -const dbName = "ignore_dbcheck_in_rollback"; -const collName = "ignore_dbcheck_in_rollback-collection"; - -const replSet = new ReplSetTest({ - name: jsTestName(), - nodes: [{}, {}, {rsConfig: {priority: 0}}], - useBridge: true, - settings: {chainingAllowed: false} -}); -replSet.startSet(); -replSet.initiateWithHighElectionTimeout(); - -let primary = replSet.getPrimary(); -let primaryDB = primary.getDB(dbName); -const secondary = replSet.getSecondary(); -const secondaryDb = secondary.getDB(dbName); -const primaryColl = primaryDB.getCollection(collName); - -const nDocs = 200; -resetAndInsert(replSet, primaryDB, collName, nDocs); -assert.commandWorked( - primaryDB.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); -replSet.awaitReplication(); -assert.eq(primaryColl.find({}).count(), nDocs); - -// Set up inconsistency. -const skipUnindexingDocumentWhenDeleted = - configureFailPoint(primaryDB, "skipUnindexingDocumentWhenDeleted", {indexName: "a_1"}); -jsTestLog("Deleting docs"); -const stableTimestamp = assert.commandWorked(primaryColl.deleteMany({})); - -replSet.awaitReplication(); -assert.eq(primaryColl.find({}).count(), 0); -assert.eq(secondaryDb.getCollection(collName).find({}).count(), 0); - -const rollbackTest = new RollbackTest(jsTestName(), replSet); -primary = rollbackTest.getPrimary(); -primaryDB = primary.getDB(dbName); - -// Hold stable timestamp. -const stableTimestampFailPoint = configureFailPoint( - primary, "holdStableTimestampAtSpecificTimestamp", {timestamp: stableTimestamp}); - -// TODO SERVER-89921: Uncomment validateMode and secondaryIndex once the relevant tickets are -// backported. -runDbCheck(rollbackTest, - primaryDB, - collName, - { - maxDocsPerBatch: 20 //, validateMode: "extraIndexKeysCheck", secondaryIndex: "a_1" - }); - -// TODO SERVER-89921: Uncomment checkHealthLog once the relevant tickets are backported. -// Check that the old primary prior to transitioning to rollback has start, batch, and stop entries. -const oldPrimaryHealthLog = primary.getDB("local").system.healthlog; -// checkHealthLog(oldPrimaryHealthLog, logQueries.recordNotFoundQuery, nDocs); -checkHealthLog(oldPrimaryHealthLog, logQueries.startStopQuery, 2); - -const rollbackNode = rollbackTest.transitionToRollbackOperations(); -rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); -rollbackTest.transitionToSyncSourceOperationsDuringRollback(); -rollbackTest.transitionToSteadyStateOperations({skipDataConsistencyChecks: true}); - -primary = rollbackTest.getPrimary(); -const primaryHealthLog = primary.getDB("local").system.healthlog; -const rollbackNodeHealthLog = rollbackNode.getDB("local").system.healthlog; - -// TODO SERVER-89921: Change the # of logs to check once the relevant tickets are backported. -// Check that the start, batch (10 batches), and stop entries on the rollback node are all warning -// logs. -checkHealthLog(rollbackNodeHealthLog, logQueries.duringStableRecovery, 3); // 12 -// The rollback node will contain start, batch, and stop entries from when it was primary. Check -// that there are no extra error logs other than recordNotFound. -checkHealthLog(rollbackNodeHealthLog, logQueries.startStopQuery, 2); -// checkHealthLog(rollbackNodeHealthLog, logQueries.recordNotFoundQuery, nDocs); -checkHealthLog(rollbackNodeHealthLog, logQueries.allErrorsOrWarningsQuery, 3); // nDocs + 12 - -// TODO SERVER-89921: Uncomment the following checkHealthLog once the relevant tickets are -// backported. -// Check that the primary only has the batch inconsistent entries from when it was secondary. -// checkHealthLog(primaryHealthLog, logQueries.inconsistentBatchQuery, 10); -// Check that the primary does not have other error/warning entries. -// checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, 10); - -skipUnindexingDocumentWhenDeleted.off(); -stableTimestampFailPoint.off(); -rollbackTest.stop(null /* checkDataConsistencyOptions */, true /* skipDataConsistencyCheck */); diff --git a/jstests/replsets/ignore_dbcheck_in_startup_recovery.js b/jstests/replsets/ignore_dbcheck_in_startup_recovery.js deleted file mode 100644 index ff770e38ec4..00000000000 --- a/jstests/replsets/ignore_dbcheck_in_startup_recovery.js +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Test that dbcheck command will be ignored during startup recovery. - * - * @tags: [ - * featureFlagSecondaryIndexChecksInDbCheck, requires_persistence - * ] - */ - -load("jstests/libs/fail_point_util.js"); -load("jstests/replsets/libs/dbcheck_utils.js"); - -// This test injects inconsistencies between replica set members; do not fail because of expected -// dbHash differences. -TestData.skipCollectionAndIndexValidation = true; -TestData.skipCheckDBHashes = true; - -const dbName = "ignore_dbcheck_in_startup_recovery"; -const collName = "ignore_dbcheck_in_startup_recovery-collection"; - -const replSet = new ReplSetTest({ - name: jsTestName(), - nodes: 2, - nodeOptions: {setParameter: {dbCheckHealthLogEveryNBatches: 1}} -}); -replSet.startSet(); -replSet.initiateWithHighElectionTimeout(); - -let primary = replSet.getPrimary(); -const primaryDb = primary.getDB(dbName); -let secondary = replSet.getSecondary(); -const secondaryDb = secondary.getDB(dbName); -const primaryColl = primaryDb.getCollection(collName); - -const nDocs = 200; -resetAndInsert(replSet, primaryDb, collName, nDocs); -assert.commandWorked( - primaryDb.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); -replSet.awaitReplication(); -assert.eq(primaryColl.find({}).count(), nDocs); - -// Set up inconsistency. -const skipUnindexingDocumentWhenDeleted = - configureFailPoint(primaryDb, "skipUnindexingDocumentWhenDeleted", {indexName: "a_1"}); -jsTestLog("Deleting docs"); -const stableTimestamp = assert.commandWorked(primaryColl.deleteMany({})); - -replSet.awaitReplication(); -assert.eq(primaryColl.find({}).count(), 0); -assert.eq(secondaryDb.getCollection(collName).find({}).count(), 0); - -configureFailPoint(primary, "holdStableTimestampAtSpecificTimestamp", {timestamp: stableTimestamp}); - -// TODO SERVER-89921: Uncomment minKey, maxKey, validateMode and secondaryIndex once the relevant -// tickets are backported. -runDbCheck(replSet, - primaryDb, - collName, - { - maxDocsPerBatch: 20, - batchWriteConcern: {w: 2}, - // minKey: {a: "0"}, - // maxKey: {a: "199"}, - // , validateMode: "extraIndexKeysCheck", secondaryIndex: "a_1" - }, - true /*awaitCompletion*/); - -// Wait to make sure that the secondary applied the stop entry before the ungraceful shutdown. -replSet.awaitNodesAgreeOnAppliedOpTime(); - -// Perform ungraceful shutdown of the secondary node and do not clean the db path directory. -replSet.stop( - 1, 9, {allowedExitCode: MongoRunner.EXIT_SIGKILL}, {forRestart: true, skipValidation: true}); - -secondary = replSet.start(secondary, - { - setParameter: { - "failpoint.stopReplProducer": tojson({"mode": "alwaysOn"}), - }, - }, - {noCleanData: true}); - -primary = replSet.getPrimary(); - -const primaryHealthLog = primary.getDB("local").system.healthlog; -const secondaryHealthLog = secondary.getDB("local").system.healthlog; - -// Check that the start and batch entries are warning logs since the node skips dbcheck during -// startup recovery. -checkHealthLog(secondaryHealthLog, logQueries.duringStableRecovery, 3); -// Check that the initial sync node does not have other info or error/warning entries. -checkHealthLog(secondaryHealthLog, logQueries.infoBatchQuery, 0); -checkHealthLog(secondaryHealthLog, logQueries.allErrorsOrWarningsQuery, 3); - -// TODO SERVER-89921: Uncomment checkHealthLog once the relevant tickets are backported. -// Check that the primary logged an error health log entry for each document with missing index key. -// checkHealthLog(primaryHealthLog, logQueries.recordNotFoundQuery, nDocs); -// Check that the primary does not have other error/warning entries. -// checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, nDocs); - -skipUnindexingDocumentWhenDeleted.off(); -replSet.stopSet(); diff --git a/jstests/replsets/initial_sync1.js b/jstests/replsets/initial_sync1.js index 22f190c89ad..0b2e3ce00b2 100644 --- a/jstests/replsets/initial_sync1.js +++ b/jstests/replsets/initial_sync1.js @@ -58,13 +58,8 @@ admin_s1.runCommand({replSetFreeze: 999999}); print("6. Bring up #3"); var hostname = getHostName(); -var secondary2 = MongoRunner.runMongod(Object.merge({ - replSet: basename, - oplogSize: 2, - // Preserve the initial sync state to validate an assertion. - setParameter: {"failpoint.skipClearInitialSyncState": tojson({mode: 'alwaysOn'})} -}, - x509_options2)); +var secondary2 = + MongoRunner.runMongod(Object.merge({replSet: basename, oplogSize: 2}, x509_options2)); var local_s2 = secondary2.getDB("local"); var admin_s2 = secondary2.getDB("admin"); @@ -118,11 +113,5 @@ assert.commandWorked(bulk.execute()); print("11. Everyone happy eventually"); replTest.awaitReplication(); -// SERVER-69001: Assert that the last oplog for initial sync was persisted in the minvalid document. -let syncingNodeMinvalid = secondary2.getDB("local").replset.minvalid.findOne()["ts"]; -let lastInitialSyncOp = - secondary2.adminCommand("replSetGetStatus")["initialSyncStatus"]["initialSyncOplogEnd"]; -assert.eq(lastInitialSyncOp, syncingNodeMinvalid); - MongoRunner.stopMongod(secondary2); replTest.stopSet(); diff --git a/jstests/replsets/initial_sync_chooses_correct_sync_source.js b/jstests/replsets/initial_sync_chooses_correct_sync_source.js index 5fd6327e575..bc44457e84e 100644 --- a/jstests/replsets/initial_sync_chooses_correct_sync_source.js +++ b/jstests/replsets/initial_sync_chooses_correct_sync_source.js @@ -48,11 +48,6 @@ const restartAndWaitForHeartbeats = (rst, initialSyncNode, setParameterOpts = {} setParameter: setParameterOpts, }); - // Wait for the restarted node to hit initial sync, then wait for heartbeats. This is to - // prevent a potential race where we wait for heartbeats in startup recovery, which satisfies - // the JS test, but then restart heartbeats and treat the other nodes as DOWN when entering - // initial sync. - rst.waitForState(initialSyncNode, ReplSetTest.State.STARTUP_2); waitForHeartbeats(initialSyncNode); }; diff --git a/jstests/replsets/initial_sync_nodes_maintain_and_gossip_commit_point.js b/jstests/replsets/initial_sync_nodes_maintain_and_gossip_commit_point.js index b3e596d1f0d..9b79b672611 100644 --- a/jstests/replsets/initial_sync_nodes_maintain_and_gossip_commit_point.js +++ b/jstests/replsets/initial_sync_nodes_maintain_and_gossip_commit_point.js @@ -118,21 +118,15 @@ assert.eq(1, rs.compareOpTimes(thirdCommitPointSecondary, secondCommitPointSecon hangBeforeCompletingOplogFetching.off(); hangBeforeFinish.wait(); -// Verify that the initial sync node receives the commit point from the primary, either via oplog -// fetching or by a heartbeat. This will usually happen via oplog fetching but in some cases it is -// possible that the OplogFetcher shuts down before this ever happens. See SERVER-76695 for details. +// Verify that the initial sync node receives the commit point from the primary via oplog fetching. // We only assert that it is greater than or equal to the second commit point because it is possible // for the commit point to not yet be advanced by the primary when we fetch the oplog entry. -assert.soon(() => { - const commitPointInitialSyncNode = getLastCommittedOpTime(initialSyncNode); - // compareOpTimes will throw an error if given an invalid opTime, and if the - // node has not yet advanced its opTime it will still have the default one, - // which is invalid. - if (!rs.isValidOpTime(commitPointInitialSyncNode)) { - return false; - } - return rs.compareOpTimes(commitPointInitialSyncNode, secondCommitPointPrimary) >= 0; -}, `commit point on initial sync node should be at least as up-to-date as the second commit point`); +const commitPointInitialSyncNode = getLastCommittedOpTime(initialSyncNode); +assert.gte( + rs.compareOpTimes(commitPointInitialSyncNode, secondCommitPointPrimary), + 0, + `commit point on initial sync node should be at least as up-to-date as the second commit point: ${ + tojson(commitPointInitialSyncNode)}`); // Verify that the non-voting secondary has received the updated commit point via heartbeats from // the initial sync node. diff --git a/jstests/replsets/internal_sessions_reaping_basic.js b/jstests/replsets/internal_sessions_reaping_basic.js index fabadc422b0..ca2dbc06597 100644 --- a/jstests/replsets/internal_sessions_reaping_basic.js +++ b/jstests/replsets/internal_sessions_reaping_basic.js @@ -1,7 +1,11 @@ /** - * Tests that the logical session cache reaper would only reap the config.transactions and - * config.image_collection entries for a transaction session if the logical session that it - * corresponds to has expired and been removed from the config.system.sessions collection. + * Tests that the reaper does not reap expired internal transaction sessions for non-retryable + * writes or non-internal transaction sessions until the logical sessions that they correspond to + * have expired. + * + * Tests that the logical session cache reaper reaps expired internal transaction sessions for old + * retryable writes even when the config.system.sessions entries for the logical sessions that they + * correspond to still exist (i.e. the logical sessions still haven't expired). * * @tags: [requires_fcv_60, uses_transactions] */ @@ -20,12 +24,9 @@ const rst = new ReplSetTest({ nodeOptions: { setParameter: { maxSessions: 1, - // Force batch size 1 on secondaries. - replBatchLimitOperations: 1, // Make transaction records expire immediately. TransactionRecordMinimumLifetimeMinutes: 0, - storeFindAndModifyImagesInSideCollection: true, - internalSessionsReapThreshold: 0 + storeFindAndModifyImagesInSideCollection: true } } }); @@ -96,17 +97,18 @@ let numTransactionsCollEntriesReaped = 0; assert.eq({_id: 0, x: 0, y: 0}, testColl.findOne({_id: 0})); - // Verify that the config.transactions entry for the internal transaction session does not get - // reaped automatically when the transaction committed. + // Verify that the config.transactions entry for the internal transaction session for + // non-retryable write does not get reaped automatically when the transaction committed. assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); assert.eq(1, transactionsColl.find(childLsidFilter).itcount()); assert.eq(1, imageColl.find(parentLsidFilter).itcount()); assert.eq(0, imageColl.find(childLsidFilter).itcount()); - // Force the logical session cache to reap, and verify that the config.transactions (and - // config.image_collection) entries for both transaction sessions do not get reaped because the - // config.system.sessions entry still has not been deleted. + // Force the logical session cache to reap, and verify that the config.transactions entries for + // the internal transaction session for non-retryable write and the non-internal transaction + // session do not get reaped because the config.system.sessions entry still has not been + // deleted. assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1})); assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); @@ -115,8 +117,7 @@ let numTransactionsCollEntriesReaped = 0; assert.eq(0, imageColl.find(childLsidFilter).itcount()); // Delete the config.system.sessions entry, force the logical session cache to reap again, and - // verify that the config.transactions (and config.image_collection) entries for both sessions - // do get reaped this time. + // verify that the config.transactions entries for both sessions do get reaped this time. assert.commandWorked(sessionsColl.remove({})); assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1})); assert.eq(0, sessionsColl.find({"_id.id": sessionUUID}).itcount()); @@ -142,7 +143,6 @@ let numTransactionsCollEntriesReaped = 0; assert.commandWorked(primary.adminCommand({refreshLogicalSessionCacheNow: 1})); assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); - assert.eq(1, imageColl.find(parentLsidFilter).itcount()); parentTxnNumber++; const childLsid = {id: sessionUUID, txnNumber: NumberLong(parentTxnNumber), txnUUID: UUID()}; @@ -172,55 +172,51 @@ let numTransactionsCollEntriesReaped = 0; autocommit: false })); - // Verify that the config.transactions and config.image_collection entries for the internal - // transaction session do not get reaped automatically when the new txnNumber started since - // eager reaping is not enabled. + // Verify that the the config.transactions entry and config.image_collection entry for the + // internal transaction session for the previous retryable write do not get reaped automatically + // when the new txnNumber started. assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); assert.eq(1, transactionsColl.find(childLsidFilter).itcount()); assert.eq(1, imageColl.find(parentLsidFilter).itcount()); assert.eq(1, imageColl.find(childLsidFilter).itcount()); - // Force the logical session cache to reap, and verify that the config.transactions and - // config.image_collection entries for both transaction sessions do not get reaped because the - // config.system.sessions entry still has not been deleted. + // Force the logical session cache to reap, and verify that the config.transactions entry and + // config.image_collection entry for the internal transaction session for the previous + // retryable write do get reaped although the config.system.sessions entry still has not been + // deleted. assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1})); assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); - assert.eq(1, transactionsColl.find(childLsidFilter).itcount()); + assert.eq(0, transactionsColl.find(childLsidFilter).itcount()); assert.eq(1, imageColl.find(parentLsidFilter).itcount()); - assert.eq(1, imageColl.find(childLsidFilter).itcount()); + assert.eq(0, imageColl.find(childLsidFilter).itcount()); + numTransactionsCollEntriesReaped++; assert.commandWorked( testDB.adminCommand(makeCommitTransactionCmdObj(parentLsid, parentTxnNumber))); assert.eq({_id: 1, x: 1, y: 1}, testColl.findOne({_id: 1})); assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); - assert.eq(1, transactionsColl.find(childLsidFilter).itcount()); assert.eq(1, imageColl.find(parentLsidFilter).itcount()); - assert.eq(1, imageColl.find(childLsidFilter).itcount()); - // Force the logical session cache to reap, and verify that the config.transactions and - // config.image_collection entries for both transaction sessions do not get reaped because the - // config.system.sessions entry still has not been deleted. + // Force the logical session cache to reap, and verify that the config.transactions entry and + // config.image_collection entry for the non-internal transaction session do not get reaped + // because the config.system.sessions entry still has not been deleted. assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1})); assert.eq(1, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(1, transactionsColl.find(parentLsidFilter).itcount()); - assert.eq(1, transactionsColl.find(childLsidFilter).itcount()); assert.eq(1, imageColl.find(parentLsidFilter).itcount()); - assert.eq(1, imageColl.find(childLsidFilter).itcount()); // Delete the config.system.sessions entry, force the logical session cache to reap again, and - // verify that the config.transactions and config.image_collection entries for both transaction - // sessions do get reaped this time. + // verify that the config.transactions entry for the expired transaction session does get + // reaped this time. assert.commandWorked(sessionsColl.remove({})); assert.commandWorked(primary.adminCommand({reapLogicalSessionCacheNow: 1})); assert.eq(0, sessionsColl.find({"_id.id": sessionUUID}).itcount()); assert.eq(0, transactionsColl.find(parentLsidFilter).itcount()); - assert.eq(0, transactionsColl.find(childLsidFilter).itcount()); assert.eq(0, imageColl.find(parentLsidFilter).itcount()); - assert.eq(0, imageColl.find(childLsidFilter).itcount()); - numTransactionsCollEntriesReaped += 2; + numTransactionsCollEntriesReaped++; } // Validate that writes to config.transactions do not generate oplog entries, with the exception of diff --git a/jstests/replsets/libs/dbcheck_utils.js b/jstests/replsets/libs/dbcheck_utils.js deleted file mode 100644 index b138322a07e..00000000000 --- a/jstests/replsets/libs/dbcheck_utils.js +++ /dev/null @@ -1,462 +0,0 @@ -/** - * Contains helper functions for testing dbCheck. - */ -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/feature_flag_util.js"); - -const logQueries = { - allErrorsOrWarningsQuery: {$or: [{"severity": "warning"}, {"severity": "error"}]}, - recordNotFoundQuery: { - "severity": "error", - "msg": "found extra index key entry without corresponding document", - "data.context.indexSpec": {$exists: true} - }, - missingIndexKeysQuery: { - "severity": "error", - "msg": "Document has missing index keys", - "data.context.missingIndexKeys": {$exists: true}, - }, - recordDoesNotMatchQuery: { - "severity": "error", - "msg": - "found index key entry with corresponding document/keystring set that does not contain the expected key string", - "data.context.indexSpec": {$exists: true} - }, - collNotFoundWarningQuery: { - severity: "warning", - "msg": "abandoning dbCheck extra index keys check because collection no longer exists" - }, - indexNotFoundWarningQuery: { - severity: "warning", - "msg": "abandoning dbCheck extra index keys check because index no longer exists" - }, - duringInitialSyncQuery: - {severity: "warning", "msg": "cannot execute dbcheck due to ongoing initial sync"}, - duringStableRecovery: - {severity: "warning", "msg": "cannot execute dbcheck due to ongoing stable recovering"}, - errorQuery: {"severity": "error"}, - warningQuery: {"severity": "warning"}, - infoOrErrorQuery: - {$or: [{"severity": "info", "operation": "dbCheckBatch"}, {"severity": "error"}]}, - infoBatchQuery: {"severity": "info", "operation": "dbCheckBatch"}, - inconsistentBatchQuery: {"severity": "error", "msg": "dbCheck batch inconsistent"}, - startStopQuery: { - $or: [ - {"operation": "dbCheckStart", "severity": "info"}, - {"operation": "dbCheckStop", "severity": "info"} - ] - }, - writeConcernErrorQuery: {severity: "error", "msg": "dbCheck failed waiting for writeConcern"}, - skipApplyingBatchOnSecondaryQuery: { - severity: "warning", - "msg": - "skipping applying dbcheck batch because the 'skipApplyingDbCheckBatchOnSecondary' parameter is on", - }, -}; - -// Apply function on all secondary nodes except arbiters. -const forEachNonArbiterSecondary = (replSet, f) => { - for (let secondary of replSet.getSecondaries()) { - if (!secondary.adminCommand({isMaster: 1}).arbiterOnly) { - f(secondary); - } - } -}; - -// Apply function on primary and all secondary nodes. -const forEachNonArbiterNode = (replSet, f) => { - f(replSet.getPrimary()); - forEachNonArbiterSecondary(replSet, f); -}; - -// Clear local.system.healthlog. -const clearHealthLog = (replSet) => { - forEachNonArbiterNode(replSet, conn => conn.getDB("local").system.healthlog.drop()); - replSet.awaitReplication(); -}; - -const logEveryBatch = (replSet) => { - forEachNonArbiterNode(replSet, conn => { - conn.adminCommand({setParameter: 1, "dbCheckHealthLogEveryNBatches": 1}); - }); -}; - -const dbCheckCompleted = (db) => { - const inprog = db.getSiblingDB("admin").currentOp().inprog; - return inprog == undefined || inprog.filter(x => x["desc"] == "dbCheck")[0] === undefined; -}; - -// Wait for dbCheck to complete (on both primaries and secondaries). -const awaitDbCheckCompletion = - (replSet, db, waitForHealthLogDbCheckStop = true, awaitCompletionTimeoutMs = null) => { - assert.soon( - () => dbCheckCompleted(db), - "dbCheck timed out for database: " + db.getName() + " for RS: " + replSet.getURL(), - awaitCompletionTimeoutMs); - - const tokens = replSet.nodes.map(node => node._securityToken); - try { - // This function might be called with a security token (to specify a tenant) on a - // connection. Calling tenant agnostic commands to await replication conflict with this - // token so temporarily remove it. - replSet.nodes.forEach(node => node._setSecurityToken(undefined)); - replSet.awaitSecondaryNodes(); - replSet.awaitReplication(); - - if (waitForHealthLogDbCheckStop) { - forEachNonArbiterNode(replSet, function(node) { - const healthlog = node.getDB('local').system.healthlog; - assert.soon( - function() { - return (healthlog.find({"operation": "dbCheckStop"}).itcount() == 1); - }, - "dbCheck command didn't complete for database: " + db.getName() + - " for RS: " + replSet.getURL()); - }); - } - } finally { - replSet.nodes.forEach((node, idx) => { - node._setSecurityToken(tokens[idx]); - }); - } - }; - -// Clear health log and insert nDocs documents. -const resetAndInsert = (replSet, db, collName, nDocs, docSuffix = null) => { - db[collName].drop(); - clearHealthLog(replSet); - - if (docSuffix) { - assert.commandWorked(db[collName].insertMany( - [...Array(nDocs).keys()].map(x => ({a: x.toString() + docSuffix})), {ordered: false})); - } else { - assert.commandWorked( - db[collName].insertMany([...Array(nDocs).keys()].map(x => ({a: x})), {ordered: false})); - } - - replSet.awaitReplication(); - assert.eq(db.getCollection(collName).find({}).count(), nDocs); -}; - -// Clear health log and insert nDocs documents with two fields `a` and `b`. -const resetAndInsertTwoFields = (replSet, db, collName, nDocs, docSuffix = null) => { - db[collName].drop(); - clearHealthLog(replSet); - - if (docSuffix) { - assert.commandWorked(db[collName].insertMany( - [...Array(nDocs).keys()].map( - x => ({a: x.toString() + docSuffix, b: x.toString() + docSuffix})), - {ordered: false})); - } else { - assert.commandWorked(db[collName].insertMany( - [...Array(nDocs).keys()].map(x => ({a: x, b: x})), {ordered: false})); - } - - replSet.awaitReplication(); - assert.eq(db.getCollection(collName).find({}).count(), nDocs); -}; - -// Clear health log and insert nDocs documents with identical 'a' field -const resetAndInsertIdentical = (replSet, db, collName, nDocs) => { - db[collName].drop(); - clearHealthLog(replSet); - - assert.commandWorked(db[collName].insertMany( - [...Array(nDocs).keys()].map(x => ({_id: x, a: 0})), {ordered: false})); - - replSet.awaitReplication(); - assert.eq(db.getCollection(collName).find({}).count(), nDocs); -}; - -// Insert numDocs documents with missing index keys for testing. -const insertDocsWithMissingIndexKeys = - (replSet, dbName, collName, doc, numDocs = 1, doPrimary = true, doSecondary = true) => { - const primaryDb = replSet.getPrimary().getDB(dbName); - const secondaryDb = replSet.getSecondary().getDB(dbName); - - assert.commandWorked(primaryDb.createCollection(collName)); - - // Create an index for every key in the document. - let index = {}; - for (let key in doc) { - index[key] = 1; - assert.commandWorked(primaryDb[collName].createIndex(index)); - index = {}; - } - replSet.awaitReplication(); - - // dbCheck requires the _id index to iterate through documents in a batch. - let skipIndexNewRecordsExceptIdPrimary; - let skipIndexNewRecordsExceptIdSecondary; - if (doPrimary) { - skipIndexNewRecordsExceptIdPrimary = - configureFailPoint(primaryDb, "skipIndexNewRecords", {skipIdIndex: false}); - } - if (doSecondary) { - skipIndexNewRecordsExceptIdSecondary = - configureFailPoint(secondaryDb, "skipIndexNewRecords", {skipIdIndex: false}); - } - for (let i = 0; i < numDocs; i++) { - assert.commandWorked(primaryDb[collName].insert(doc)); - } - replSet.awaitReplication(); - if (doPrimary) { - skipIndexNewRecordsExceptIdPrimary.off(); - } - if (doSecondary) { - skipIndexNewRecordsExceptIdSecondary.off(); - } - - // Verify that index has been replicated to all nodes, including _id index. - forEachNonArbiterNode(replSet, function(node) { - assert.eq(Object.keys(doc).length + 1, - node.getDB(dbName)[collName].getIndexes().length); - }); - }; - -// Run dbCheck with given parameters and potentially wait for completion. -const runDbCheck = (replSet, - db, - collName, - parameters = {}, - awaitCompletion = false, - waitForHealthLogDbCheckStop = true, - allowedErrorCodes = []) => { - if (!parameters.hasOwnProperty('maxBatchTimeMillis')) { - // Make this huge because stalls and pauses sometimes break this test. - parameters['maxBatchTimeMillis'] = 20000; - } - let dbCheckCommand = {dbCheck: collName}; - for (let parameter in parameters) { - dbCheckCommand[parameter] = parameters[parameter]; - } - - let res = - assert.commandWorkedOrFailedWithCode(db.runCommand(dbCheckCommand), allowedErrorCodes); - if (res.ok && awaitCompletion) { - awaitDbCheckCompletion(replSet, db, waitForHealthLogDbCheckStop); - } -}; - -const checkHealthLog = (healthlog, query, numExpected, timeout = 60 * 1000) => { - let query_count; - assert.soon( - function() { - query_count = healthlog.find(query).count(); - if (query_count != numExpected) { - jsTestLog("health log query returned " + query_count + " entries, expected " + - numExpected + " query: " + tojson(query) + - " found: " + tojson(healthlog.find(query).toArray())); - } - return query_count == numExpected; - }, - "health log query returned " + query_count + " entries, expected " + numExpected + - " query: " + tojson(query) + " found: " + tojson(healthlog.find(query).toArray()) + - " HealthLog: " + tojson(healthlog.find().toArray()), - timeout); -}; - -// Temporarily restart the secondary as a standalone, inject an inconsistency and -// restart it back as a secondary. -const injectInconsistencyOnSecondary = (replSet, dbName, cmd, noCleanData = true) => { - const secondaryConn = replSet.getSecondary(); - const secondaryNodeId = replSet.getNodeId(secondaryConn); - replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */}); - - const standaloneConn = MongoRunner.runMongod({ - dbpath: secondaryConn.dbpath, - noCleanData: noCleanData, - }); - - const standaloneDB = standaloneConn.getDB(dbName); - assert.commandWorked(standaloneDB.runCommand(cmd)); - - // Shut down the secondary and restart it as a member of the replica set. - MongoRunner.stopMongod(standaloneConn); - replSet.start(secondaryNodeId, {}, true /*restart*/); - replSet.awaitNodesAgreeOnPrimaryNoAuth(); -}; - -// Returns a list of all collections in a given database excluding views. -function listCollectionsWithoutViews(database) { - var failMsg = "'listCollections' command failed"; - // Some tests adds an invalid view, resulting in a failure of the 'listCollections' operation - // with an 'InvalidViewDefinition' error. - let res = assert.commandWorkedOrFailedWithCode( - database.runCommand("listCollections"), ErrorCodes.InvalidViewDefinition, failMsg); - if (res.ok) { - return res.cursor.firstBatch.filter(c => c.type == "collection"); - } - return []; -} - -// Returns a list of names of all indexes. -function getIndexNames(db, collName, allowedErrorCodes) { - var failMsg = "'listIndexes' command failed"; - let res = assert.commandWorkedOrFailedWithCode( - db[collName].runCommand("listIndexes"), allowedErrorCodes, failMsg); - if (res.ok) { - return new DBCommandCursor(db, res).toArray().map(spec => spec.name); - } - return []; -} - -// List of collection names that are ignored from dbcheck. -const collNamesIgnoredFromDBCheck = [ - "operationalLatencyHistogramTest_coll_temp", - "top_coll_temp", -]; - -// Run dbCheck for all collections in the database with given parameters and potentially wait for -// completion. -const runDbCheckForDatabase = - (replSet, db, awaitCompletion = false, awaitCompletionTimeoutMs = null) => { - const secondaryIndexCheckEnabled = - checkSecondaryIndexChecksInDbCheckFeatureFlagEnabled(replSet.getPrimary()); - let collDbCheckParameters = {}; - if (secondaryIndexCheckEnabled) { - collDbCheckParameters = {validateMode: "dataConsistencyAndMissingIndexKeysCheck"}; - } - - const allowedErrorCodes = [ - ErrorCodes.NamespaceNotFound /* collection got dropped. */, - ErrorCodes.CommandNotSupportedOnView /* collection got dropped and a view - got created with the same name. */ - , - 40619 /* collection is not replicated error. */, - // Some tests adds an invalid view, resulting in a failure of the 'dbcheck' - // operation with an 'InvalidViewDefinition' error. - ErrorCodes.InvalidViewDefinition, - // Might hit stale shardVersion response from shard config while racing with - // 'dropCollection' command. - ErrorCodes.StaleConfig - ]; - - listCollectionsWithoutViews(db).map(c => c.name).forEach(collName => { - if (collNamesIgnoredFromDBCheck.includes(collName)) { - jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is skipped on ns: " + - db.getName() + "." + collName + " for RS: " + replSet.getURL()); - return; - } - - jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is starting on ns: " + - db.getName() + "." + collName + " for RS: " + replSet.getURL()); - runDbCheck(replSet, - db, - collName, - collDbCheckParameters /* parameters */, - false /* awaitCompletion */, - false /* waitForHealthLogDbCheckStop */, - allowedErrorCodes); - jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is done on ns: " + - db.getName() + "." + collName + " for RS: " + replSet.getURL()); - - if (!secondaryIndexCheckEnabled) { - return; - } - - getIndexNames(db, collName, allowedErrorCodes).forEach(indexName => { - let extraIndexDbCheckParameters = { - validateMode: "extraIndexKeysCheck", - secondaryIndex: indexName - }; - jsTestLog("dbCheck (" + tojson(extraIndexDbCheckParameters) + - ") is starting on ns: " + db.getName() + "." + collName + - " for RS: " + replSet.getURL()); - runDbCheck(replSet, - db, - collName, - extraIndexDbCheckParameters /* parameters */, - false /* awaitCompletion */, - false /* waitForHealthLogDbCheckStop */, - allowedErrorCodes); - jsTestLog("dbCheck (" + tojson(extraIndexDbCheckParameters) + ") is done on ns: " + - db.getName() + "." + collName + " for RS: " + replSet.getURL()); - }); - }); - - if (awaitCompletion) { - awaitDbCheckCompletion( - replSet, db, false /*waitForHealthLogDbCheckStop*/, awaitCompletionTimeoutMs); - } - }; - -// Assert no errors/warnings (i.e., found inconsistencies). Tolerate -// SnapshotTooOld errors, as they can occur if the primary is slow enough processing a -// batch that the secondary is unable to obtain the timestamp the primary used. -const assertForDbCheckErrors = (node, - assertForErrors = true, - assertForWarnings = false, - errorsFound = []) => { - let severityValues = []; - if (assertForErrors == true) { - severityValues.push("error"); - } - - if (assertForWarnings == true) { - severityValues.push("warning"); - } - - const healthlog = node.getDB('local').system.healthlog; - // Regex matching strings that start without "SnapshotTooOld" - const regexStringWithoutSnapTooOld = /^((?!^SnapshotTooOld).)*$/; - - // healthlog is a capped collection, truncation during scan might cause cursor - // invalidation. Truncated data is most likely from previous tests in the fixture, so we - // should still be able to catch errors by retrying. - assert.soon(() => { - try { - let errs = healthlog.find( - {"severity": {$in: severityValues}, "data.error": regexStringWithoutSnapTooOld}); - if (errs.hasNext()) { - const errMsg = "dbCheck found inconsistency on " + node.host; - jsTestLog(errMsg + ". Errors/Warnings: "); - let err; - for (let count = 0; errs.hasNext() && count < 20; count++) { - err = errs.next(); - errorsFound.push(err); - jsTestLog(tojson(err)); - } - assert(false, errMsg); - } - return true; - } catch (e) { - if (e.code !== ErrorCodes.CappedPositionLost) { - throw e; - } - jsTestLog(`Retrying on CappedPositionLost error: ${tojson(e)}`); - return false; - } - }, "healthlog scan could not complete.", 60000); - - jsTestLog("Checked health log for on " + node.host); -}; - -// Check for dbcheck errors for all nodes in a replica set and ignoring arbiters. -const assertForDbCheckErrorsForAllNodes = - (rst, assertForErrors = true, assertForWarnings = false) => { - forEachNonArbiterNode( - rst, node => assertForDbCheckErrors(node, assertForErrors, assertForWarnings)); - }; - -/** - * Utility for checking if the featureFlagSecondaryIndexChecksInDbCheck is on. - */ -function checkSecondaryIndexChecksInDbCheckFeatureFlagEnabled(conn) { - return FeatureFlagUtil.isEnabled(conn.getDB("admin"), 'SecondaryIndexChecksInDbCheck'); -} - -function checkNumSnapshots(debugBuild, expectedNumSnapshots) { - if (debugBuild) { - const actualNumSnapshots = - rawMongoProgramOutput() - .split(/7844808.*Catalog snapshot for reverse lookup check ending/) - .length - - 1; - assert.eq(actualNumSnapshots, - expectedNumSnapshots, - "expected " + expectedNumSnapshots + - " catalog snapshots during reverse lookup, found " + actualNumSnapshots); - } -} diff --git a/jstests/replsets/libs/oplog_rollover_test.js b/jstests/replsets/libs/oplog_rollover_test.js deleted file mode 100644 index 63439f2849f..00000000000 --- a/jstests/replsets/libs/oplog_rollover_test.js +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Test that oplog (on both primary and secondary) rolls over when its size exceeds the configured - * maximum, with parameters for setting the initial sync method and the storage engine. - */ - -"use strict"; - -load("jstests/libs/fail_point_util.js"); - -function oplogRolloverTest(storageEngine, initialSyncMethod) { - jsTestLog("Testing with storageEngine: " + storageEngine); - if (initialSyncMethod) { - jsTestLog(" and initial sync method: " + initialSyncMethod); - } - - // Pause the oplog cap maintainer thread for this test until oplog truncation is needed. The - // truncation thread can hold a mutex for a short period of time which prevents new oplog stones - // from being created during an insertion if the mutex cannot be obtained immediately. Instead, - // the next insertion will attempt to create a new oplog stone, which this test does not do. - let parameters = { - logComponentVerbosity: tojson({storage: 2}), - 'failpoint.hangOplogCapMaintainerThread': tojson({mode: 'alwaysOn'}) - }; - if (initialSyncMethod) { - parameters = Object.merge(parameters, {initialSyncMethod: initialSyncMethod}); - } - const replSet = new ReplSetTest({ - // Set the syncdelay to 1s to speed up checkpointing. - nodeOptions: { - syncdelay: 1, - setParameter: parameters, - }, - nodes: [{}, {rsConfig: {priority: 0, votes: 0}}] - }); - // Set max oplog size to 1MB. - replSet.startSet({storageEngine: storageEngine, oplogSize: 1}); - replSet.initiate(); - - const primary = replSet.getPrimary(); - const primaryOplog = primary.getDB("local").oplog.rs; - const secondary = replSet.getSecondary(); - const secondaryOplog = secondary.getDB("local").oplog.rs; - - // Verify that the oplog cap maintainer thread is paused. - assert.commandWorked(primary.adminCommand({ - waitForFailPoint: "hangOplogCapMaintainerThread", - timesEntered: 1, - maxTimeMS: kDefaultWaitForFailPointTimeout - })); - assert.commandWorked(secondary.adminCommand({ - waitForFailPoint: "hangOplogCapMaintainerThread", - timesEntered: 1, - maxTimeMS: kDefaultWaitForFailPointTimeout - })); - - const coll = primary.getDB("test").foo; - // 400KB each so that oplog can keep at most two insert oplog entries. - const longString = new Array(400 * 1024).join("a"); - - function numInsertOplogEntry(oplog) { - print(`Oplog times for ${oplog.getMongo().host}: ${ - tojsononeline(oplog.find().projection({ts: 1, t: 1, op: 1, ns: 1}).toArray())}`); - return oplog.find({op: "i", "ns": "test.foo"}).itcount(); - } - - // Insert the first document. - const firstInsertTimestamp = - assert - .commandWorked(coll.runCommand( - "insert", {documents: [{_id: 0, longString: longString}], writeConcern: {w: 2}})) - .operationTime; - jsTestLog("First insert timestamp: " + tojson(firstInsertTimestamp)); - - // Test that oplog entry of the first insert exists on both primary and secondary. - assert.eq(1, numInsertOplogEntry(primaryOplog)); - assert.eq(1, numInsertOplogEntry(secondaryOplog)); - - // Insert the second document. - const secondInsertTimestamp = - assert - .commandWorked(coll.runCommand( - "insert", {documents: [{_id: 1, longString: longString}], writeConcern: {w: 2}})) - .operationTime; - jsTestLog("Second insert timestamp: " + tojson(secondInsertTimestamp)); - - // Test that oplog entries of both inserts exist on both primary and secondary. - assert.eq(2, numInsertOplogEntry(primaryOplog)); - assert.eq(2, numInsertOplogEntry(secondaryOplog)); - - // Have a more fine-grained test for enableMajorityReadConcern=true to also test oplog - // truncation happens at the time we expect it to happen. When - // enableMajorityReadConcern=false the lastStableRecoveryTimestamp is not available, so - // switch to a coarser-grained mode to only test that oplog truncation will eventually - // happen when oplog size exceeds the configured maximum. - if (primary.getDB('admin').serverStatus().storageEngine.supportsCommittedReads) { - const awaitCheckpointer = function(timestamp) { - assert.soon( - () => { - const primaryReplSetStatus = - assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})); - const primaryRecoveryTimestamp = - primaryReplSetStatus.lastStableRecoveryTimestamp; - const primaryDurableTimestamp = primaryReplSetStatus.optimes.durableOpTime.ts; - const secondaryReplSetStatus = - assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})); - const secondaryRecoveryTimestamp = - secondaryReplSetStatus.lastStableRecoveryTimestamp; - const secondaryDurableTimestamp = - secondaryReplSetStatus.optimes.durableOpTime.ts; - jsTestLog( - "Awaiting durable & last stable recovery timestamp " + - `(primary last stable recovery: ${tojson(primaryRecoveryTimestamp)}, ` + - `primary durable: ${tojson(primaryDurableTimestamp)}, ` + - `secondary last stable recovery: ${tojson(secondaryRecoveryTimestamp)}, ` + - `secondary durable: ${tojson(secondaryDurableTimestamp)}) ` + - `target: ${tojson(timestamp)}`); - return ((timestampCmp(primaryRecoveryTimestamp, timestamp) >= 0) && - (timestampCmp(primaryDurableTimestamp, timestamp) >= 0) && - (timestampCmp(secondaryDurableTimestamp, timestamp) >= 0) && - (timestampCmp(secondaryRecoveryTimestamp, timestamp) >= 0)); - }, - "Timeout waiting for checkpointing to catch up", - ReplSetTest.kDefaultTimeoutMS, - 2000); - }; - - // Wait for checkpointing/stable timestamp to catch up with the second insert so oplog - // entry of the first insert is allowed to be deleted by the oplog cap maintainer thread - // when a new oplog stone is created. "inMemory" WT engine does not run checkpoint - // thread and lastStableRecoveryTimestamp is the stable timestamp in this case. - awaitCheckpointer(secondInsertTimestamp); - - // Insert the third document which will trigger a new oplog stone to be created. The - // oplog cap maintainer thread will then be unblocked on the creation of the new oplog - // stone and will start truncating oplog entries. The oplog entry for the first - // insert will be truncated after the oplog cap maintainer thread finishes. - const thirdInsertTimestamp = - assert - .commandWorked(coll.runCommand( - "insert", - {documents: [{_id: 2, longString: longString}], writeConcern: {w: 2}})) - .operationTime; - jsTestLog("Third insert timestamp: " + tojson(thirdInsertTimestamp)); - - // There is a race between how we calculate the pinnedOplog and checkpointing. The timestamp - // of the pinnedOplog could be less than the actual stable timestamp used in a checkpoint. - // Wait for the checkpointer to run for another round to make sure the first insert oplog is - // not pinned. - awaitCheckpointer(thirdInsertTimestamp); - - // Verify that there are three oplog entries while the oplog cap maintainer thread is - // paused. - assert.eq(3, numInsertOplogEntry(primaryOplog)); - assert.eq(3, numInsertOplogEntry(secondaryOplog)); - - // Let the oplog cap maintainer thread start truncating the oplog. - assert.commandWorked(primary.adminCommand( - {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); - assert.commandWorked(secondary.adminCommand( - {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); - - // Test that oplog entry of the initial insert rolls over on both primary and secondary. - // Use assert.soon to wait for oplog cap maintainer thread to run. - assert.soon(() => { - return numInsertOplogEntry(primaryOplog) === 2; - }, "Timeout waiting for oplog to roll over on primary"); - assert.soon(() => { - return numInsertOplogEntry(secondaryOplog) === 2; - }, "Timeout waiting for oplog to roll over on secondary"); - - const res = primary.getDB("test").runCommand({serverStatus: 1}); - assert.commandWorked(res); - assert.eq(res.oplogTruncation.truncateCount, 1, tojson(res.oplogTruncation)); - assert.gt(res.oplogTruncation.totalTimeTruncatingMicros, 0, tojson(res.oplogTruncation)); - } else { - // Let the oplog cap maintainer thread start truncating the oplog. - assert.commandWorked(primary.adminCommand( - {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); - assert.commandWorked(secondary.adminCommand( - {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); - - // Only test that oplog truncation will eventually happen. - let numInserted = 2; - assert.soon(function() { - // Insert more documents. - assert.commandWorked( - coll.insert({_id: numInserted++, longString: longString}, {writeConcern: {w: 2}})); - const numInsertOplogEntryPrimary = numInsertOplogEntry(primaryOplog); - const numInsertOplogEntrySecondary = numInsertOplogEntry(secondaryOplog); - // Oplog has been truncated if the number of insert oplog entries is less than - // number of inserted. - if (numInsertOplogEntryPrimary < numInserted && - numInsertOplogEntrySecondary < numInserted) - return true; - jsTestLog("Awaiting oplog truncation: number of oplog entries: " + - `(primary: ${tojson(numInsertOplogEntryPrimary)}, ` + - `secondary: ${tojson(numInsertOplogEntrySecondary)}) ` + - `number inserted: ${numInserted}`); - return false; - }, "Timeout waiting for oplog to roll over", ReplSetTest.kDefaultTimeoutMS, 1000); - } - - replSet.stopSet(); -} diff --git a/jstests/replsets/libs/rollback_resumable_index_build.js b/jstests/replsets/libs/rollback_resumable_index_build.js index 2bd3c28e99d..5420b45ddb0 100644 --- a/jstests/replsets/libs/rollback_resumable_index_build.js +++ b/jstests/replsets/libs/rollback_resumable_index_build.js @@ -79,10 +79,8 @@ const RollbackResumableIndexBuildTest = class { rollbackTest.awaitLastOpCommitted(); - assert.commandWorked(originalPrimary.adminCommand({ - setParameter: 1, - logComponentVerbosity: {index: 1, replication: {election: 0, heartbeats: 0}}, - })); + assert.commandWorked(originalPrimary.adminCommand( + {setParameter: 1, logComponentVerbosity: {index: 1, replication: {heartbeats: 0}}})); // Set internalQueryExecYieldIterations to 0, internalIndexBuildBulkLoadYieldIterations to // 1, and maxIndexBuildDrainBatchSize to 1 so that the index builds are guaranteed to yield diff --git a/jstests/replsets/libs/rollback_test.js b/jstests/replsets/libs/rollback_test.js index d6838deffcf..5e70cd96614 100644 --- a/jstests/replsets/libs/rollback_test.js +++ b/jstests/replsets/libs/rollback_test.js @@ -294,10 +294,6 @@ function RollbackTest(name = "RollbackTest", replSet) { return rst.getPrimary(ReplSetTest.kDefaultTimeoutMS, kRetryIntervalMS); } - this.stepUpNode = function(conn) { - stepUp(conn); - }; - function oplogTop(conn) { return conn.getDB("local").oplog.rs.find().limit(1).sort({$natural: -1}).next(); } @@ -543,11 +539,11 @@ function RollbackTest(name = "RollbackTest", replSet) { return curPrimary; }; - this.stop = function(checkDataConsistencyOptions, skipDataConsistencyCheck = false) { + this.stop = function(checkDataConsistencyOptions) { const start = new Date(); restartServerReplication(tiebreakerNode); rst.awaitReplication(); - if (!doneConsistencyChecks && !skipDataConsistencyCheck) { + if (!doneConsistencyChecks) { this.checkDataConsistency(checkDataConsistencyOptions); } transitionIfAllowed(State.kStopped); diff --git a/jstests/replsets/libs/secondary_reads_test.js b/jstests/replsets/libs/secondary_reads_test.js index 9c91c871b84..4840708dba2 100644 --- a/jstests/replsets/libs/secondary_reads_test.js +++ b/jstests/replsets/libs/secondary_reads_test.js @@ -99,8 +99,8 @@ function SecondaryReadsTest(name = "secondary_reads_test") { assert.gt(readers.length, 0, "no readers to stop"); assert.commandWorked(primaryDB.getCollection(signalColl).insert({_id: testDoneId})); for (let i = 0; i < readers.length; i++) { - const awaitReader = readers[i]; - awaitReader(); + const await = readers[i]; + await (); print("reader " + i + " done"); } readers = []; diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js index b12512b2b8d..b7b81e3f01d 100644 --- a/jstests/replsets/libs/tenant_migration_test.js +++ b/jstests/replsets/libs/tenant_migration_test.js @@ -38,7 +38,6 @@ function TenantMigrationTest({ initiateRstWithHighElectionTimeout = true, quickGarbageCollection = false, insertDataForTenant, - optimizeMigrations = true, }) { const donorPassedIn = (donorRst !== undefined); const recipientPassedIn = (recipientRst !== undefined); @@ -48,15 +47,9 @@ function TenantMigrationTest({ const nodes = sharedOptions.nodes || 2; const setParameterOpts = sharedOptions.setParameter || {}; - if (optimizeMigrations) { - // A tenant migration recipient's `OplogFetcher` uses aggregation which does not support - // tailable awaitdata cursors. For aggregation commands `OplogFetcher` will default to half - // the election timeout (e.g: 5 seconds) between getMores. That wait is largely unnecessary. - setParameterOpts["failpoint.setSmallOplogGetMoreMaxTimeMS"] = tojson({"mode": "alwaysOn"}); - } if (quickGarbageCollection) { - setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 0; - setParameterOpts.ttlMonitorSleepSecs = 1; + setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 3 * 1000; + setParameterOpts.ttlMonitorSleepSecs = 3; } donorRst = donorPassedIn ? donorRst : performSetUp(true /* isDonor */); diff --git a/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js b/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js deleted file mode 100644 index d7a9d8858de..00000000000 --- a/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Tests that the server prints storage statistics along with slow secondary oplog application - * logs. - * - * @tags: [requires_wiredtiger, requires_persistence] - */ - -load("jstests/libs/fail_point_util.js"); -load("jstests/replsets/rslib.js"); - -const name = "log_wt_stats_during_secondary_oplog_application"; -const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); -rst.startSet(); -rst.initiate(); - -let primary = rst.getPrimary(); -let secondary = rst.getSecondary(); - -// Create a collection and write some data to it. -assert.commandWorked(primary.getDB(name).createCollection("readFromDisk")); -assert.commandWorked(primary.getDB(name)["readFromDisk"].insert({x: "value"})); - -// Cleanly shut down the secondary and restart it. This will clear the local cache and ensure -// that we need to read from disk the next time we write to the collection, resulting in -// actual storage statistics instead of empty storage statistics. -rst.stop(secondary, undefined /* signal */, {} /* opts */, {forRestart: true}); -rst.start(secondary, {} /* options */, true /* restart */, false /* waitForHealth */); - -// Ensure secondary completes startup recovery. -secondary = rst.getSecondary(); - -// Set profiling level to 2 so that we log all operations. -assert.commandWorked(secondary.getDB(name).setProfilingLevel(2, 0)); - -// Issue a new write to the primary that will require reading from disk. -assert.commandWorked(primary.getDB(name)["readFromDisk"].insert({x: "sloth"})); -rst.awaitReplication(); - -// Make sure we log that insert op. -const slowLogLine = checkLog.containsLog(secondary, "sloth"); -jsTestLog(slowLogLine); - -// Make sure we've logged a storage statistic as well. -assert(slowLogLine.includes("bytesRead")); - -rst.stopSet(); diff --git a/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js b/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js index c29e7705614..8ba24bdaa9b 100644 --- a/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js +++ b/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js @@ -105,13 +105,7 @@ assert.soon(() => { const syncSourcePingTime = replSetGetStatus.members[0].pingMs; const receivedSyncSourceHb = (syncSourcePingTime > 60); - // Wait for enough heartbeat's from the desired sync source so that our understanding of the - // ping time to that node is at least 'changeSyncSourceThresholdMillis' less than the ping time - // to our current sync source. - const centralSecondaryPingTime = replSetGetStatus.members[1].pingMs; - const exceedsChangeSyncSourceThreshold = (syncSourcePingTime - centralSecondaryPingTime > 5); - - return (receivedCentralHb && receivedSyncSourceHb && exceedsChangeSyncSourceThreshold); + return (receivedCentralHb && receivedSyncSourceHb); }); const replSetGetStatus = assert.commandWorked(testNode.adminCommand({replSetGetStatus: 1})); diff --git a/jstests/replsets/noop_writes_wait_for_write_concern.js b/jstests/replsets/noop_writes_wait_for_write_concern.js index 5bb6a882571..a25794dd12c 100644 --- a/jstests/replsets/noop_writes_wait_for_write_concern.js +++ b/jstests/replsets/noop_writes_wait_for_write_concern.js @@ -9,7 +9,6 @@ "use strict"; load('jstests/libs/write_concern_util.js'); load('jstests/noPassthrough/libs/index_build.js'); -load('jstests/libs/noop_write_commands.js'); var name = 'noop_writes_wait_for_write_concern'; var replTest = new ReplSetTest({ @@ -29,13 +28,246 @@ var dbName = 'testDB'; var db = primary.getDB(dbName); var collName = 'testColl'; var coll = db[collName]; -const commands = getNoopWriteCommands(coll); function dropTestCollection() { coll.drop(); assert.eq(0, coll.find().itcount(), "test collection not empty"); } +// Each entry in this array contains a command whose noop write concern behavior needs to be +// tested. Entries have the following structure: +// { +// req: <object>, // Command request object that will result in a noop +// // write after the setup function is called. +// +// setupFunc: <function()>, // Function to run to ensure that the request is a +// // noop. +// +// confirmFunc: <function(res)>, // Function to run after the command is run to ensure +// // that it executed properly. Accepts the result of +// // the noop request to validate it. +// } +var commands = []; + +// 'applyOps' where the update has already been done. +commands.push({ + req: {applyOps: [{op: "u", ns: coll.getFullName(), o: {_id: 1}, o2: {_id: 1}}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.applied, 1); + assert.eq(res.results[0], true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({_id: 1}), 1); + } +}); + +// 'applyOps' where the preCondition fails. +commands.push({ + req: { + applyOps: [{op: "i", ns: coll.getFullName(), o: {_id: 2}}], + preCondition: [{ns: coll.getFullName(), q: {_id: 99}, res: {_id: 99}}] + }, + setupFunc: function() { + assert.commandWorked(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandFailed(res, + "The applyOps command was expected to fail, but instead succeeded."); + assert.eq( + res.errmsg, "preCondition failed", "The applyOps command failed for the wrong reason."); + } +}); + +// 'update' where the document to update does not exist. +commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {b: 2}}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorked(coll.update({a: 1}, {b: 2})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.n, 0); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } +}); + +// 'update' where the update has already been done. +commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {$set: {b: 2}}}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorked(coll.update({a: 1}, {$set: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.n, 1); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } +}); + +// 'update' with immutable field error. +commands.push({ + req: {update: collName, updates: [{q: {_id: 1}, u: {$set: {_id: 2}}}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.eq(res.n, 0); + assert.eq(res.nModified, 0); + assert.eq(coll.count({_id: 1}), 1); + } +}); + +// 'delete' where the document to delete does not exist. +commands.push({ + req: {delete: collName, deletes: [{q: {a: 1}, limit: 1}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorked(coll.remove({a: 1})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.n, 0); + assert.eq(coll.count({a: 1}), 0); + } +}); + +// 'createIndexes' where the index has already been created. +// All voting data bearing nodes are not up for this test. So 'createIndexes' command can't succeed +// with the default index commitQuorum value "votingMembers". So, running createIndexes cmd using +// commit quorum "majority". +commands.push({ + req: {createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}], commitQuorum: "majority"}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorkedIgnoringWriteConcernErrors(db.runCommand({ + createIndexes: collName, + indexes: [{key: {a: 1}, name: "a_1"}], + commitQuorum: "majority" + })); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.numIndexesBefore, res.numIndexesAfter); + assert.eq(res.note, 'all indexes already exist'); + } +}); + +// 'findAndModify' where the document to update does not exist. +commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {b: 2}}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorkedIgnoringWriteConcernErrors( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.lastErrorObject.updatedExisting, false); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } +}); + +// 'findAndModify' where the update has already been done. +commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorkedIgnoringWriteConcernErrors( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.lastErrorObject.updatedExisting, true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } +}); + +// 'findAndModify' with immutable field error. +commands.push({ + req: {findAndModify: collName, query: {_id: 1}, update: {$set: {_id: 2}}}, + setupFunc: function() { + assert.commandWorked(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.ImmutableField); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({_id: 1}), 1); + } +}); + +// 'findAndModify' where the document to delete does not exist. +commands.push({ + req: {findAndModify: collName, query: {a: 1}, remove: true}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorked(coll.remove({a: 1})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + assert.eq(res.lastErrorObject.n, 0); + } +}); + +// 'dropDatabase' where the database has already been dropped. +commands.push({ + req: {dropDatabase: 1}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorkedIgnoringWriteConcernErrors(db.runCommand({dropDatabase: 1})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteConcernErrors(res); + } +}); + +// 'drop' where the collection has already been dropped. +commands.push({ + req: {drop: collName}, + setupFunc: function() { + assert.commandWorked(coll.insert({a: 1})); + assert.commandWorkedIgnoringWriteConcernErrors(db.runCommand({drop: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceNotFound); + } +}); + +// 'create' where the collection has already been created. +commands.push({ + req: {create: collName}, + setupFunc: function() { + assert.commandWorkedIgnoringWriteConcernErrors(db.runCommand({create: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceExists); + } +}); + +// 'insert' where the document with the same _id has already been inserted. +commands.push({ + req: {insert: collName, documents: [{_id: 1}]}, + setupFunc: function() { + assert.commandWorked(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorkedIgnoringWriteErrorsAndWriteConcernErrors(res); + assert.eq(res.n, 0); + assert.eq(res.writeErrors[0].code, ErrorCodes.DuplicateKey); + assert.eq(coll.count({_id: 1}), 1); + } +}); + function testCommandWithWriteConcern(cmd) { // Provide a small wtimeout that we expect to time out. cmd.req.writeConcern = {w: 3, wtimeout: 1000}; diff --git a/jstests/replsets/oplog_rollover.js b/jstests/replsets/oplog_rollover.js index f2e3ef323ba..81115231c3f 100644 --- a/jstests/replsets/oplog_rollover.js +++ b/jstests/replsets/oplog_rollover.js @@ -5,9 +5,188 @@ (function() { "use strict"; -load("jstests/replsets/libs/oplog_rollover_test.js"); +load("jstests/libs/fail_point_util.js"); -oplogRolloverTest("wiredTiger"); +function doTest(storageEngine) { + jsTestLog("Testing with storageEngine: " + storageEngine); + + // Pause the oplog cap maintainer thread for this test until oplog truncation is needed. The + // truncation thread can hold a mutex for a short period of time which prevents new oplog stones + // from being created during an insertion if the mutex cannot be obtained immediately. Instead, + // the next insertion will attempt to create a new oplog stone, which this test does not do. + const replSet = new ReplSetTest({ + // Set the syncdelay to 1s to speed up checkpointing. + nodeOptions: { + syncdelay: 1, + setParameter: { + logComponentVerbosity: tojson({storage: 2}), + 'failpoint.hangOplogCapMaintainerThread': tojson({mode: 'alwaysOn'}) + } + }, + nodes: [{}, {rsConfig: {priority: 0, votes: 0}}] + }); + // Set max oplog size to 1MB. + replSet.startSet({storageEngine: storageEngine, oplogSize: 1}); + replSet.initiate(); + + const primary = replSet.getPrimary(); + const primaryOplog = primary.getDB("local").oplog.rs; + const secondary = replSet.getSecondary(); + const secondaryOplog = secondary.getDB("local").oplog.rs; + + // Verify that the oplog cap maintainer thread is paused. + assert.commandWorked(primary.adminCommand({ + waitForFailPoint: "hangOplogCapMaintainerThread", + timesEntered: 1, + maxTimeMS: kDefaultWaitForFailPointTimeout + })); + assert.commandWorked(secondary.adminCommand({ + waitForFailPoint: "hangOplogCapMaintainerThread", + timesEntered: 1, + maxTimeMS: kDefaultWaitForFailPointTimeout + })); + + const coll = primary.getDB("test").foo; + // 400KB each so that oplog can keep at most two insert oplog entries. + const longString = new Array(400 * 1024).join("a"); + + function numInsertOplogEntry(oplog) { + print(`Oplog times for ${oplog.getMongo().host}: ${ + tojsononeline(oplog.find().projection({ts: 1, t: 1, op: 1, ns: 1}).toArray())}`); + return oplog.find({op: "i", "ns": "test.foo"}).itcount(); + } + + // Insert the first document. + const firstInsertTimestamp = + assert + .commandWorked(coll.runCommand( + "insert", {documents: [{_id: 0, longString: longString}], writeConcern: {w: 2}})) + .operationTime; + jsTestLog("First insert timestamp: " + tojson(firstInsertTimestamp)); + + // Test that oplog entry of the first insert exists on both primary and secondary. + assert.eq(1, numInsertOplogEntry(primaryOplog)); + assert.eq(1, numInsertOplogEntry(secondaryOplog)); + + // Insert the second document. + const secondInsertTimestamp = + assert + .commandWorked(coll.runCommand( + "insert", {documents: [{_id: 1, longString: longString}], writeConcern: {w: 2}})) + .operationTime; + jsTestLog("Second insert timestamp: " + tojson(secondInsertTimestamp)); + + // Test that oplog entries of both inserts exist on both primary and secondary. + assert.eq(2, numInsertOplogEntry(primaryOplog)); + assert.eq(2, numInsertOplogEntry(secondaryOplog)); + + // Have a more fine-grained test for enableMajorityReadConcern=true to also test oplog + // truncation happens at the time we expect it to happen. When + // enableMajorityReadConcern=false the lastStableRecoveryTimestamp is not available, so + // switch to a coarser-grained mode to only test that oplog truncation will eventually + // happen when oplog size exceeds the configured maximum. + if (primary.getDB('admin').serverStatus().storageEngine.supportsCommittedReads) { + const awaitCheckpointer = function(timestamp) { + assert.soon( + () => { + const primaryTimestamp = + assert.commandWorked(primary.adminCommand({replSetGetStatus: 1})) + .lastStableRecoveryTimestamp; + const secondaryTimestamp = + assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})) + .lastStableRecoveryTimestamp; + jsTestLog("Awaiting last stable recovery timestamp " + + `(primary: ${tojson(primaryTimestamp)}, secondary: ${ + tojson(secondaryTimestamp)}) ` + + `target: ${tojson(timestamp)}`); + return ((timestampCmp(primaryTimestamp, timestamp) >= 0) && + (timestampCmp(secondaryTimestamp, timestamp) >= 0)); + }, + "Timeout waiting for checkpointing to catch up", + ReplSetTest.kDefaultTimeoutMS, + 2000); + }; + + // Wait for checkpointing/stable timestamp to catch up with the second insert so oplog + // entry of the first insert is allowed to be deleted by the oplog cap maintainer thread + // when a new oplog stone is created. "inMemory" WT engine does not run checkpoint + // thread and lastStableRecoveryTimestamp is the stable timestamp in this case. + awaitCheckpointer(secondInsertTimestamp); + + // Insert the third document which will trigger a new oplog stone to be created. The + // oplog cap maintainer thread will then be unblocked on the creation of the new oplog + // stone and will start truncating oplog entries. The oplog entry for the first + // insert will be truncated after the oplog cap maintainer thread finishes. + const thirdInsertTimestamp = + assert + .commandWorked(coll.runCommand( + "insert", + {documents: [{_id: 2, longString: longString}], writeConcern: {w: 2}})) + .operationTime; + jsTestLog("Third insert timestamp: " + tojson(thirdInsertTimestamp)); + + // There is a race between how we calculate the pinnedOplog and checkpointing. The timestamp + // of the pinnedOplog could be less than the actual stable timestamp used in a checkpoint. + // Wait for the checkpointer to run for another round to make sure the first insert oplog is + // not pinned. + awaitCheckpointer(thirdInsertTimestamp); + + // Verify that there are three oplog entries while the oplog cap maintainer thread is + // paused. + assert.eq(3, numInsertOplogEntry(primaryOplog)); + assert.eq(3, numInsertOplogEntry(secondaryOplog)); + + // Let the oplog cap maintainer thread start truncating the oplog. + assert.commandWorked(primary.adminCommand( + {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); + + // Test that oplog entry of the initial insert rolls over on both primary and secondary. + // Use assert.soon to wait for oplog cap maintainer thread to run. + assert.soon(() => { + return numInsertOplogEntry(primaryOplog) === 2; + }, "Timeout waiting for oplog to roll over on primary"); + assert.soon(() => { + return numInsertOplogEntry(secondaryOplog) === 2; + }, "Timeout waiting for oplog to roll over on secondary"); + + const res = primary.getDB("test").runCommand({serverStatus: 1}); + assert.commandWorked(res); + assert.eq(res.oplogTruncation.truncateCount, 1, tojson(res.oplogTruncation)); + assert.gt(res.oplogTruncation.totalTimeTruncatingMicros, 0, tojson(res.oplogTruncation)); + } else { + // Let the oplog cap maintainer thread start truncating the oplog. + assert.commandWorked(primary.adminCommand( + {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"})); + + // Only test that oplog truncation will eventually happen. + let numInserted = 2; + assert.soon(function() { + // Insert more documents. + assert.commandWorked( + coll.insert({_id: numInserted++, longString: longString}, {writeConcern: {w: 2}})); + const numInsertOplogEntryPrimary = numInsertOplogEntry(primaryOplog); + const numInsertOplogEntrySecondary = numInsertOplogEntry(secondaryOplog); + // Oplog has been truncated if the number of insert oplog entries is less than + // number of inserted. + if (numInsertOplogEntryPrimary < numInserted && + numInsertOplogEntrySecondary < numInserted) + return true; + jsTestLog("Awaiting oplog truncation: number of oplog entries: " + + `(primary: ${tojson(numInsertOplogEntryPrimary)}, ` + + `secondary: ${tojson(numInsertOplogEntrySecondary)}) ` + + `number inserted: ${numInserted}`); + return false; + }, "Timeout waiting for oplog to roll over", ReplSetTest.kDefaultTimeoutMS, 1000); + } + + replSet.stopSet(); +} + +doTest("wiredTiger"); if (jsTest.options().storageEngine !== "inMemory") { jsTestLog( @@ -15,5 +194,5 @@ if (jsTest.options().storageEngine !== "inMemory") { return; } -oplogRolloverTest("inMemory"); +doTest("inMemory"); })(); diff --git a/jstests/replsets/oplog_sampling.js b/jstests/replsets/oplog_sampling.js index 1dddbfb2720..ff477521dbe 100644 --- a/jstests/replsets/oplog_sampling.js +++ b/jstests/replsets/oplog_sampling.js @@ -12,7 +12,7 @@ const replSet = new ReplSetTest({ nodeOptions: { setParameter: { "maxOplogTruncationPointsDuringStartup": 10, - logComponentVerbosity: tojson({storage: {verbosity: 3}}), + logComponentVerbosity: tojson({storage: {verbosity: 2}}), } } }); @@ -29,36 +29,12 @@ assert.gt(res.oplogTruncation.totalTimeProcessingMicros, 0); assert.eq(res.oplogTruncation.processingMethod, "scanning"); // Insert enough documents to force oplog sampling to occur on the following start up. -// Ensure that fast count of oplog collection increases while we insert the documents. -const oplogColl = replSet.getPrimary().getDB("local").getCollection("oplog.rs"); -let oplogFastCount = oplogColl.count(); const maxOplogDocsForScanning = 2000; -jsTestLog("Inserting " + maxOplogDocsForScanning + " documents to force oplog sampling on restart"); for (let i = 0; i < maxOplogDocsForScanning + 1; i++) { - let doc = {m: 1 + i}; - assert.commandWorked(coll.insert(doc), "failed to insert " + tojson(doc)); - - let newOplogFastCount = oplogColl.count(); - assert.gt( - newOplogFastCount, - oplogFastCount, - "fast count of oplog collection did not increase after successfully inserting " + - tojson(doc) + ". Previous fast count of oplog: " + oplogFastCount + - ". New fast count: " + newOplogFastCount + ". Last 5 oplog entries: " + - tojson(replSet.findOplog(replSet.getPrimary(), /*query=*/{}, /*limit=*/5).toArray())); - oplogFastCount = newOplogFastCount; + assert.commandWorked(coll.insert({m: 1 + i})); } -// Do not proceed with test if the oplog collection has a lower than expected fast count that -// will result in an oplog scan on restart. -assert.gt( - oplogFastCount, - maxOplogDocsForScanning, - "fast count of oplog collection is not large enough to trigger oplog sampling on restart"); - // Restart replica set to load entries from the oplog for sampling. -jsTestLog("Inserted " + maxOplogDocsForScanning + " documents. Oplog fast count: " + - oplogFastCount + ". Restarting server to force oplog sampling."); replSet.stopSet(null /* signal */, true /* forRestart */); replSet.startSet({restart: true}); diff --git a/jstests/replsets/optime.js b/jstests/replsets/optime.js index ff99784c1d7..69d5f875df4 100644 --- a/jstests/replsets/optime.js +++ b/jstests/replsets/optime.js @@ -62,11 +62,8 @@ var replTest = new ReplSetTest( const nodes = replTest.startSet(); -// Tests that serverStatus oplog returns null timestamps if the oplog collection doesn't exist. -const zeroTs = new Timestamp(0, 0); -const oplogStatus = nodes[0].getDB('admin').serverStatus({oplog: true}).oplog; -assert.eq(oplogStatus.earliestOptime, zeroTs); -assert.eq(oplogStatus.latestOptime, zeroTs); +// Tests that serverStatus oplog returns an error if the oplog collection doesn't exist. +assert.commandFailedWithCode(nodes[0].getDB('admin').serverStatus({oplog: true}), 17347); replTest.initiate(); var primary = replTest.getPrimary(); diff --git a/jstests/replsets/primary_casts_vote_on_stepdown.js b/jstests/replsets/primary_casts_vote_on_stepdown.js index 05a924fae1d..f07951a69c8 100644 --- a/jstests/replsets/primary_casts_vote_on_stepdown.js +++ b/jstests/replsets/primary_casts_vote_on_stepdown.js @@ -12,7 +12,7 @@ let name = "primary_casts_vote_on_stepdown"; let replTest = new ReplSetTest({name: name, nodes: 2}); let nodes = replTest.startSet(); -replTest.initiateWithHighElectionTimeout(); +replTest.initiate(); // Make sure node 0 is initially primary, and then step up node 1 and make sure it is able to // become primary in one election, gathering the vote of node 0, who will be forced to step diff --git a/jstests/replsets/profile.js b/jstests/replsets/profile.js index d7c6cfde9e3..caa5d7f8871 100644 --- a/jstests/replsets/profile.js +++ b/jstests/replsets/profile.js @@ -1,11 +1,7 @@ // Confirm that implicitly created profile collections are successful and do not trigger assertions. // In order to implicitly create a profile collection with a read, we must set up the server with // some data to read without the profiler being active. -// @tags: [ -// # 'assert' log component is not available in 5.0. -// requires_fcv_60, -// requires_persistence, -// ] +// @tags: [requires_persistence] (function() { "use strict"; let rst = new ReplSetTest({nodes: {n0: {profile: "0"}}}); @@ -24,22 +20,12 @@ primary = rst.getPrimary(); primaryDB = primary.getDB('test'); let oldAssertCounts = primaryDB.serverStatus().asserts; -jsTestLog('Before running aggregation: Assert counts reported by db.serverStatus(): ' + - tojson(oldAssertCounts)); -primaryDB.setLogLevel(1, 'assert'); -try { - assert.eq(0, primaryDB.system.profile.count()); - assert.eq([{_id: 1}], primaryDB.foo.aggregate([]).toArray()); - - let newAssertCounts = primaryDB.serverStatus().asserts; - jsTestLog('After running aggregation: Assert counts reported by db.serverStatus(): ' + - tojson(newAssertCounts)); - assert.eq(oldAssertCounts, newAssertCounts); - // Should have 2 entries, one for the count command and one for the aggregate command. - assert.eq(2, primaryDB.system.profile.count()); -} finally { - primaryDB.setLogLevel(0, 'assert'); -} +assert.eq(0, primaryDB.system.profile.count()); +assert.eq([{_id: 1}], primaryDB.foo.aggregate([]).toArray()); +let newAssertCounts = primaryDB.serverStatus().asserts; +assert.eq(oldAssertCounts, newAssertCounts); +// Should have 2 entries, one for the count command and one for the aggregate command. +assert.eq(2, primaryDB.system.profile.count()); rst.stopSet(); })(); diff --git a/jstests/replsets/quiesce_mode_fails_elections.js b/jstests/replsets/quiesce_mode_fails_elections.js deleted file mode 100644 index be80af8017c..00000000000 --- a/jstests/replsets/quiesce_mode_fails_elections.js +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Test that once a node enters quiesce mode, any concurrent or new elections cannot succeed. - */ -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallel_shell_helpers.js"); - -const rst = new ReplSetTest({ - name: jsTestName(), - nodes: 3, - // Override the quiesce period. - nodeOptions: {setParameter: "shutdownTimeoutMillisForSignaledShutdown=5000"} -}); - -rst.startSet(); -rst.initiateWithHighElectionTimeout(); - -const dbName = "test"; -const primary = rst.getPrimary(); -const secondary = rst.getSecondaries()[0]; -const primaryDB = primary.getDB(dbName); - -assert.commandWorked( - primaryDB.coll.insert([{_id: 0, data: "initial data"}], {writeConcern: {w: "majority"}})); -rst.awaitReplication(); - -jsTestLog("Make the secondary hang before processing real election vote result."); -let voteRequestCompleteFailPoint = - configureFailPoint(secondary, "hangBeforeOnVoteRequestCompleteCallback"); - -jsTestLog("Stepping up the secondary."); -const awaitStepUp = startParallelShell(() => { - assert.commandFailedWithCode(db.adminCommand({replSetStepUp: 1}), ErrorCodes.CommandFailed); -}, secondary.port); - -// Wait for secondary to hit the failpoint. Even though the election on secondary has not finished, -// the primary should step down due to seeing a higher term. -voteRequestCompleteFailPoint.wait(); -rst.waitForState(primary, ReplSetTest.State.SECONDARY); - -jsTestLog("Make the secondary hang after entering quiesce mode."); -let quiesceModeFailPoint = configureFailPoint(secondary, "hangDuringQuiesceMode"); -rst.stop(secondary, null /*signal*/, {skipValidation: true}, {forRestart: true, waitpid: false}); -quiesceModeFailPoint.wait(); - -jsTestLog("Unblock secondary election, the in-progress step up attempt should be cancelled"); -voteRequestCompleteFailPoint.off(); -awaitStepUp(); -// Check log line with id 214480: "Not becoming primary, election has been cancelled". -checkLog.checkContainsOnceJson(secondary, 214480); - -jsTestLog("Attempting another stepup should fail immediately due to being in quiesce mode"); -assert.commandFailedWithCode(secondary.adminCommand({replSetStepUp: 1}), ErrorCodes.CommandFailed); -// Check log line with id 4615654: "Not starting an election, since we are shutting down". -checkLog.checkContainsOnceJson(secondary, 4615654); - -jsTestLog("Unblock the secondary from quiesce mode"); -quiesceModeFailPoint.off(); - -rst.stopSet(); -})(); diff --git a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js index 56169ee402c..99018d864c5 100644 --- a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js +++ b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js @@ -59,10 +59,6 @@ assert.commandWorked(sessionColl1.insert({_id: 1})); assert.commandWorked(sessionColl2.insert({_id: 2})); assert.commandWorked(sessionColl3.insert({_id: 3})); assert.commandWorked(sessionColl3.insert({_id: 4})); -assert.commandWorked(sessionColl3.insert({_id: 5})); - -// Add a validator so we can make sure validation doesn't cause initial sync to fail. -assert.commandWorked(testDB.runCommand({collMod: collName, validator: {b: {$exists: false}}})); jsTestLog("Preparing three transactions"); @@ -115,7 +111,7 @@ jsTestLog("Running operations while collection cloning is paused"); // Perform writes while collection cloning is paused so that we know they must be applied during // the oplog application stage of initial sync. -assert.commandWorked(testColl.insert({_id: 999})); +assert.commandWorked(testColl.insert({_id: 5})); let session4 = primary.startSession(); let sessionDB4 = session4.getDatabase(dbName); @@ -129,22 +125,6 @@ session4.startTransaction(); assert.commandWorked(sessionColl4.update({_id: 4}, {_id: 4, a: 1})); const prepareTimestamp4 = PrepareHelpers.prepareTransaction(session4, {w: 1}); -let session5 = primary.startSession(); -let sessionDB5 = session5.getDatabase(dbName); -const sessionColl5 = sessionDB5.getCollection(collName); - -jsTestLog("Preparing the fifth transaction"); - -// Prepare a transaction that would fail validation while collection cloning is paused so that its -// oplog entry must be applied during the oplog application phase of initial sync. -session5.startTransaction(); -assert.commandWorked(sessionDB5.runCommand({ - update: collName, - updates: [{q: {_id: 5}, u: {_id: 5, a: 1, b: 2}}], - bypassDocumentValidation: true -})); -const prepareTimestamp5 = PrepareHelpers.prepareTransaction(session5, {w: 1}); - jsTestLog("Resuming initial sync"); // Resume initial sync. @@ -165,7 +145,7 @@ const secondaryColl = secondary.getDB(dbName).getCollection(collName); // changes to the documents from any of the prepared transactions after initial sync. Also, make // sure that the writes that happened when collection cloning was paused happened. const res = secondaryColl.find().sort({_id: 1}).toArray(); -assert.eq(res, [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 999}], res); +assert.eq(res, [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}], res); jsTestLog("Checking that the first transaction is properly prepared"); @@ -193,15 +173,9 @@ jsTestLog("Committing the fourth transaction"); assert.commandWorked(PrepareHelpers.commitTransaction(session4, prepareTimestamp4)); replTest.awaitReplication(); -jsTestLog("Committing the fifth transaction"); - -assert.commandWorked(PrepareHelpers.commitTransaction(session5, prepareTimestamp5)); -replTest.awaitReplication(); - // Make sure that we can see the data from a committed transaction on the secondary if it was // applied during secondary oplog application. -assert.docEq({_id: 4, a: 1}, secondaryColl.findOne({_id: 4})); -assert.docEq({_id: 5, a: 1, b: 2}, secondaryColl.findOne({_id: 5})); +assert.docEq(secondaryColl.findOne({_id: 4}), {_id: 4, a: 1}); jsTestLog("Stepping up the secondary"); @@ -297,4 +271,4 @@ assert.commandWorked(sessionDB3.adminCommand( assert.eq(testColl.find({_id: 3}).toArray(), [{_id: 3}]); replTest.stopSet(); -})(); +})();
\ No newline at end of file diff --git a/jstests/replsets/rollback_drop_database.js b/jstests/replsets/rollback_drop_database.js index 8efd9a9f65f..01d3e3c0659 100644 --- a/jstests/replsets/rollback_drop_database.js +++ b/jstests/replsets/rollback_drop_database.js @@ -3,20 +3,9 @@ * a collection, then executes a 'dropDatabase' command, partitioning the primary such that the * final 'dropDatabase' oplog entry is not replicated. The test then forces rollback of that entry. * - * The 'dropDatabase' command drops each collection, ensures that the last drop is majority - * committed, and only then logs a 'dropDatabase' oplog entry. This is therefore the only entry that - * could get rolled back. - * - * Additionally test handling of an incompletely dropped database across a replica set. If a primary - * writes a dropDatabase oplog entry and clears in-memory database state, but subsequently rolls - * back the dropDatabase oplog entry, then the replica set secondaries will still have the in-memory - * state. If the original primary is re-elected, it will allow a subsequent createCollection with a - * database name conflicting with the original database. The secondaries should close the original - * empty database and open the new database on receipt of the createCollection. - * - * @tags: [ - * multiversion_incompatible, - * ] + * The 'dropDatabase' command drops each collection, ensures that the last drop is committed, + * and only then logs a 'dropDatabase' oplog entry. This is therefore the only entry that could + * get rolled back. */ (function() { @@ -24,19 +13,15 @@ load("jstests/replsets/libs/rollback_test.js"); const testName = "rollback_drop_database"; - -// MongoDB does not allow multiple databases to exist that differ only in letter case. These -// database names will differ only in letter case, to test that secondaries will safely close -// conflicting empty databases. -const dbName = "olddatabase"; -const conflictingDbName = "OLDDATABASE"; +const oldDbName = "oldDatabase"; +const newDbName = "newDatabase"; let rollbackTest = new RollbackTest(testName); let rollbackNode = rollbackTest.getPrimary(); let syncSourceNode = rollbackTest.getSecondary(); // Perform initial insert (common operation). -assert.commandWorked(rollbackNode.getDB(dbName)["beforeRollback"].insert({"num": 1})); +assert.commandWorked(rollbackNode.getDB(oldDbName)["beforeRollback"].insert({"num": 1})); // Set a failpoint on the original primary, so that it blocks after it commits the last // 'dropCollection' entry but before the 'dropDatabase' entry is logged. @@ -45,7 +30,7 @@ assert.commandWorked(rollbackNode.adminCommand( // Issue a 'dropDatabase' command. let dropDatabaseFn = function() { - const rollbackDb = "olddatabase"; + const rollbackDb = "oldDatabase"; var primary = db.getMongo(); jsTestLog("Dropping database " + rollbackDb + " on primary node " + primary.host); var dbToDrop = db.getSiblingDB(rollbackDb); @@ -60,7 +45,7 @@ checkLog.contains(rollbackNode, // Wait for the secondary to finish dropping the collection (the last replicated entry). // We use the default 10-minute timeout for this. assert.soon(function() { - let res = syncSourceNode.getDB(dbName).getCollectionNames().includes("beforeRollback"); + let res = syncSourceNode.getDB(oldDbName).getCollectionNames().includes("beforeRollback"); return !res; }, "Sync source did not finish dropping collection beforeRollback", 10 * 60 * 1000); @@ -71,31 +56,17 @@ rollbackTest.transitionToRollbackOperations(); assert.commandWorked(rollbackNode.adminCommand( {configureFailPoint: "dropDatabaseHangBeforeInMemoryDrop", mode: "off"})); waitForDropDatabaseToFinish(); - -assert.eq(false, rollbackNode.getDB(dbName).getCollectionNames().includes("beforeRollback")); -jsTestLog("Database " + dbName + " successfully dropped on primary node " + rollbackNode.host); +assert.eq(false, rollbackNode.getDB(oldDbName).getCollectionNames().includes("beforeRollback")); +jsTestLog("Database " + oldDbName + " successfully dropped on primary node " + rollbackNode.host); rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); // Perform an insert on another database while interfacing with the new primary. // This is the sync source's divergent oplog entry. -assert.commandWorked(syncSourceNode.getDB("someDB")["afterRollback"].insert({"num": 2})); +assert.commandWorked(syncSourceNode.getDB(newDbName)["afterRollback"].insert({"num": 2})); rollbackTest.transitionToSyncSourceOperationsDuringRollback(); rollbackTest.transitionToSteadyStateOperations(); -jsTestLog("Transitioned to steady state, going to run test operations"); - -// Check that replication rollback occurred on the old primary. -assert(checkLog.checkContainsOnceJson(rollbackNode, 21612)); - -// The syncSourceNode never received the dropDatabase oplog entry from the rollbackNode. Therefore, -// syncSourceNode never cleared the in-memory database state for that database. Check that -// syncSourceNode will safely clear the original empty database when applying a createCollection -// with a new database name that conflicts with the original. -rollbackTest.stepUpNode(rollbackNode); -// Using only w:2 because the third node is frozen / not replicating. -assert.commandWorked(rollbackNode.getDB(conflictingDbName)["afterRollback"].insert( - {"num": 2}, {writeConcern: {w: 2}})); rollbackTest.stop(); })(); diff --git a/jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js b/jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js index e1bdc8ce00c..22adec4a939 100644 --- a/jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js +++ b/jstests/replsets/rollback_with_coalesced_txn_table_updates_during_oplog_application.js @@ -39,20 +39,10 @@ function runTest(crashAfterRollbackTruncation) { rst.startSet(); rst.initiate(); - const lsid = ({id: UUID()}); const primary = rst.getPrimary(); const ns = "test.retryable_write_partial_rollback"; assert.commandWorked( primary.getCollection(ns).insert({_id: 0, counter: 0}, {writeConcern: {w: 5}})); - // SERVER-65971: Do a write with `lsid` to add an entry to config.transactions. This write will - // persist after rollback and be updated when the rollback code corrects for omitted writes to - // the document. - assert.commandWorked(primary.getCollection(ns).runCommand("insert", { - documents: [{_id: ObjectId()}], - lsid, - txnNumber: NumberLong(1), - writeConcern: {w: 5}, - })); // The default WC is majority and this test can't satisfy majority writes. assert.commandWorked(primary.adminCommand( {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}})); @@ -75,10 +65,12 @@ function runTest(crashAfterRollbackTruncation) { 'stopReplProducerOnDocument', {document: {"diff.u.counter": counterMajorityCommitted + 1}})); + const lsid = ({id: UUID()}); + assert.commandWorked(primary.getCollection(ns).runCommand("update", { updates: Array.from({length: counterTotal}, () => ({q: {_id: 0}, u: {$inc: {counter: 1}}})), lsid, - txnNumber: NumberLong(2), + txnNumber: NumberLong(1), })); const stmtMajorityCommitted = primary.getCollection("local.oplog.rs") @@ -177,7 +169,7 @@ function runTest(crashAfterRollbackTruncation) { assert.commandWorked(secondary1.getCollection(ns).runCommand("update", { updates: Array.from({length: counterTotal}, () => ({q: {_id: 0}, u: {$inc: {counter: 1}}})), lsid, - txnNumber: NumberLong(2), + txnNumber: NumberLong(1), writeConcern: {w: 5}, })); diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js index 75a553091af..40f544e2287 100644 --- a/jstests/replsets/rslib.js +++ b/jstests/replsets/rslib.js @@ -531,8 +531,8 @@ reInitiateWithoutThrowingOnAbortedMember = function(replSetTest) { try { replSetTest.reInitiate(); } catch (e) { - // reInitiate can throw because it tries to run a "hello" command on all secondaries, - // including the new one that may have already aborted + // reInitiate can throw because it tries to run an ismaster command on + // all secondaries, including the new one that may have already aborted const errMsg = tojson(e); if (isNetworkError(e)) { // Ignore these exceptions, which are indicative of an aborted node @@ -586,7 +586,7 @@ awaitRSClientHosts = function(conn, host, hostOk, rs, timeout) { // Check that *all* host properties are set correctly var propOk = true; for (var prop in hostOk) { - // Use special comparator for tags because hello can return the fields in + // Use special comparator for tags because isMaster can return the fields in // different order. The fields of the tags should be treated like a set of // strings and 2 tags should be considered the same if the set is equal. if (prop == 'tags') { diff --git a/jstests/replsets/secondary_as_sync_source.js b/jstests/replsets/secondary_as_sync_source.js index 54c0d90fec3..60e15e55a71 100644 --- a/jstests/replsets/secondary_as_sync_source.js +++ b/jstests/replsets/secondary_as_sync_source.js @@ -4,25 +4,22 @@ * sync operation. * * @tags: [ - * requires_fcv_60, * requires_replication, * ] */ (function() { 'use strict'; -load("jstests/core/timeseries/libs/timeseries.js"); load('jstests/noPassthrough/libs/index_build.js'); load("jstests/replsets/rslib.js"); const dbName = "test"; const collName = "coll"; -const timeseriesCollName = "tscoll"; -function addTestDocuments(coll) { +function addTestDocuments(db) { let size = 100; jsTest.log("Creating " + size + " test documents."); - var bulk = coll.initializeUnorderedBulkOp(); + var bulk = db.getCollection(collName).initializeUnorderedBulkOp(); for (var i = 0; i < size; ++i) { bulk.insert({i: i}); } @@ -52,32 +49,20 @@ let primaryDB = primary.getDB(dbName); let secondary = replSet.getSecondary(); let secondaryDB = secondary.getDB(dbName); -const coll = primaryDB.getCollection(collName); -addTestDocuments(coll); - -// Create time-series collection with a single measurement. -// We need a non-empty collection to use two-phase index builds. -assert.commandWorked( - primaryDB.createCollection(timeseriesCollName, {timeseries: {timeField: 'time'}})); -const timeseriesColl = primaryDB.getCollection(timeseriesCollName); -assert.commandWorked(timeseriesColl.insert({time: ISODate(), x: 1})); +addTestDocuments(primaryDB); // Used to wait for two-phase builds to complete. let awaitIndex; -let awaitIndexTimeseries; jsTest.log("Hanging index build on the primary node"); IndexBuildTest.pauseIndexBuilds(primary); jsTest.log("Beginning index build"); +const coll = primaryDB.getCollection(collName); awaitIndex = IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {i: 1}); -awaitIndexTimeseries = - IndexBuildTest.startIndexBuild(primary, timeseriesColl.getFullName(), {x: 1}); jsTest.log("Waiting for index build to start on secondary"); -IndexBuildTest.waitForIndexBuildToStart(secondaryDB, collName, 'i_1'); -IndexBuildTest.waitForIndexBuildToStart( - secondaryDB, TimeseriesTest.getBucketsCollName(timeseriesCollName), 'x_1'); +IndexBuildTest.waitForIndexBuildToStart(secondaryDB); jsTest.log("Adding a new node to the replica set"); let newNode = replSet.add({ @@ -100,7 +85,6 @@ waitForState(newNode, ReplSetTest.State.SECONDARY); jsTest.log("Removing index build hang to allow it to finish"); IndexBuildTest.resumeIndexBuilds(primary); awaitIndex(); -awaitIndexTimeseries(); // Wait for the index builds to finish. replSet.awaitReplication(); @@ -115,13 +99,5 @@ printjson(secondaryDB.getCollection(collName).getIndexes()); assert.eq(newNodeDB.getCollection(collName).getIndexes().length, secondaryDB.getCollection(collName).getIndexes().length); -jsTest.log("New nodes indexes for time-series collection:"); -printjson(newNodeDB.getCollection(timeseriesCollName).getIndexes()); -jsTest.log("Secondary nodes indexes for time-series collection:"); -printjson(secondaryDB.getCollection(timeseriesCollName).getIndexes()); - -assert.eq(newNodeDB.getCollection(timeseriesCollName).getIndexes().length, - secondaryDB.getCollection(timeseriesCollName).getIndexes().length); - replSet.stopSet(); })(); diff --git a/jstests/replsets/session_cache_refresh_write_error_fail.js b/jstests/replsets/session_cache_refresh_write_error_fail.js deleted file mode 100644 index 8a2749b0fac..00000000000 --- a/jstests/replsets/session_cache_refresh_write_error_fail.js +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Test that write errors resulting as part of refreshing logical session do not kill open cursors. - */ -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); - -const rst = new ReplSetTest({nodes: 1}); - -rst.startSet(); -rst.initiate(); - -const db = rst.getPrimary().getDB("test"); -const fp = configureFailPoint(db, "failAllUpdates"); -const collection = db.getCollection("mycoll"); -const sessionDb = db.getMongo().startSession().getDatabase(db.getName()); -const sessionCollection = sessionDb.getCollection(collection.getName()); - -assert.commandWorked(sessionCollection.insert(Array.from({length: 5}, (_, i) => ({_id: i})))); - -const res = assert.commandWorked(sessionCollection.runCommand("find", {batchSize: 2})); - -assert.commandFailedWithCode(db.adminCommand({refreshLogicalSessionCacheNow: 1}), - ErrorCodes.InternalError); - -assert.commandWorked( - sessionDb.runCommand({getMore: res.cursor.id, collection: sessionCollection.getName()})); - -fp.off(); - -rst.stopSet(); -})(); diff --git a/jstests/replsets/standalone_replication_recovery_relaxes_index_constraints.js b/jstests/replsets/standalone_replication_recovery_relaxes_index_constraints.js index 83934a6e138..530140a3dad 100644 --- a/jstests/replsets/standalone_replication_recovery_relaxes_index_constraints.js +++ b/jstests/replsets/standalone_replication_recovery_relaxes_index_constraints.js @@ -57,11 +57,6 @@ node = rst.restart(node, { noReplSet: true, setParameter: {recoverFromOplogAsStandalone: true, logComponentVerbosity: logLevel} }); - -// Verify that the 'config.system.indexBuilds' collection is empty after recovering from the oplog -// in standalone mode. -assert.eq(0, node.getCollection("config.system.indexBuilds").count()); - reconnect(node); rst.stopSet(); diff --git a/jstests/replsets/stepup_with_linearizable_read.js b/jstests/replsets/stepup_with_linearizable_read.js index 4d3942c8fcf..abbc2753252 100644 --- a/jstests/replsets/stepup_with_linearizable_read.js +++ b/jstests/replsets/stepup_with_linearizable_read.js @@ -31,12 +31,12 @@ var sendLinearizableReadOnFailpoint = function() { jsTestLog('Sending in linearizable read in secondary thread'); - // In lock free reads this will error with NotWritablePrimary. Without lock free reads we - // timeout because we can't acquire the RSTL. + // In lock free reads this will timeout as we cannot perform the necessary write after the + // read. Without lock free reads we timeout because we can't acquire the RSTL. assert.commandFailedWithCode( coll.runCommand( {'find': 'foo', readConcern: {level: "linearizable"}, maxTimeMS: 10000}), - [ErrorCodes.NotWritablePrimary, ErrorCodes.MaxTimeMSExpired]); + ErrorCodes.MaxTimeMSExpired); } finally { // Turn off fail point so we can cleanup. assert.commandWorked(db.getMongo().adminCommand( diff --git a/jstests/replsets/sync_source_changes.js b/jstests/replsets/sync_source_changes.js index 02bce0ba3b9..80655c06667 100644 --- a/jstests/replsets/sync_source_changes.js +++ b/jstests/replsets/sync_source_changes.js @@ -10,34 +10,6 @@ load("jstests/replsets/rslib.js"); // reconfig load("jstests/replsets/libs/sync_source.js"); // assertSyncSourceMatchesSoon -// We need to wait for a heartbeat from the secondary to the sync source, then run sync -// source selection, because: -// 1) The sync source changes only after retrieving a batch and -// 2) The sync source won't change if the secondary isn't behind the expected sync source, as -// determined by heartbeats. -function assertSyncSourceChangesTo(rst, secondary, expectedSyncSource) { - // Insert a document while 'secondary' is not replicating to force it to run - // shouldChangeSyncSource. - stopServerReplication(secondary); - assert.commandWorked( - rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, { - writeConcern: {w: 1} - })); - const sourceId = rst.getNodeId(expectedSyncSource); - // Waits for the secondary to see the expected sync source advance beyond it. - assert.soon(function() { - const status = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})); - const appliedTimestamp = status.optimes.appliedOpTime.ts; - const sourceMember = status.members.find((x) => x._id == sourceId); - return timestampCmp(sourceMember.optime.ts, appliedTimestamp) > 0; - }); - restartServerReplication(secondary); - assertSyncSourceMatchesSoon(secondary, expectedSyncSource.host); -} - -// Replication verbosity 2 includes the sync source change debug logs. -TestData["setParameters"]["logComponentVerbosity"]["replication"]["verbosity"] = 2; - // Start RST with only one voting node, node 0 -- this will be the only valid voting node and sync // source const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]}); @@ -49,18 +21,14 @@ const primary = rst.getPrimary(); assert.eq(primary, rst.nodes[0]); // Add a new voting node, node 2 -- voting nodes will choose voting nodes as sync sources. -jsTestLog("Adding node 2"); const newNode = rst.add({}); rst.reInitiate(); rst.waitForState(newNode, ReplSetTest.State.SECONDARY); rst.awaitReplication(); rst.awaitSecondaryNodes(); -// Wait for the new node to no longer be newlyAdded, so that it becomes a voting node. -rst.waitForAllNewlyAddedRemovals(); - // Assure that node 2 will set node 0 as its sync source, since it is the best option. -assertSyncSourceChangesTo(rst, newNode, rst.nodes[0]); +assertSyncSourceMatchesSoon(newNode, rst.nodes[0].host); // Make node 1 a voter so that it will be a valid option for sync source let cfg = rst.getReplSetConfigFromNode(); @@ -70,18 +38,36 @@ reconfig(rst, cfg); // Force a stepup of node 1 -- we need to step node 0 down so that we can set it as a non-voter // without causing errors. -jsTestLog("Stepping up node 1"); rst.stepUp(rst.nodes[1]); -jsTestLog("Reconfiguring node 0 as nonvoter"); // Make node 0 a nonvoter so that it will be an invalid option for sync source cfg = rst.getReplSetConfigFromNode(); cfg.members[0].priority = 0; cfg.members[0].votes = 0; reconfig(rst, cfg); -jsTestLog("Reconfig complete"); -assertSyncSourceChangesTo(rst, newNode, rst.nodes[1]); +// Run this repeatedly, as sometimes the stop, insert, restart won't cause the sync source to be +// switched correctly due to transient issues with the sync source we want to switch to. +assert.soon(() => { + // Insert a document while newNode is not replicating to force it to run shouldChangeSyncSource + stopServerReplication(newNode); + assert.commandWorked( + rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, { + writeConcern: {w: 1} + })); + restartServerReplication(newNode); + try { + assertSyncSourceMatchesSoon(newNode, + cfg.members[1].host, + undefined /* msg */, + 5 * 1000 /* timeout */, + undefined /* interval */, + {runHangAnalyzer: false}); + return true; + } catch (e) { + return false; + } +}); rst.stopSet(); })(); diff --git a/jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js b/jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js index e4ab88ed90d..ee7c23b0db0 100644 --- a/jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js +++ b/jstests/replsets/sync_source_selection_ignores_minvalid_after_rollback.js @@ -64,9 +64,6 @@ assert.commandWorked(node1.adminCommand({clearLog: 'global'})); jsTestLog("Stepping up node 2"); -// Make sure id:5972100 debug log is enabled. -setLogVerbosity([node1], {"replication": {"verbosity": 1}}); - // Node 2 runs for election. This is needed before node 1 steps up because otherwise it will always // lose future elections and will not be considered the proper branch of history. const electionShell = startParallelShell(() => { @@ -129,4 +126,4 @@ assert.eq(node2Coll.find({_id: "proper branch of history"}).itcount(), 1); assert.eq(node2Coll.find({_id: "diverging point"}).itcount(), 0); rst.stopSet(); -})(); +})();
\ No newline at end of file diff --git a/jstests/replsets/tenant_migration_abort_forget_retry.js b/jstests/replsets/tenant_migration_abort_forget_retry.js new file mode 100644 index 00000000000..21913140ba6 --- /dev/null +++ b/jstests/replsets/tenant_migration_abort_forget_retry.js @@ -0,0 +1,119 @@ +/** + * Starts a tenant migration that aborts, either due to the + * abortTenantMigrationBeforeLeavingBlockingState failpoint or due to receiving donorAbortMigration, + * and then issues a donorForgetMigration command. Finally, starts a second tenant migration with + * the same tenantId as the aborted migration, and expects this second migration to go through. + * + * @tags: [ + * incompatible_with_eft, + * incompatible_with_macos, + * incompatible_with_windows_tls, + * requires_majority_read_concern, + * requires_persistence, + * serverless, + * ] + */ + +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); +load("jstests/libs/parallelTester.js"); +load("jstests/libs/uuid_util.js"); +load("jstests/replsets/libs/tenant_migration_test.js"); +load("jstests/replsets/libs/tenant_migration_util.js"); + +const kTenantIdPrefix = "testTenantId"; +let testNum = 0; + +function makeTenantId() { + return kTenantIdPrefix + testNum++; +} + +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); + +(() => { + const migrationId1 = extractUUIDFromObject(UUID()); + const migrationId2 = extractUUIDFromObject(UUID()); + const tenantId = makeTenantId(); + + // Start a migration with the "abortTenantMigrationBeforeLeavingBlockingState" failPoint + // enabled. The migration will abort as a result, and a status of "kAborted" should be returned. + jsTestLog( + "Starting a migration that is expected to abort due to setting abortTenantMigrationBeforeLeavingBlockingState failpoint. migrationId: " + + migrationId1 + ", tenantId: " + tenantId); + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + const abortFp = + configureFailPoint(donorPrimary, "abortTenantMigrationBeforeLeavingBlockingState"); + TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration( + {migrationIdString: migrationId1, tenantId: tenantId}, {automaticForgetMigration: false})); + abortFp.off(); + + // Forget the aborted migration. + jsTestLog("Forgetting aborted migration with migrationId: " + migrationId1); + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationId1)); + + // Try running a new migration with the same tenantId. It should succeed, since the previous + // migration with the same tenantId was aborted. + jsTestLog("Attempting to run a new migration with the same tenantId. New migrationId: " + + migrationId2 + ", tenantId: " + tenantId); + TenantMigrationTest.assertCommitted( + tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); +})(); + +(() => { + const migrationId1 = extractUUIDFromObject(UUID()); + const migrationId2 = extractUUIDFromObject(UUID()); + const tenantId = makeTenantId(); + + jsTestLog( + "Starting a migration that is expected to abort in blocking state due to receiving donorAbortMigration. migrationId: " + + migrationId1 + ", tenantId: " + tenantId); + + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + let fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState"); + assert.commandWorked( + tenantMigrationTest.startMigration({migrationIdString: migrationId1, tenantId: tenantId})); + + fp.wait(); + + const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()); + const tryAbortThread = new Thread(TenantMigrationUtil.tryAbortMigrationAsync, + {migrationIdString: migrationId1, tenantId: tenantId}, + donorRstArgs, + TenantMigrationUtil.runTenantMigrationCommand); + tryAbortThread.start(); + + // Wait for donorAbortMigration command to start. + assert.soon(() => { + const res = assert.commandWorked( + donorPrimary.adminCommand({currentOp: true, desc: "tenant donor migration"})); + const op = res.inprog.find(op => extractUUIDFromObject(op.instanceID) === migrationId1); + return op.receivedCancellation; + }); + + fp.off(); + + tryAbortThread.join(); + assert.commandWorked(tryAbortThread.returnData()); + + TenantMigrationTest.assertAborted(tenantMigrationTest.waitForMigrationToComplete( + {migrationIdString: migrationId1, tenantId: tenantId})); + + // Forget the aborted migration. + jsTestLog("Forgetting aborted migration with migrationId: " + migrationId1); + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationId1)); + + // Try running a new migration with the same tenantId. It should succeed, since the previous + // migration with the same tenantId was aborted. + jsTestLog("Attempting to run a new migration with the same tenantId. New migrationId: " + + migrationId2 + ", tenantId: " + tenantId); + TenantMigrationTest.assertCommitted( + tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); +})(); + +tenantMigrationTest.stop(); +})(); diff --git a/jstests/replsets/tenant_migration_cloner_stats_with_failover.js b/jstests/replsets/tenant_migration_cloner_stats_with_failover.js index e84f77b5d20..43c4a63826e 100644 --- a/jstests/replsets/tenant_migration_cloner_stats_with_failover.js +++ b/jstests/replsets/tenant_migration_cloner_stats_with_failover.js @@ -124,7 +124,8 @@ jsTestLog("Bytes copied after first batch of second database: " + bytesCopiedInc // original primary to the new primary. Then, step up the new primary. const fpAfterCreatingCollectionOfSecondDB = configureFailPoint(newRecipientPrimary, "tenantCollectionClonerHangAfterCreateCollection"); -tenantMigrationTest.getRecipientRst().stepUp(newRecipientPrimary); +tenantMigrationTest.getRecipientRst().awaitReplication(); +newRecipientPrimary.adminCommand({replSetStepUp: 1}); fpAfterBatchOfSecondDB.off(); jsTestLog("Wait until the new primary creates collection of second database."); diff --git a/jstests/replsets/tenant_migration_collection_ttl.js b/jstests/replsets/tenant_migration_collection_ttl.js index def0c2da923..1f4515f568e 100644 --- a/jstests/replsets/tenant_migration_collection_ttl.js +++ b/jstests/replsets/tenant_migration_collection_ttl.js @@ -33,14 +33,8 @@ const garbageCollectionOpts = { 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}), }; -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: {setParameter: garbageCollectionOpts}, - // This test relies on ttl monitor deletion to be delayed long enough to observe documents prior - // to being deleted. That result is unintuitively achieved better with a large awaitData timeout - // than a slow ttl monitor. - optimizeMigrations: false -}); +const tenantMigrationTest = new TenantMigrationTest( + {name: jsTestName(), sharedOptions: {setParameter: garbageCollectionOpts}}); const collName = "testColl"; diff --git a/jstests/replsets/tenant_migration_commit_transaction_retry.js b/jstests/replsets/tenant_migration_commit_transaction_retry.js index 89f0bb1a04c..2d2eedb4bfe 100644 --- a/jstests/replsets/tenant_migration_commit_transaction_retry.js +++ b/jstests/replsets/tenant_migration_commit_transaction_retry.js @@ -19,6 +19,15 @@ load("jstests/replsets/libs/tenant_migration_util.js"); load("jstests/replsets/rslib.js"); load("jstests/libs/uuid_util.js"); +const kGarbageCollectionParams = { + // Set the delay before a donor state doc is garbage collected to be short to speed up + // the test. + tenantMigrationGarbageCollectionDelayMS: 3 * 1000, + + // Set the TTL monitor to run at a smaller interval to speed up the test. + ttlMonitorSleepSecs: 1, +}; + const tenantMigrationTest = new TenantMigrationTest( {name: jsTestName(), sharedOptions: {nodes: 1}, quickGarbageCollection: true}); @@ -87,11 +96,7 @@ waitAfterStartingOplogApplier.off(); waitInOplogApplier.off(); TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); -// With `quickGarbageCollection` it's likely that forgetting the migration will race with its -// natural destruction. -assert.commandWorkedOrFailedWithCode( - tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString), - [ErrorCodes.NoSuchTenantMigration]); +assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, kTenantId); // Test the client can retry commitTransaction against the recipient for transactions that committed @@ -110,8 +115,7 @@ jsTestLog("Running a back-to-back migration"); const tenantMigrationTest2 = new TenantMigrationTest({ name: jsTestName() + "2", donorRst: tenantMigrationTest.getRecipientRst(), - sharedOptions: {nodes: 1}, - quickGarbageCollection: true, + sharedOptions: {nodes: 1, setParameter: kGarbageCollectionParams} }); const migrationId2 = UUID(); const migrationOpts2 = { @@ -128,11 +132,7 @@ donorTxnEntries.forEach((txnEntry) => { assert.commandWorked(recipientPrimary2.adminCommand( {commitTransaction: 1, lsid: txnEntry._id, txnNumber: txnEntry.txnNum, autocommit: false})); }); -// With `quickGarbageCollection` it's likely that forgetting the migration will race with its -// natural destruction. -assert.commandWorkedOrFailedWithCode( - tenantMigrationTest2.forgetMigration(migrationOpts2.migrationIdString), - [ErrorCodes.NoSuchTenantMigration]); +assert.commandWorked(tenantMigrationTest2.forgetMigration(migrationOpts2.migrationIdString)); tenantMigrationTest2.waitForMigrationGarbageCollection(migrationId2, kTenantId); tenantMigrationTest2.stop(); diff --git a/jstests/replsets/tenant_migration_concurrent_migrations_stress_test.js b/jstests/replsets/tenant_migration_concurrent_migrations_stress_test.js index 187aef9af39..57c3027ecab 100644 --- a/jstests/replsets/tenant_migration_concurrent_migrations_stress_test.js +++ b/jstests/replsets/tenant_migration_concurrent_migrations_stress_test.js @@ -37,11 +37,8 @@ const setParameterOpts = { maxTenantMigrationRecipientThreadPoolSize: 1000, maxTenantMigrationDonorServiceThreadPoolSize: 1000 }; -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: {setParameter: setParameterOpts}, - optimizeMigrations: false -}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), sharedOptions: {setParameter: setParameterOpts}}); const donorPrimary = tenantMigrationTest.getDonorPrimary(); const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js index 70f82a3a2db..761e5967446 100644 --- a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js +++ b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js @@ -91,7 +91,6 @@ function testRejectAllReadsAfterCloningDone({testCase, dbName, collName, tenantM beforeFetchingTransactionsFp.off(); TenantMigrationTest.assertCommitted(runMigrationThread.returnData()); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); } /** @@ -171,7 +170,6 @@ function testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp( }); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); } /** @@ -224,7 +222,6 @@ function testDoNotRejectReadsAfterMigrationAbortedBeforeReachingRejectReadsBefor runCommand(db, testCase.command(collName), null); } }); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); } /** @@ -416,34 +413,25 @@ const testFuncs = { testDoNotRejectReadsAfterMigrationAbortedAfterReachingRejectReadsBeforeTimestamp }; -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); for (const [testName, testFunc] of Object.entries(testFuncs)) { for (const [testCaseName, testCase] of Object.entries(testCases)) { jsTest.log("Testing " + testName + " with testCase " + testCaseName); let tenantId = `${testCaseName}-${testName}`; - let migrationDb = `${tenantId}_test`; - tenantMigrationTest.insertDonorDB(migrationDb, "test"); let dbName = `${tenantId}_${kTenantDefinedDbName}`; + const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + quickGarbageCollection: true, + insertDataForTenant: tenantId, + }); - // Force the recipient to preserve all snapshot history to ensure that snapshot reads do - // not fail with SnapshotTooOld due to snapshot being unavailable. + // Force the recipient to preserve all snapshot history to ensure that snapshot reads do not + // fail with SnapshotTooOld due to snapshot being unavailable. tenantMigrationTest.getRecipientRst().nodes.forEach(node => { configureFailPoint(node, "WTPreserveSnapshotHistoryIndefinitely"); }); testFunc({testCase, dbName, collName: kCollName, tenantMigrationTest}); - - // ShardMerge is not robust to migrating the twice in quick succession. We drop the data - // files to ensure a subsequent tenant migration will avoid trying to merge files from the - // previous migration. - assert.commandWorked( - tenantMigrationTest.getDonorRst().getPrimary().getDB(migrationDb).dropDatabase()); - assert.commandWorked( - tenantMigrationTest.getRecipientRst().getPrimary().getDB(migrationDb).dropDatabase()); + tenantMigrationTest.stop(); } } -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js index ae148cffbee..49eb4baeb71 100644 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js +++ b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js @@ -1,5 +1,6 @@ /** - * Tests that writes on the donor set succeeds when there is no migration. + * Tests that the donor blocks writes that are executed while the migration in the blocking state, + * then rejects the writes when the migration completes. * * Tenant migrations are not expected to be run on servers with ephemeralForTest, and in particular * this test fails on ephemeralForTest because the donor has to wait for the write to set the @@ -23,7 +24,6 @@ load("jstests/libs/parallelTester.js"); load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); const tenantMigrationTest = new TenantMigrationTest({ name: jsTestName(), @@ -36,48 +36,1031 @@ const donorPrimary = donorRst.getPrimary(); const kCollName = "testColl"; const kTenantDefinedDbName = "0"; +const kTestDoc = { + x: -1 +}; +const kTestDoc2 = { + x: -2 +}; + +const kTestIndexKey = { + x: 1 +}; +const kExpireAfterSeconds = 1000000; +const kTestIndex = { + key: kTestIndexKey, + name: "testIndex", + expireAfterSeconds: kExpireAfterSeconds +}; + +const kNumInitialDocs = 2; // num initial docs to insert into test collections. +const kMaxSize = 1024; // max size of capped collections. +const kTxnNumber = NumberLong(0); +const kMaxTimeMS = 1 * 1000; /** - * Tests that the write succeeds when there is no migration. + * Asserts that the TenantMigrationAccessBlocker for the given tenant on the given node has the + * expected statistics. */ -function testWritesNoMigration(testCase, testOpts) { - runCommandForConcurrentWritesTest(testOpts); - testCase.assertCommandSucceeded(testOpts.primaryDB, testOpts.dbName, testOpts.collName); +function checkTenantMigrationAccessBlocker(node, tenantId, { + numBlockedWrites = 0, + numTenantMigrationCommittedErrors = 0, + numTenantMigrationAbortedErrors = 0 +}) { + const mtab = + TenantMigrationUtil.getTenantMigrationAccessBlocker({donorNode: node, tenantId}).donor; + if (!mtab) { + assert.eq(0, numBlockedWrites); + assert.eq(0, numTenantMigrationCommittedErrors); + assert.eq(0, numTenantMigrationAbortedErrors); + return; + } + + assert.eq(mtab.numBlockedReads, 0, tojson(mtab)); + assert.eq(mtab.numBlockedWrites, numBlockedWrites, tojson(mtab)); + assert.eq( + mtab.numTenantMigrationCommittedErrors, numTenantMigrationCommittedErrors, tojson(mtab)); + assert.eq(mtab.numTenantMigrationAbortedErrors, numTenantMigrationAbortedErrors, tojson(mtab)); +} + +/** + * To be used to resume a migration that is paused after entering the blocking state. Waits for the + * number of blocked reads to reach 'targetNumBlockedWrites' and unpauses the migration. + */ +function resumeMigrationAfterBlockingWrite(host, tenantId, targetNumBlockedWrites) { + load("jstests/libs/fail_point_util.js"); + load("jstests/replsets/libs/tenant_migration_util.js"); + const primary = new Mongo(host); + + assert.soon(() => TenantMigrationUtil.getNumBlockedWrites(primary, tenantId) == + targetNumBlockedWrites); + + assert.commandWorked(primary.adminCommand( + {configureFailPoint: "pauseTenantMigrationBeforeLeavingBlockingState", mode: "off"})); +} + +function createCollectionAndInsertDocs(primaryDB, collName, isCapped, numDocs = kNumInitialDocs) { + const createCollCommand = {create: collName}; + if (isCapped) { + createCollCommand.capped = true; + createCollCommand.size = kMaxSize; + } + assert.commandWorked(primaryDB.runCommand(createCollCommand)); + + let bulk = primaryDB[collName].initializeUnorderedBulkOp(); + for (let i = 0; i < numDocs; ++i) { + bulk.insert({x: i}); + } + assert.commandWorked(bulk.execute()); +} + +function insertTestDoc(primaryDB, collName) { + assert.commandWorked(primaryDB.runCommand({insert: collName, documents: [kTestDoc]})); +} + +function insertTwoTestDocs(primaryDB, collName) { + assert.commandWorked( + primaryDB.runCommand({insert: collName, documents: [kTestDoc, kTestDoc2]})); +} + +function createTestIndex(primaryDB, collName) { + assert.commandWorked(primaryDB.runCommand({createIndexes: collName, indexes: [kTestIndex]})); } -const testCases = TenantMigrationConcurrentWriteUtil.testCases; +function countDocs(db, collName, query) { + const res = assert.commandWorked(db.runCommand({count: collName, query: query})); + return res.n; +} + +function databaseExists(db, dbName) { + const res = assert.commandWorked(db.adminCommand({listDatabases: 1})); + return res.databases.some((dbDoc => dbDoc.name === dbName)); +} + +function collectionExists(db, collName) { + const res = assert.commandWorked(db.runCommand({listCollections: 1, filter: {name: collName}})); + return res.cursor.firstBatch.length == 1; +} + +function indexExists(db, collName, targetIndex) { + const res = assert.commandWorked(db.runCommand({listIndexes: collName})); + return res.cursor.firstBatch.some( + (index) => bsonWoCompare(index.key, targetIndex.key) === 0 && + bsonWoCompare(index.expireAfterSeconds, targetIndex.expireAfterSeconds) === 0); +} -// Run test cases with no migration. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-noMigration0"; +function validateTestCase(testCase) { + assert(testCase.skip || testCase.command, + "must specify exactly one of 'skip' or 'command' for test case " + tojson(testCase)); if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; + return; + } + + assert(testCase.command, "must specify 'command' for test case " + tojson(testCase)); + + // Check that all present fields are of the correct type. + assert(typeof (testCase.command) === "function"); + assert(typeof (testCase.assertCommandFailed) === "function"); + assert(testCase.setUp ? typeof (testCase.setUp) === "function" : true); + assert(testCase.runAgainstAdminDb ? typeof (testCase.runAgainstAdminDb) === "boolean" : true); + assert(testCase.explicitlyCreateCollection + ? typeof (testCase.explicitlyCreateCollection) === "boolean" + : true); + assert(testCase.testInTransaction ? typeof (testCase.testInTransaction) === "boolean" : true); + assert(testCase.testAsRetryableWrite ? typeof (testCase.testAsRetryableWrite) === "boolean" + : true); +} + +function makeTestOptions( + primary, testCase, dbName, collName, testInTransaction, testAsRetryableWrite) { + assert(!testInTransaction || !testAsRetryableWrite); + + const useSession = testInTransaction || testAsRetryableWrite || testCase.isTransactionCommand; + const primaryConn = useSession ? primary.startSession({causalConsistency: false}) : primary; + const primaryDB = useSession ? primaryConn.getDatabase(dbName) : primaryConn.getDB(dbName); + + let command = testCase.command(dbName, collName); + + if (testInTransaction || testAsRetryableWrite) { + command.txnNumber = kTxnNumber; + } + if (testInTransaction) { + command.startTransaction = true; + command.autocommit = false; + } + + return { + primaryConn, + primaryDB, + primaryHost: useSession ? primaryConn.getClient().host : primaryConn.host, + runAgainstAdminDb: testCase.runAgainstAdminDb, + command, + dbName, + collName, + useSession, + testInTransaction, + isBatchWrite: testCase.isBatchWrite, + isMultiUpdate: testCase.isMultiUpdate + }; +} + +function cleanUp(dbName) { + // To avoid disk space errors, ensure a new snapshot after dropping the DB, + // so subsequent 'Shard Merge' migrations don't copy it again. + const donorDB = donorPrimary.getDB(dbName); + assert.commandWorked(donorDB.dropDatabase()); +} + +function runTest( + primary, testCase, testFunc, dbName, collName, {testInTransaction, testAsRetryableWrite} = {}) { + const testOpts = makeTestOptions( + primary, testCase, dbName, collName, testInTransaction, testAsRetryableWrite); + jsTest.log("Testing testOpts: " + tojson(testOpts) + " with testFunc " + testFunc.name); + + if (testCase.explicitlyCreateCollection) { + createCollectionAndInsertDocs(testOpts.primaryDB, collName, testCase.isCapped); + } + + if (testCase.setUp) { + testCase.setUp(testOpts.primaryDB, collName, testInTransaction); + } + + testFunc(testCase, testOpts); + + // This cleanup step is necessary for the shard merge protocol to work correctly. + cleanUp(dbName); +} + +function runCommand(testOpts, expectedError) { + let res; + + if (testOpts.isMultiUpdate && !testOpts.testInTransaction) { + // Multi writes outside a transaction cannot be automatically retried, so we return a + // different error code than usual. This does not apply to the MaxTimeMS case because the + // error in that case is already not retryable. + if (expectedError == ErrorCodes.TenantMigrationCommitted || + expectedError == ErrorCodes.TenantMigrationAborted) { + expectedError = ErrorCodes.Interrupted; + } + } + + if (testOpts.testInTransaction) { + // Since oplog entries for write commands inside a transaction are not generated until the + // commitTransaction command is run, here we assert on the response of the commitTransaction + // command instead. + assert.commandWorked(testOpts.runAgainstAdminDb + ? testOpts.primaryDB.adminCommand(testOpts.command) + : testOpts.primaryDB.runCommand(testOpts.command)); + + let commitTxnCommand = { + commitTransaction: 1, + txnNumber: testOpts.command.txnNumber, + autocommit: false, + writeConcern: {w: "majority"} + }; + + // 'testBlockWritesAfterMigrationEnteredBlocking' runs each write command with maxTimeMS + // attached and asserts that the command blocks and fails with MaxTimeMSExpired. So in the + // case of transactions, we want to assert that commitTransaction blocks and fails + // MaxTimeMSExpired instead. + if (testOpts.command.maxTimeMS) { + commitTxnCommand.maxTimeMS = testOpts.command.maxTimeMS; + } + + res = testOpts.primaryDB.adminCommand(commitTxnCommand); + } else { + res = testOpts.runAgainstAdminDb ? testOpts.primaryDB.adminCommand(testOpts.command) + : testOpts.primaryDB.runCommand(testOpts.command); } - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testWritesNoMigration, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testWritesNoMigration, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); + if (expectedError) { + assert.commandFailedWithCode(res, expectedError); + + const expectTransientTransactionError = testOpts.testInTransaction && + (expectedError == ErrorCodes.TenantMigrationAborted || + expectedError == ErrorCodes.TenantMigrationCommitted); + if (expectTransientTransactionError) { + assert(res["errorLabels"] != null, "Error labels are absent from " + tojson(res)); + const expectedErrorLabels = ['TransientTransactionError']; + assert.sameMembers(res["errorLabels"], + expectedErrorLabels, + "Error labels " + tojson(res["errorLabels"]) + + " are different from expected " + expectedErrorLabels); + } + + const expectTopLevelError = !testOpts.isBatchWrite || + ErrorCodes.isInterruption(expectedError) || expectTransientTransactionError; + if (expectTopLevelError) { + assert.eq(res.code, expectedError, tojson(res)); + assert.eq(res.ok, 0, tojson(res)); + } else { + assert.isnull(res.code, tojson(res)); + assert.eq(res.ok, 1, tojson(res)); + } + } else { + assert.commandWorked(res); } +} + +/** + * Tests that the write succeeds when there is no migration. + */ +function testWritesNoMigration(testCase, testOpts) { + runCommand(testOpts); + testCase.assertCommandSucceeded(testOpts.primaryDB, testOpts.dbName, testOpts.collName); +} + +/** + * Tests that the donor rejects writes after the migration commits. + */ +function testRejectWritesAfterMigrationCommitted(testCase, testOpts) { + const tenantId = testOpts.dbName.split('_')[0]; + const migrationOpts = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId, + }; + + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts, { + retryOnRetryableErrors: false, + automaticForgetMigration: false, + enableDonorStartMigrationFsync: true + })); + + runCommand(testOpts, ErrorCodes.TenantMigrationCommitted); + testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); + checkTenantMigrationAccessBlocker( + testOpts.primaryDB, tenantId, {numTenantMigrationCommittedErrors: 1}); + + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +} + +/** + * Tests that the donor does not reject writes after the migration aborts. + */ +function testDoNotRejectWritesAfterMigrationAborted(testCase, testOpts) { + const tenantId = testOpts.dbName.split('_')[0]; + const migrationOpts = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId, + }; + + let abortFp = + configureFailPoint(testOpts.primaryDB, "abortTenantMigrationBeforeLeavingBlockingState"); + TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration(migrationOpts, { + retryOnRetryableErrors: false, + automaticForgetMigration: false, + enableDonorStartMigrationFsync: true + })); + abortFp.off(); + + // Wait until the in-memory migration state is updated after the migration has majority + // committed the abort decision. Otherwise, the command below is expected to block and then get + // rejected. + assert.soon(() => { + const mtab = TenantMigrationUtil.getTenantMigrationAccessBlocker( + {donorNode: testOpts.primaryDB, tenantId}); + return mtab.donor.state === TenantMigrationTest.DonorAccessState.kAborted; + }); + + runCommand(testOpts); + testCase.assertCommandSucceeded(testOpts.primaryDB, testOpts.dbName, testOpts.collName); + checkTenantMigrationAccessBlocker( + testOpts.primaryDB, tenantId, {numTenantMigrationAbortedErrors: 0}); + + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +} + +/** + * Tests that the donor blocks writes that are executed in the blocking state. + */ +function testBlockWritesAfterMigrationEnteredBlocking(testCase, testOpts) { + const tenantId = testOpts.dbName.split('_')[0]; + const migrationOpts = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId, + }; + + let blockingFp = + configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); + + assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); + + // Run the command after the migration enters the blocking state. + blockingFp.wait(); + testOpts.command.maxTimeMS = kMaxTimeMS; + runCommand(testOpts, ErrorCodes.MaxTimeMSExpired); + + // Allow the migration to complete. + blockingFp.off(); + TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete( + migrationOpts, false /* retryOnRetryableErrors */)); + + testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); + checkTenantMigrationAccessBlocker(testOpts.primaryDB, tenantId, {numBlockedWrites: 1}); + + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +} + +/** + * Tests that the donor blocks writes that are executed in the blocking state and rejects them after + * the migration commits. + */ +function testRejectBlockedWritesAfterMigrationCommitted(testCase, testOpts) { + const tenantId = testOpts.dbName.split('_')[0]; + const migrationOpts = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId, + }; + + let blockingFp = + configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); + + let resumeMigrationThread = + new Thread(resumeMigrationAfterBlockingWrite, testOpts.primaryHost, tenantId, 1); + + // Run the command after the migration enters the blocking state. + resumeMigrationThread.start(); + assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); + blockingFp.wait(); + + // The migration should unpause and commit after the write is blocked. Verify that the write is + // rejected. + runCommand(testOpts, ErrorCodes.TenantMigrationCommitted); + + // Verify that the migration succeeded. + resumeMigrationThread.join(); + TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete( + migrationOpts, false /* retryOnRetryableErrors */)); + + testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); + checkTenantMigrationAccessBlocker( + testOpts.primaryDB, tenantId, {numBlockedWrites: 1, numTenantMigrationCommittedErrors: 1}); + + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +} + +/** + * Tests that the donor blocks writes that are executed in the blocking state and rejects them after + * the migration aborts. + */ +function testRejectBlockedWritesAfterMigrationAborted(testCase, testOpts) { + const tenantId = testOpts.dbName.split('_')[0]; + const migrationOpts = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId, + }; + + let blockingFp = + configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); + let abortFp = + configureFailPoint(testOpts.primaryDB, "abortTenantMigrationBeforeLeavingBlockingState"); + + let resumeMigrationThread = + new Thread(resumeMigrationAfterBlockingWrite, testOpts.primaryHost, tenantId, 1); + + // Run the command after the migration enters the blocking state. + assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); + resumeMigrationThread.start(); + blockingFp.wait(); + + // The migration should unpause and abort after the write is blocked. Verify that the write is + // rejected. + runCommand(testOpts, ErrorCodes.TenantMigrationAborted); + + // Verify that the migration aborted due to the simulated error. + resumeMigrationThread.join(); + TenantMigrationTest.assertAborted(tenantMigrationTest.waitForMigrationToComplete( + migrationOpts, false /* retryOnRetryableErrors */)); + abortFp.off(); + + testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); + checkTenantMigrationAccessBlocker( + testOpts.primaryDB, tenantId, {numBlockedWrites: 1, numTenantMigrationAbortedErrors: 1}); + + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +} + +const isNotWriteCommand = "not a write command"; +const isNotRunOnUserDatabase = "not run on user database"; +const isNotSupportedInServerless = "not supported in serverless cluster"; +const isAuthCommand = "is an auth command"; +const isOnlySupportedOnStandalone = "is only supported on standalone"; +const isOnlySupportedOnShardedCluster = "is only supported on sharded cluster"; +const isDeprecated = "is only deprecated"; + +const testCases = { + _addShard: {skip: isNotRunOnUserDatabase}, + _cloneCollectionOptionsFromPrimaryShard: {skip: isNotRunOnUserDatabase}, + _configsvrAddShard: {skip: isNotRunOnUserDatabase}, + _configsvrAddShardToZone: {skip: isNotRunOnUserDatabase}, + _configsvrBalancerCollectionStatus: {skip: isNotRunOnUserDatabase}, + _configsvrBalancerStart: {skip: isNotRunOnUserDatabase}, + _configsvrBalancerStatus: {skip: isNotRunOnUserDatabase}, + _configsvrBalancerStop: {skip: isNotRunOnUserDatabase}, + _configsvrClearJumboFlag: {skip: isNotRunOnUserDatabase}, + _configsvrCommitChunksMerge: {skip: isNotRunOnUserDatabase}, + _configsvrCommitChunkMigration: {skip: isNotRunOnUserDatabase}, + _configsvrCommitChunkSplit: {skip: isNotRunOnUserDatabase}, + _configsvrCommitMovePrimary: + {skip: isNotRunOnUserDatabase}, // Can be removed once 6.0 is last LTS + _configsvrCreateDatabase: {skip: isNotRunOnUserDatabase}, + _configsvrEnsureChunkVersionIsGreaterThan: {skip: isNotRunOnUserDatabase}, + _configsvrMoveChunk: {skip: isNotRunOnUserDatabase}, // Can be removed once 6.0 is last LTS + _configsvrMovePrimary: {skip: isNotRunOnUserDatabase}, + _configsvrMoveRange: {skip: isNotRunOnUserDatabase}, + _configsvrRefineCollectionShardKey: {skip: isNotRunOnUserDatabase}, + _configsvrRemoveShard: {skip: isNotRunOnUserDatabase}, + _configsvrRemoveShardFromZone: {skip: isNotRunOnUserDatabase}, + _configsvrUpdateZoneKeyRange: {skip: isNotRunOnUserDatabase}, + _flushDatabaseCacheUpdates: {skip: isNotRunOnUserDatabase}, + _flushDatabaseCacheUpdatesWithWriteConcern: {skip: isNotRunOnUserDatabase}, + _flushReshardingStateChange: {skip: isNotRunOnUserDatabase}, + _flushRoutingTableCacheUpdates: {skip: isNotRunOnUserDatabase}, + _flushRoutingTableCacheUpdatesWithWriteConcern: {skip: isNotRunOnUserDatabase}, + _getNextSessionMods: {skip: isNotRunOnUserDatabase}, + _getUserCacheGeneration: {skip: isNotRunOnUserDatabase}, + _hashBSONElement: {skip: isNotRunOnUserDatabase}, + _isSelf: {skip: isNotRunOnUserDatabase}, + _killOperations: {skip: isNotRunOnUserDatabase}, + _mergeAuthzCollections: {skip: isNotRunOnUserDatabase}, + _migrateClone: {skip: isNotRunOnUserDatabase}, + _recvChunkAbort: {skip: isNotRunOnUserDatabase}, + _recvChunkCommit: {skip: isNotRunOnUserDatabase}, + _recvChunkReleaseCritSec: {skip: isNotRunOnUserDatabase}, + _recvChunkStart: {skip: isNotRunOnUserDatabase}, + _recvChunkStatus: {skip: isNotRunOnUserDatabase}, + _shardsvrCloneCatalogData: {skip: isNotRunOnUserDatabase}, + _shardsvrCompactStructuredEncryptionData: {skip: isOnlySupportedOnShardedCluster}, + _shardsvrCreateCollection: {skip: isOnlySupportedOnShardedCluster}, + _shardsvrCreateCollectionParticipant: {skip: isOnlySupportedOnShardedCluster}, + _shardsvrMovePrimary: {skip: isNotRunOnUserDatabase}, + _shardsvrSetAllowMigrations: {skip: isOnlySupportedOnShardedCluster}, + _shardsvrShardCollection: + {skip: isNotRunOnUserDatabase}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS + _shardsvrRenameCollection: {skip: isOnlySupportedOnShardedCluster}, + _transferMods: {skip: isNotRunOnUserDatabase}, + abortTransaction: { + skip: isNotWriteCommand // aborting unprepared transaction doesn't create an abort oplog + // entry. + }, + aggregate: { + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return { + aggregate: collName, + pipeline: [{$out: collName + "Out"}], + cursor: {batchSize: 1} + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(collectionExists(db, collName + "Out")); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!collectionExists(db, collName + "Out")); + } + }, + appendOplogNote: {skip: isNotRunOnUserDatabase}, + applyOps: {skip: isNotSupportedInServerless}, + authenticate: {skip: isAuthCommand}, + availableQueryOptions: {skip: isNotWriteCommand}, + buildInfo: {skip: isNotWriteCommand}, + captrunc: { + skip: isNotWriteCommand, // TODO (SERVER-49834) + explicitlyCreateCollection: true, // creates a collection with kNumInitialDocs > 1 docs. + isCapped: true, + command: function(dbName, collName) { + return {captrunc: collName, n: 1}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, {}), 1); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, {}), kNumInitialDocs); + } + }, + checkShardingIndex: {skip: isNotRunOnUserDatabase}, + cleanupOrphaned: {skip: isNotRunOnUserDatabase}, + clearLog: {skip: isNotRunOnUserDatabase}, + cloneCollectionAsCapped: { + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return { + cloneCollectionAsCapped: collName, + toCollection: collName + "CloneCollectionAsCapped", + size: kMaxSize + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(collectionExists(db, collName + "CloneCollectionAsCapped")); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!collectionExists(db, collName + "CloneCollectionAsCapped")); + } + }, + collMod: { + explicitlyCreateCollection: true, + setUp: createTestIndex, + command: function(dbName, collName) { + return { + collMod: collName, + index: {keyPattern: kTestIndexKey, expireAfterSeconds: kExpireAfterSeconds + 1} + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(indexExists( + db, collName, {key: kTestIndexKey, expireAfterSeconds: kExpireAfterSeconds + 1})); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!indexExists( + db, collName, {key: kTestIndexKey, expireAfterSeconds: kExpireAfterSeconds + 1})); + } + }, + collStats: {skip: isNotWriteCommand}, + commitTransaction: { + isTransactionCommand: true, + runAgainstAdminDb: true, + setUp: function(primaryDB, collName) { + assert.commandWorked(primaryDB.runCommand({ + insert: collName, + documents: [kTestDoc], + txnNumber: NumberLong(kTxnNumber), + startTransaction: true, + autocommit: false + })); + }, + command: function(dbName, collName) { + return { + commitTransaction: 1, + txnNumber: NumberLong(kTxnNumber), + autocommit: false, + writeConcern: {w: "majority"} + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName), 1); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName), 0); + } + }, + compact: { + skip: isNotWriteCommand, // TODO (SERVER-49834) + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {compact: collName, force: true}; + }, + assertCommandSucceeded: function(db, dbName, collName) {}, + assertCommandFailed: function(db, dbName, collName) {} + }, + configureFailPoint: {skip: isNotRunOnUserDatabase}, + connPoolStats: {skip: isNotRunOnUserDatabase}, + connPoolSync: {skip: isNotRunOnUserDatabase}, + connectionStatus: {skip: isNotRunOnUserDatabase}, + convertToCapped: { + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {convertToCapped: collName, size: kMaxSize}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(db[collName].stats().capped); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!db[collName].stats().capped); + } + }, + coordinateCommitTransaction: {skip: isNotRunOnUserDatabase}, + count: {skip: isNotWriteCommand}, + cpuload: {skip: isNotRunOnUserDatabase}, + create: { + testInTransaction: true, + command: function(dbName, collName) { + return {create: collName}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(collectionExists(db, collName)); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!collectionExists(db, collName)); + } + }, + createIndexes: { + testInTransaction: true, + explicitlyCreateCollection: true, + setUp: function(primaryDB, collName, testInTransaction) { + if (testInTransaction) { + // Drop the collection that was explicitly created above since inside transactions + // the index to create must either be on a non-existing collection, or on a new + // empty collection created earlier in the same transaction. + assert.commandWorked(primaryDB.runCommand({drop: collName})); + } + }, + command: function(dbName, collName) { + return {createIndexes: collName, indexes: [kTestIndex]}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(indexExists(db, collName, kTestIndex)); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!collectionExists(db, collName) || !indexExists(db, collName, kTestIndex)); + } + }, + createRole: {skip: isAuthCommand}, + createUser: {skip: isAuthCommand}, + currentOp: {skip: isNotRunOnUserDatabase}, + dataSize: {skip: isNotWriteCommand}, + dbCheck: {skip: isNotWriteCommand}, + dbHash: {skip: isNotWriteCommand}, + dbStats: {skip: isNotWriteCommand}, + delete: { + testInTransaction: true, + testAsRetryableWrite: true, + setUp: insertTestDoc, + command: function(dbName, collName) { + return {delete: collName, deletes: [{q: kTestDoc, limit: 1}]}; + }, + isBatchWrite: true, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 0); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 1); + } + }, + distinct: {skip: isNotWriteCommand}, + donorForgetMigration: {skip: isNotRunOnUserDatabase}, + donorStartMigration: {skip: isNotRunOnUserDatabase}, + donorWaitForMigrationToCommit: {skip: isNotRunOnUserDatabase}, + driverOIDTest: {skip: isNotRunOnUserDatabase}, + drop: { + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {drop: collName}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(!collectionExists(db, collName)); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(collectionExists(db, collName)); + } + }, + dropAllRolesFromDatabase: {skip: isAuthCommand}, + dropAllUsersFromDatabase: {skip: isAuthCommand}, + dropConnections: {skip: isNotRunOnUserDatabase}, + dropDatabase: { + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {dropDatabase: 1}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(!databaseExists(db, dbName)); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(databaseExists(db, dbName)); + } + }, + dropIndexes: { + explicitlyCreateCollection: true, + setUp: createTestIndex, + command: function(dbName, collName) { + return {dropIndexes: collName, index: "*"}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(!indexExists(db, collName, kTestIndex)); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(indexExists(db, collName, kTestIndex)); + } + }, + dropRole: {skip: isAuthCommand}, + dropUser: {skip: isAuthCommand}, + echo: {skip: isNotRunOnUserDatabase}, + emptycapped: { + explicitlyCreateCollection: true, + setUp: insertTestDoc, + command: function(dbName, collName) { + return {emptycapped: collName}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 0); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 1); + } + }, + endSessions: {skip: isNotRunOnUserDatabase}, + explain: {skip: isNotRunOnUserDatabase}, + features: {skip: isNotRunOnUserDatabase}, + filemd5: {skip: isNotWriteCommand}, + find: {skip: isNotWriteCommand}, + findAndModify: { + testInTransaction: true, + testAsRetryableWrite: true, + setUp: insertTestDoc, + command: function(dbName, collName) { + return {findAndModify: collName, query: kTestDoc, remove: true}; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 0); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 1); + } + }, + flushRouterConfig: {skip: isNotRunOnUserDatabase}, + fsync: {skip: isNotRunOnUserDatabase}, + fsyncUnlock: {skip: isNotRunOnUserDatabase}, + getCmdLineOpts: {skip: isNotRunOnUserDatabase}, + getDatabaseVersion: {skip: isNotRunOnUserDatabase}, + getDefaultRWConcern: {skip: isNotRunOnUserDatabase}, + getDiagnosticData: {skip: isNotRunOnUserDatabase}, + getFreeMonitoringStatus: {skip: isNotRunOnUserDatabase}, + getLastError: {skip: isNotWriteCommand}, + getLog: {skip: isNotRunOnUserDatabase}, + getMore: {skip: isNotWriteCommand}, + getParameter: {skip: isNotRunOnUserDatabase}, + getShardMap: {skip: isNotRunOnUserDatabase}, + getShardVersion: {skip: isNotRunOnUserDatabase}, + getnonce: {skip: isNotRunOnUserDatabase}, + godinsert: {skip: isNotRunOnUserDatabase}, + grantPrivilegesToRole: {skip: isAuthCommand}, + grantRolesToRole: {skip: isAuthCommand}, + grantRolesToUser: {skip: isAuthCommand}, + hello: {skip: isNotRunOnUserDatabase}, + hostInfo: {skip: isNotRunOnUserDatabase}, + httpClientRequest: {skip: isNotRunOnUserDatabase}, + insert: { + testInTransaction: true, + testAsRetryableWrite: true, + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {insert: collName, documents: [kTestDoc]}; + }, + isBatchWrite: true, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 1); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, kTestDoc), 0); + } + }, + internalRenameIfOptionsAndIndexesMatch: {skip: isNotRunOnUserDatabase}, + invalidateUserCache: {skip: isNotRunOnUserDatabase}, + killAllSessions: {skip: isNotRunOnUserDatabase}, + killAllSessionsByPattern: {skip: isNotRunOnUserDatabase}, + killCursors: {skip: isNotWriteCommand}, + killOp: {skip: isNotRunOnUserDatabase}, + killSessions: {skip: isNotRunOnUserDatabase}, + listCollections: {skip: isNotRunOnUserDatabase}, + listCommands: {skip: isNotRunOnUserDatabase}, + listDatabases: {skip: isNotRunOnUserDatabase}, + listIndexes: {skip: isNotWriteCommand}, + lockInfo: {skip: isNotRunOnUserDatabase}, + logRotate: {skip: isNotRunOnUserDatabase}, + logout: {skip: isNotRunOnUserDatabase}, + makeSnapshot: {skip: isNotRunOnUserDatabase}, + mapReduce: { + command: function(dbName, collName) { + return { + mapReduce: collName, + map: function mapFunc() { + emit(this.x, 1); + }, + reduce: function reduceFunc(key, values) { + return Array.sum(values); + }, + out: {replace: collName + "MrOut"}, + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(collectionExists(db, collName + "MrOut")); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(!collectionExists(db, collName + "MrOut")); + } + }, + mergeChunks: {skip: isNotRunOnUserDatabase}, + moveChunk: {skip: isNotRunOnUserDatabase}, + ping: {skip: isNotRunOnUserDatabase}, + planCacheClear: {skip: isNotWriteCommand}, + planCacheClearFilters: {skip: isNotWriteCommand}, + planCacheListFilters: {skip: isNotWriteCommand}, + planCacheSetFilter: {skip: isNotWriteCommand}, + prepareTransaction: {skip: isOnlySupportedOnShardedCluster}, + profile: {skip: isNotRunOnUserDatabase}, + reIndex: {skip: isOnlySupportedOnStandalone}, + reapLogicalSessionCacheNow: {skip: isNotRunOnUserDatabase}, + refreshLogicalSessionCacheNow: {skip: isNotRunOnUserDatabase}, + refreshSessions: {skip: isNotRunOnUserDatabase}, + recipientVoteImportedFiles: {skip: isNotRunOnUserDatabase}, + renameCollection: { + runAgainstAdminDb: true, + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return { + renameCollection: dbName + "." + collName, + to: dbName + "." + collName + "Renamed" + }; + }, + assertCommandSucceeded: function(db, dbName, collName) { + assert(!collectionExists(db, collName)); + assert(collectionExists(db, collName + "Renamed")); + }, + assertCommandFailed: function(db, dbName, collName) { + assert(collectionExists(db, collName)); + assert(!collectionExists(db, collName + "Renamed")); + } + }, + repairDatabase: {skip: isDeprecated}, + replSetAbortPrimaryCatchUp: {skip: isNotRunOnUserDatabase}, + replSetFreeze: {skip: isNotRunOnUserDatabase}, + replSetGetConfig: {skip: isNotRunOnUserDatabase}, + replSetGetRBID: {skip: isNotRunOnUserDatabase}, + replSetGetStatus: {skip: isNotRunOnUserDatabase}, + replSetHeartbeat: {skip: isNotRunOnUserDatabase}, + replSetInitiate: {skip: isNotRunOnUserDatabase}, + replSetMaintenance: {skip: isNotRunOnUserDatabase}, + replSetReconfig: {skip: isNotRunOnUserDatabase}, + replSetRequestVotes: {skip: isNotRunOnUserDatabase}, + replSetResizeOplog: {skip: isNotRunOnUserDatabase}, + replSetStepDown: {skip: isNotRunOnUserDatabase}, + replSetStepUp: {skip: isNotRunOnUserDatabase}, + replSetSyncFrom: {skip: isNotRunOnUserDatabase}, + replSetTest: {skip: isNotRunOnUserDatabase}, + replSetTestEgress: {skip: isNotRunOnUserDatabase}, + replSetUpdatePosition: {skip: isNotRunOnUserDatabase}, + revokePrivilegesFromRole: {skip: isAuthCommand}, + revokeRolesFromRole: {skip: isAuthCommand}, + revokeRolesFromUser: {skip: isAuthCommand}, + rolesInfo: {skip: isNotWriteCommand}, + rotateCertificates: {skip: isAuthCommand}, + saslContinue: {skip: isAuthCommand}, + saslStart: {skip: isAuthCommand}, + sbe: {skip: isNotRunOnUserDatabase}, + serverStatus: {skip: isNotRunOnUserDatabase}, + setAllowMigrations: {skip: isNotRunOnUserDatabase}, + setCommittedSnapshot: {skip: isNotRunOnUserDatabase}, + setDefaultRWConcern: {skip: isNotRunOnUserDatabase}, + setFeatureCompatibilityVersion: {skip: isNotRunOnUserDatabase}, + setFreeMonitoring: {skip: isNotRunOnUserDatabase}, + setIndexCommitQuorum: {skip: isNotRunOnUserDatabase}, + setParameter: {skip: isNotRunOnUserDatabase}, + setShardVersion: {skip: isNotRunOnUserDatabase}, + shardingState: {skip: isNotRunOnUserDatabase}, + shutdown: {skip: isNotRunOnUserDatabase}, + sleep: {skip: isNotRunOnUserDatabase}, + splitChunk: {skip: isNotRunOnUserDatabase}, + splitVector: {skip: isNotRunOnUserDatabase}, + stageDebug: {skip: isNotRunOnUserDatabase}, + startRecordingTraffic: {skip: isNotRunOnUserDatabase}, + startSession: {skip: isNotRunOnUserDatabase}, + stopRecordingTraffic: {skip: isNotRunOnUserDatabase}, + top: {skip: isNotRunOnUserDatabase}, + update: { + testInTransaction: true, + testAsRetryableWrite: true, + setUp: insertTestDoc, + command: function(dbName, collName) { + return { + update: collName, + updates: [{q: kTestDoc, u: {$set: {y: 0}}, upsert: false, multi: false}] + }; + }, + isBatchWrite: true, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, Object.assign({y: 0}, kTestDoc)), 1); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, Object.assign({y: 0}, kTestDoc)), 0); + } + }, + multiUpdate: { + testInTransaction: true, + testAsRetryableWrite: false, + setUp: insertTwoTestDocs, + command: function(dbName, collName) { + return { + update: collName, + updates: [{q: {}, u: {$set: {y: 0}}, upsert: false, multi: true}] + }; + }, + isBatchWrite: true, + isMultiUpdate: true, + assertCommandSucceeded: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, Object.assign({y: 0})), 2); + }, + assertCommandFailed: function(db, dbName, collName) { + assert.eq(countDocs(db, collName, Object.assign({y: 0})), 0); + } + }, + updateRole: {skip: isAuthCommand}, + updateUser: {skip: isNotRunOnUserDatabase}, + usersInfo: {skip: isNotRunOnUserDatabase}, + validate: {skip: isNotWriteCommand}, + voteCommitIndexBuild: {skip: isNotRunOnUserDatabase}, + // TODO (SERVER-64296): Remove voteCommitMigrationProgress in 6.1. + voteCommitMigrationProgress: {skip: isNotRunOnUserDatabase}, + waitForFailPoint: {skip: isNotRunOnUserDatabase}, + waitForOngoingChunkSplits: {skip: isNotRunOnUserDatabase}, + whatsmysni: {skip: isNotRunOnUserDatabase}, + whatsmyuri: {skip: isNotRunOnUserDatabase} +}; + +// Validate test cases for all commands. +for (let command of Object.keys(testCases)) { + validateTestCase(testCases[command]); +} + +// Run test cases. +const testFuncs = { + noMigration: testWritesNoMigration, // verify that the test cases are correct. + inCommitted: testRejectWritesAfterMigrationCommitted, + inAborted: testDoNotRejectWritesAfterMigrationAborted, + inBlocking: testBlockWritesAfterMigrationEnteredBlocking, + inBlockingThenCommitted: testRejectBlockedWritesAfterMigrationCommitted, + inBlockingThenAborted: testRejectBlockedWritesAfterMigrationAborted +}; + +for (const [testName, testFunc] of Object.entries(testFuncs)) { + for (const [commandName, testCase] of Object.entries(testCases)) { + let baseDbName = commandName + "-" + testName + "0"; + + if (testCase.skip) { + print("Skipping " + commandName + ": " + testCase.skip); + continue; + } + + runTest(donorPrimary, + testCase, + testFunc, + baseDbName + "Basic_" + kTenantDefinedDbName, + kCollName); + + if (testCase.testInTransaction) { + runTest(donorPrimary, + testCase, + testFunc, + baseDbName + "Txn_" + kTenantDefinedDbName, + kCollName, + {testInTransaction: true}); + } - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testWritesNoMigration, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); + if (testCase.testAsRetryableWrite) { + runTest(donorPrimary, + testCase, + testFunc, + baseDbName + "Retryable_" + kTenantDefinedDbName, + kCollName, + {testAsRetryableWrite: true}); + } } } diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_aborted.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_aborted.js deleted file mode 100644 index 2d6ba4e4d3e..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_aborted.js +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Tests that the donor accepts writes after the migration aborts. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); - -const donorRst = tenantMigrationTest.getDonorRst(); -const donorPrimary = donorRst.getPrimary(); - -const kCollName = "testColl"; - -const kTenantDefinedDbName = "0"; - -/** - * Tests that the donor does not reject writes after the migration aborts. - */ -function testDoNotRejectWritesAfterMigrationAborted(testCase, testOpts) { - const tenantId = testOpts.dbName.split('_')[0]; - const migrationOpts = { - migrationIdString: extractUUIDFromObject(UUID()), - tenantId, - }; - - let abortFp = - configureFailPoint(testOpts.primaryDB, "abortTenantMigrationBeforeLeavingBlockingState"); - TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration(migrationOpts, { - retryOnRetryableErrors: false, - automaticForgetMigration: false, - enableDonorStartMigrationFsync: true - })); - abortFp.off(); - - // Wait until the in-memory migration state is updated after the migration has majority - // committed the abort decision. Otherwise, the command below is expected to block and then get - // rejected. - assert.soon(() => { - const mtab = TenantMigrationUtil.getTenantMigrationAccessBlocker( - {donorNode: testOpts.primaryDB, tenantId}); - return mtab.donor.state === TenantMigrationTest.DonorAccessState.kAborted; - }); - - runCommandForConcurrentWritesTest(testOpts); - testCase.assertCommandSucceeded(testOpts.primaryDB, testOpts.dbName, testOpts.collName); - checkTenantMigrationAccessBlockerForConcurrentWritesTest( - testOpts.primaryDB, tenantId, {numTenantMigrationAbortedErrors: 0}); - - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); -} - -const testCases = TenantMigrationConcurrentWriteUtil.testCases; - -// Run test cases after an aborted migration. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-inAborted0"; - - if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; - } - - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testDoNotRejectWritesAfterMigrationAborted, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testDoNotRejectWritesAfterMigrationAborted, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); - } - - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testDoNotRejectWritesAfterMigrationAborted, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); - } -} - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking.js deleted file mode 100644 index f4737d26d54..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking.js +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Tests that the donor blocks writes that are executed while the migration in the blocking state, - * then rejects the writes when the migration completes. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); - -const donorRst = tenantMigrationTest.getDonorRst(); -const donorPrimary = donorRst.getPrimary(); - -const kCollName = "testColl"; - -const kTenantDefinedDbName = "0"; - -const kMaxTimeMS = 1 * 1000; - -/** - * Tests that the donor blocks writes that are executed in the blocking state. - */ -function testBlockWritesAfterMigrationEnteredBlocking(testCase, testOpts) { - const tenantId = testOpts.dbName.split('_')[0]; - const migrationOpts = { - migrationIdString: extractUUIDFromObject(UUID()), - tenantId, - }; - - let blockingFp = - configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); - - assert.commandWorked( - tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); - - // Run the command after the migration enters the blocking state. - blockingFp.wait(); - testOpts.command.maxTimeMS = kMaxTimeMS; - runCommandForConcurrentWritesTest(testOpts, ErrorCodes.MaxTimeMSExpired); - - // Allow the migration to complete. - blockingFp.off(); - TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete( - migrationOpts, false /* retryOnRetryableErrors */)); - - testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); - checkTenantMigrationAccessBlockerForConcurrentWritesTest( - testOpts.primaryDB, tenantId, {numBlockedWrites: 1}); - - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); -} - -const testCases = TenantMigrationConcurrentWriteUtil.testCases; - -// Run test cases while the migration is in blocking state. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-inBlocking0"; - - if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; - } - - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testBlockWritesAfterMigrationEnteredBlocking, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testBlockWritesAfterMigrationEnteredBlocking, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); - } - - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testBlockWritesAfterMigrationEnteredBlocking, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); - } -} - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_aborted.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_aborted.js deleted file mode 100644 index a6446dbfc1a..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_aborted.js +++ /dev/null @@ -1,131 +0,0 @@ -/** - * Tests that the donor blocks writes that are executed while the migration in the blocking state, - * then rejects the writes when the migration aborted. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); - -const donorRst = tenantMigrationTest.getDonorRst(); -const donorPrimary = donorRst.getPrimary(); - -const kCollName = "testColl"; - -const kTenantDefinedDbName = "0"; - -/** - * To be used to resume a migration that is paused after entering the blocking state. Waits for the - * number of blocked reads to reach 'targetNumBlockedWrites' and unpauses the migration. - */ -function resumeMigrationAfterBlockingWrite(host, tenantId, targetNumBlockedWrites) { - load("jstests/libs/fail_point_util.js"); - load("jstests/replsets/libs/tenant_migration_util.js"); - const primary = new Mongo(host); - - assert.soon(() => TenantMigrationUtil.getNumBlockedWrites(primary, tenantId) == - targetNumBlockedWrites); - - assert.commandWorked(primary.adminCommand( - {configureFailPoint: "pauseTenantMigrationBeforeLeavingBlockingState", mode: "off"})); -} - -/** - * Tests that the donor blocks writes that are executed in the blocking state and rejects them after - * the migration aborts. - */ -function testRejectBlockedWritesAfterMigrationAborted(testCase, testOpts) { - const tenantId = testOpts.dbName.split('_')[0]; - const migrationOpts = { - migrationIdString: extractUUIDFromObject(UUID()), - tenantId, - }; - - let blockingFp = - configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); - let abortFp = - configureFailPoint(testOpts.primaryDB, "abortTenantMigrationBeforeLeavingBlockingState"); - - let resumeMigrationThread = - new Thread(resumeMigrationAfterBlockingWrite, testOpts.primaryHost, tenantId, 1); - - // Run the command after the migration enters the blocking state. - assert.commandWorked( - tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); - resumeMigrationThread.start(); - blockingFp.wait(); - - // The migration should unpause and abort after the write is blocked. Verify that the write is - // rejected. - runCommandForConcurrentWritesTest(testOpts, ErrorCodes.TenantMigrationAborted); - - // Verify that the migration aborted due to the simulated error. - resumeMigrationThread.join(); - TenantMigrationTest.assertAborted(tenantMigrationTest.waitForMigrationToComplete( - migrationOpts, false /* retryOnRetryableErrors */)); - abortFp.off(); - - testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); - checkTenantMigrationAccessBlockerForConcurrentWritesTest( - testOpts.primaryDB, tenantId, {numBlockedWrites: 1, numTenantMigrationAbortedErrors: 1}); - - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); -} - -const testCases = TenantMigrationConcurrentWriteUtil.testCases; - -// Run test cases while the migration is blocked and then rejects after aborted. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-inBlockingThenAborted0"; - - if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; - } - - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationAborted, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationAborted, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); - } - - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationAborted, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); - } -} - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_committed.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_committed.js deleted file mode 100644 index 777b590a375..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_blocking_then_committed.js +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Tests that the donor blocks writes that are executed while the migration in the blocking state, - * then rejects the writes when the migration committed. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); - -const donorRst = tenantMigrationTest.getDonorRst(); -const donorPrimary = donorRst.getPrimary(); - -const kCollName = "testColl"; - -const kTenantDefinedDbName = "0"; - -/** - * To be used to resume a migration that is paused after entering the blocking state. Waits for the - * number of blocked reads to reach 'targetNumBlockedWrites' and unpauses the migration. - */ -function resumeMigrationAfterBlockingWrite(host, tenantId, targetNumBlockedWrites) { - load("jstests/libs/fail_point_util.js"); - load("jstests/replsets/libs/tenant_migration_util.js"); - const primary = new Mongo(host); - - assert.soon(() => TenantMigrationUtil.getNumBlockedWrites(primary, tenantId) == - targetNumBlockedWrites); - - assert.commandWorked(primary.adminCommand( - {configureFailPoint: "pauseTenantMigrationBeforeLeavingBlockingState", mode: "off"})); -} - -/** - * Tests that the donor blocks writes that are executed in the blocking state and rejects them after - * the migration commits. - */ -function testRejectBlockedWritesAfterMigrationCommitted(testCase, testOpts) { - const tenantId = testOpts.dbName.split('_')[0]; - const migrationOpts = { - migrationIdString: extractUUIDFromObject(UUID()), - tenantId, - }; - - let blockingFp = - configureFailPoint(testOpts.primaryDB, "pauseTenantMigrationBeforeLeavingBlockingState"); - - let resumeMigrationThread = - new Thread(resumeMigrationAfterBlockingWrite, testOpts.primaryHost, tenantId, 1); - - // Run the command after the migration enters the blocking state. - resumeMigrationThread.start(); - assert.commandWorked( - tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); - blockingFp.wait(); - - // The migration should unpause and commit after the write is blocked. Verify that the write is - // rejected. - runCommandForConcurrentWritesTest(testOpts, ErrorCodes.TenantMigrationCommitted); - - // Verify that the migration succeeded. - resumeMigrationThread.join(); - TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete( - migrationOpts, false /* retryOnRetryableErrors */)); - - testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); - checkTenantMigrationAccessBlockerForConcurrentWritesTest( - testOpts.primaryDB, tenantId, {numBlockedWrites: 1, numTenantMigrationCommittedErrors: 1}); - - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); -} - -const testCases = TenantMigrationConcurrentWriteUtil.testCases; - -// Run test cases while the migration is blocked and then rejects after committed. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-inBlockingThenCommitted0"; - - if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; - } - - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationCommitted, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationCommitted, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); - } - - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectBlockedWritesAfterMigrationCommitted, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); - } -} - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_committed.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_committed.js deleted file mode 100644 index 63bec6201ee..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_committed.js +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Tests that the donor blocks writes that are executed after the migration committed are rejected. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ -(function() { -'use strict'; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); -load("jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js"); - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - quickGarbageCollection: true, -}); - -const donorRst = tenantMigrationTest.getDonorRst(); -const donorPrimary = donorRst.getPrimary(); - -const kCollName = "testColl"; -const kTenantDefinedDbName = "0"; - -/** - * Tests that the donor rejects writes after the migration commits. - */ -function testRejectWritesAfterMigrationCommitted(testCase, testOpts) { - const tenantId = testOpts.dbName.split('_')[0]; - const migrationOpts = { - migrationIdString: extractUUIDFromObject(UUID()), - tenantId, - }; - - TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts, { - retryOnRetryableErrors: false, - automaticForgetMigration: false, - enableDonorStartMigrationFsync: true - })); - - runCommandForConcurrentWritesTest(testOpts, ErrorCodes.TenantMigrationCommitted); - testCase.assertCommandFailed(testOpts.primaryDB, testOpts.dbName, testOpts.collName); - checkTenantMigrationAccessBlockerForConcurrentWritesTest( - testOpts.primaryDB, tenantId, {numTenantMigrationCommittedErrors: 1}); - - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); -} - -const testCases = TenantMigrationConcurrentWriteUtil.testCases; - -// Run test cases after the migration has committed. -for (const [commandName, testCase] of Object.entries(testCases)) { - let baseDbName = commandName + "-inCommitted0"; - - if (testCase.skip) { - print("Skipping " + commandName + ": " + testCase.skip); - continue; - } - - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectWritesAfterMigrationCommitted, - baseDbName + "Basic_" + kTenantDefinedDbName, - kCollName); - - if (testCase.testInTransaction) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectWritesAfterMigrationCommitted, - baseDbName + "Txn_" + kTenantDefinedDbName, - kCollName, - {testInTransaction: true}); - } - - if (testCase.testAsRetryableWrite) { - runTestForConcurrentWritesTest(donorPrimary, - testCase, - testRejectWritesAfterMigrationCommitted, - baseDbName + "Retryable_" + kTenantDefinedDbName, - kCollName, - {testAsRetryableWrite: true}); - } -} - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js deleted file mode 100644 index f561b661352..00000000000 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor_util.js +++ /dev/null @@ -1,826 +0,0 @@ -/** - * This utility file is used to list the different test cases needed for the - * tenant_migration_concurrent_writes_on_donor*tests. - */ - -'use strict'; - -var TenantMigrationConcurrentWriteUtil = (function() {}); - -/** - * Asserts that the TenantMigrationAccessBlocker for the given tenant on the given node has the - * expected statistics. - */ -function checkTenantMigrationAccessBlockerForConcurrentWritesTest(node, tenantId, { - numBlockedWrites = 0, - numTenantMigrationCommittedErrors = 0, - numTenantMigrationAbortedErrors = 0 -}) { - const mtab = - TenantMigrationUtil.getTenantMigrationAccessBlocker({donorNode: node, tenantId}).donor; - if (!mtab) { - assert.eq(0, numBlockedWrites); - assert.eq(0, numTenantMigrationCommittedErrors); - assert.eq(0, numTenantMigrationAbortedErrors); - return; - } - - assert.eq(mtab.numBlockedReads, 0, tojson(mtab)); - assert.eq(mtab.numBlockedWrites, numBlockedWrites, tojson(mtab)); - assert.eq( - mtab.numTenantMigrationCommittedErrors, numTenantMigrationCommittedErrors, tojson(mtab)); - assert.eq(mtab.numTenantMigrationAbortedErrors, numTenantMigrationAbortedErrors, tojson(mtab)); -} - -function runCommandForConcurrentWritesTest(testOpts, expectedError) { - let res; - - if (testOpts.isMultiUpdate && !testOpts.testInTransaction) { - // Multi writes outside a transaction cannot be automatically retried, so we return a - // different error code than usual. This does not apply to the MaxTimeMS case because the - // error in that case is already not retryable. - if (expectedError == ErrorCodes.TenantMigrationCommitted || - expectedError == ErrorCodes.TenantMigrationAborted) { - expectedError = ErrorCodes.Interrupted; - } - } - - if (testOpts.testInTransaction) { - // Since oplog entries for write commands inside a transaction are not generated until the - // commitTransaction command is run, here we assert on the response of the commitTransaction - // command instead. - assert.commandWorked(testOpts.runAgainstAdminDb - ? testOpts.primaryDB.adminCommand(testOpts.command) - : testOpts.primaryDB.runCommand(testOpts.command)); - - let commitTxnCommand = { - commitTransaction: 1, - txnNumber: testOpts.command.txnNumber, - autocommit: false, - writeConcern: {w: "majority"} - }; - - // 'testBlockWritesAfterMigrationEnteredBlocking' runs each write command with maxTimeMS - // attached and asserts that the command blocks and fails with MaxTimeMSExpired. So in the - // case of transactions, we want to assert that commitTransaction blocks and fails - // MaxTimeMSExpired instead. - if (testOpts.command.maxTimeMS) { - commitTxnCommand.maxTimeMS = testOpts.command.maxTimeMS; - } - - res = testOpts.primaryDB.adminCommand(commitTxnCommand); - } else { - res = testOpts.runAgainstAdminDb ? testOpts.primaryDB.adminCommand(testOpts.command) - : testOpts.primaryDB.runCommand(testOpts.command); - } - - if (expectedError) { - assert.commandFailedWithCode(res, expectedError); - - const expectTransientTransactionError = testOpts.testInTransaction && - (expectedError == ErrorCodes.TenantMigrationAborted || - expectedError == ErrorCodes.TenantMigrationCommitted); - if (expectTransientTransactionError) { - assert(res["errorLabels"] != null, "Error labels are absent from " + tojson(res)); - const expectedErrorLabels = ['TransientTransactionError']; - assert.sameMembers(res["errorLabels"], - expectedErrorLabels, - "Error labels " + tojson(res["errorLabels"]) + - " are different from expected " + expectedErrorLabels); - } - - const expectTopLevelError = !testOpts.isBatchWrite || - ErrorCodes.isInterruption(expectedError) || expectTransientTransactionError; - if (expectTopLevelError) { - assert.eq(res.code, expectedError, tojson(res)); - assert.eq(res.ok, 0, tojson(res)); - } else { - assert.isnull(res.code, tojson(res)); - assert.eq(res.ok, 1, tojson(res)); - } - } else { - assert.commandWorked(res); - } -} - -function createCollectionAndInsertDocsForConcurrentWritesTest( - primaryDB, collName, isCapped, numDocs = TenantMigrationConcurrentWriteUtil.kNumInitialDocs) { - const createCollCommand = {create: collName}; - if (isCapped) { - createCollCommand.capped = true; - createCollCommand.size = kMaxSize; - } - assert.commandWorked(primaryDB.runCommand(createCollCommand)); - - let bulk = primaryDB[collName].initializeUnorderedBulkOp(); - for (let i = 0; i < numDocs; ++i) { - bulk.insert({x: i}); - } - assert.commandWorked(bulk.execute()); -} - -function cleanUpForConcurrentWritesTest(dbName, donorPrimary) { - // To avoid disk space errors, ensure a new snapshot after dropping the DB, - // so subsequent 'Shard Merge' migrations don't copy it again. - const donorDB = donorPrimary.getDB(dbName); - assert.commandWorked(donorDB.dropDatabase()); -} - -function makeTestOptionsForConcurrentWritesTest( - primary, testCase, dbName, collName, testInTransaction, testAsRetryableWrite) { - assert(!testInTransaction || !testAsRetryableWrite); - - const useSession = testInTransaction || testAsRetryableWrite || testCase.isTransactionCommand; - const primaryConn = useSession ? primary.startSession({causalConsistency: false}) : primary; - const primaryDB = useSession ? primaryConn.getDatabase(dbName) : primaryConn.getDB(dbName); - - let command = testCase.command(dbName, collName); - - if (testInTransaction || testAsRetryableWrite) { - command.txnNumber = TenantMigrationConcurrentWriteUtil.kTxnNumber; - } - if (testInTransaction) { - command.startTransaction = true; - command.autocommit = false; - } - - return { - primaryConn, - primaryDB, - primaryHost: useSession ? primaryConn.getClient().host : primaryConn.host, - runAgainstAdminDb: testCase.runAgainstAdminDb, - command, - dbName, - collName, - useSession, - testInTransaction, - isBatchWrite: testCase.isBatchWrite, - isMultiUpdate: testCase.isMultiUpdate - }; -} - -function runTestForConcurrentWritesTest( - primary, testCase, testFunc, dbName, collName, {testInTransaction, testAsRetryableWrite} = {}) { - const testOpts = makeTestOptionsForConcurrentWritesTest( - primary, testCase, dbName, collName, testInTransaction, testAsRetryableWrite); - jsTest.log("Testing testOpts: " + tojson(testOpts) + " with testFunc " + testFunc.name); - - if (testCase.explicitlyCreateCollection) { - createCollectionAndInsertDocsForConcurrentWritesTest( - testOpts.primaryDB, collName, testCase.isCapped); - } - - if (testCase.setUp) { - testCase.setUp(testOpts.primaryDB, collName, testInTransaction); - } - - testFunc(testCase, testOpts); - - // This cleanup step is necessary for the shard merge protocol to work correctly. - cleanUpForConcurrentWritesTest(dbName, primary); -} - -const isNotWriteCommand = "not a write command"; -const isNotRunOnUserDatabase = "not run on user database"; -const isNotSupportedInServerless = "not supported in serverless cluster"; -const isAuthCommand = "is an auth command"; -const isOnlySupportedOnStandalone = "is only supported on standalone"; -const isOnlySupportedOnShardedCluster = "is only supported on sharded cluster"; -const isDeprecated = "is only deprecated"; - -TenantMigrationConcurrentWriteUtil.kTestDoc = { - x: -1 -}; -TenantMigrationConcurrentWriteUtil.kTestDoc2 = { - x: -2 -}; - -TenantMigrationConcurrentWriteUtil.kTestIndexKey = { - x: 1 -}; -TenantMigrationConcurrentWriteUtil.kExpireAfterSeconds = 1000000; -TenantMigrationConcurrentWriteUtil.kTestIndex = { - key: TenantMigrationConcurrentWriteUtil.kTestIndexKey, - name: "testIndex", - expireAfterSeconds: TenantMigrationConcurrentWriteUtil.kExpireAfterSeconds -}; - -function collectionExists(db, collName) { - const res = assert.commandWorked(db.runCommand({listCollections: 1, filter: {name: collName}})); - return res.cursor.firstBatch.length == 1; -} - -function insertTestDoc(primaryDB, collName) { - assert.commandWorked(primaryDB.runCommand( - {insert: collName, documents: [TenantMigrationConcurrentWriteUtil.kTestDoc]})); -} - -function insertTwoTestDocs(primaryDB, collName) { - assert.commandWorked(primaryDB.runCommand({ - insert: collName, - documents: [ - TenantMigrationConcurrentWriteUtil.kTestDoc, - TenantMigrationConcurrentWriteUtil.kTestDoc2 - ] - })); -} - -function createTestIndex(primaryDB, collName) { - assert.commandWorked(primaryDB.runCommand( - {createIndexes: collName, indexes: [TenantMigrationConcurrentWriteUtil.kTestIndex]})); -} - -function countDocs(db, collName, query) { - const res = assert.commandWorked(db.runCommand({count: collName, query: query})); - return res.n; -} - -function databaseExists(db, dbName) { - const res = assert.commandWorked(db.adminCommand({listDatabases: 1})); - return res.databases.some((dbDoc => dbDoc.name === dbName)); -} - -function indexExists(db, collName, targetIndex) { - const res = assert.commandWorked(db.runCommand({listIndexes: collName})); - return res.cursor.firstBatch.some( - (index) => bsonWoCompare(index.key, targetIndex.key) === 0 && - bsonWoCompare(index.expireAfterSeconds, targetIndex.expireAfterSeconds) === 0); -} - -TenantMigrationConcurrentWriteUtil.kMaxSize = 1024; // max size of capped collections. -TenantMigrationConcurrentWriteUtil.kNumInitialDocs = - 2; // num initial docs to insert into test collections. -TenantMigrationConcurrentWriteUtil.kTxnNumber = NumberLong(0); - -TenantMigrationConcurrentWriteUtil.testCases = { - _addShard: {skip: isNotRunOnUserDatabase}, - _cloneCollectionOptionsFromPrimaryShard: {skip: isNotRunOnUserDatabase}, - _configsvrAddShard: {skip: isNotRunOnUserDatabase}, - _configsvrAddShardToZone: {skip: isNotRunOnUserDatabase}, - _configsvrBalancerCollectionStatus: {skip: isNotRunOnUserDatabase}, - _configsvrBalancerStart: {skip: isNotRunOnUserDatabase}, - _configsvrBalancerStatus: {skip: isNotRunOnUserDatabase}, - _configsvrBalancerStop: {skip: isNotRunOnUserDatabase}, - _configsvrClearJumboFlag: {skip: isNotRunOnUserDatabase}, - _configsvrCommitChunksMerge: {skip: isNotRunOnUserDatabase}, - _configsvrCommitChunkMigration: {skip: isNotRunOnUserDatabase}, - _configsvrCommitChunkSplit: {skip: isNotRunOnUserDatabase}, - _configsvrCommitMovePrimary: - {skip: isNotRunOnUserDatabase}, // Can be removed once 6.0 is last LTS - _configsvrCreateDatabase: {skip: isNotRunOnUserDatabase}, - _configsvrEnsureChunkVersionIsGreaterThan: {skip: isNotRunOnUserDatabase}, - _configsvrMoveChunk: {skip: isNotRunOnUserDatabase}, // Can be removed once 6.0 is last LTS - _configsvrMovePrimary: {skip: isNotRunOnUserDatabase}, - _configsvrMoveRange: {skip: isNotRunOnUserDatabase}, - _configsvrRefineCollectionShardKey: {skip: isNotRunOnUserDatabase}, - _configsvrRemoveShard: {skip: isNotRunOnUserDatabase}, - _configsvrRemoveShardFromZone: {skip: isNotRunOnUserDatabase}, - _configsvrUpdateZoneKeyRange: {skip: isNotRunOnUserDatabase}, - _flushDatabaseCacheUpdates: {skip: isNotRunOnUserDatabase}, - _flushDatabaseCacheUpdatesWithWriteConcern: {skip: isNotRunOnUserDatabase}, - _flushReshardingStateChange: {skip: isNotRunOnUserDatabase}, - _flushRoutingTableCacheUpdates: {skip: isNotRunOnUserDatabase}, - _flushRoutingTableCacheUpdatesWithWriteConcern: {skip: isNotRunOnUserDatabase}, - _getNextSessionMods: {skip: isNotRunOnUserDatabase}, - _getUserCacheGeneration: {skip: isNotRunOnUserDatabase}, - _hashBSONElement: {skip: isNotRunOnUserDatabase}, - _isSelf: {skip: isNotRunOnUserDatabase}, - _killOperations: {skip: isNotRunOnUserDatabase}, - _mergeAuthzCollections: {skip: isNotRunOnUserDatabase}, - _migrateClone: {skip: isNotRunOnUserDatabase}, - _recvChunkAbort: {skip: isNotRunOnUserDatabase}, - _recvChunkCommit: {skip: isNotRunOnUserDatabase}, - _recvChunkReleaseCritSec: {skip: isNotRunOnUserDatabase}, - _recvChunkStart: {skip: isNotRunOnUserDatabase}, - _recvChunkStatus: {skip: isNotRunOnUserDatabase}, - _shardsvrCloneCatalogData: {skip: isNotRunOnUserDatabase}, - _shardsvrCompactStructuredEncryptionData: {skip: isOnlySupportedOnShardedCluster}, - _shardsvrCreateCollection: {skip: isOnlySupportedOnShardedCluster}, - _shardsvrCreateCollectionParticipant: {skip: isOnlySupportedOnShardedCluster}, - _shardsvrMovePrimary: {skip: isNotRunOnUserDatabase}, - _shardsvrSetAllowMigrations: {skip: isOnlySupportedOnShardedCluster}, - _shardsvrShardCollection: - {skip: isNotRunOnUserDatabase}, // TODO SERVER-58843: Remove once 6.0 becomes last LTS - _shardsvrRenameCollection: {skip: isOnlySupportedOnShardedCluster}, - _transferMods: {skip: isNotRunOnUserDatabase}, - abortTransaction: { - skip: isNotWriteCommand // aborting unprepared transaction doesn't create an abort oplog - // entry. - }, - aggregate: { - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return { - aggregate: collName, - pipeline: [{$out: collName + "Out"}], - cursor: {batchSize: 1} - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(collectionExists(db, collName + "Out")); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!collectionExists(db, collName + "Out")); - } - }, - appendOplogNote: {skip: isNotRunOnUserDatabase}, - applyOps: {skip: isNotSupportedInServerless}, - authenticate: {skip: isAuthCommand}, - availableQueryOptions: {skip: isNotWriteCommand}, - buildInfo: {skip: isNotWriteCommand}, - captrunc: { - skip: isNotWriteCommand, // TODO (SERVER-49834) - explicitlyCreateCollection: true, // creates a collection with kNumInitialDocs > 1 docs. - isCapped: true, - command: function(dbName, collName) { - return {captrunc: collName, n: 1}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, {}), 1); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, {}), kNumInitialDocs); - } - }, - checkShardingIndex: {skip: isNotRunOnUserDatabase}, - cleanupOrphaned: {skip: isNotRunOnUserDatabase}, - clearLog: {skip: isNotRunOnUserDatabase}, - cloneCollectionAsCapped: { - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return { - cloneCollectionAsCapped: collName, - toCollection: collName + "CloneCollectionAsCapped", - size: TenantMigrationConcurrentWriteUtil.kMaxSize - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(collectionExists(db, collName + "CloneCollectionAsCapped")); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!collectionExists(db, collName + "CloneCollectionAsCapped")); - } - }, - collMod: { - explicitlyCreateCollection: true, - setUp: createTestIndex, - command: function(dbName, collName) { - return { - collMod: collName, - index: { - keyPattern: TenantMigrationConcurrentWriteUtil.kTestIndexKey, - expireAfterSeconds: TenantMigrationConcurrentWriteUtil.kExpireAfterSeconds + 1 - } - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(indexExists(db, collName, { - key: TenantMigrationConcurrentWriteUtil.kTestIndexKey, - expireAfterSeconds: TenantMigrationConcurrentWriteUtil.kExpireAfterSeconds + 1 - })); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!indexExists(db, collName, { - key: TenantMigrationConcurrentWriteUtil.kTestIndexKey, - expireAfterSeconds: TenantMigrationConcurrentWriteUtil.kExpireAfterSeconds + 1 - })); - } - }, - collStats: {skip: isNotWriteCommand}, - commitTransaction: { - isTransactionCommand: true, - runAgainstAdminDb: true, - setUp: function(primaryDB, collName) { - assert.commandWorked(primaryDB.runCommand({ - insert: collName, - documents: [TenantMigrationConcurrentWriteUtil.kTestDoc], - txnNumber: NumberLong(TenantMigrationConcurrentWriteUtil.kTxnNumber), - startTransaction: true, - autocommit: false - })); - }, - command: function(dbName, collName) { - return { - commitTransaction: 1, - txnNumber: NumberLong(TenantMigrationConcurrentWriteUtil.kTxnNumber), - autocommit: false, - writeConcern: {w: "majority"} - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName), 1); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName), 0); - } - }, - compact: { - skip: isNotWriteCommand, // TODO (SERVER-49834) - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return {compact: collName, force: true}; - }, - assertCommandSucceeded: function(db, dbName, collName) {}, - assertCommandFailed: function(db, dbName, collName) {} - }, - configureFailPoint: {skip: isNotRunOnUserDatabase}, - connPoolStats: {skip: isNotRunOnUserDatabase}, - connPoolSync: {skip: isNotRunOnUserDatabase}, - connectionStatus: {skip: isNotRunOnUserDatabase}, - convertToCapped: { - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return {convertToCapped: collName, size: TenantMigrationConcurrentWriteUtil.kMaxSize}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(db[collName].stats().capped); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!db[collName].stats().capped); - } - }, - coordinateCommitTransaction: {skip: isNotRunOnUserDatabase}, - count: {skip: isNotWriteCommand}, - cpuload: {skip: isNotRunOnUserDatabase}, - create: { - testInTransaction: true, - command: function(dbName, collName) { - return {create: collName}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(collectionExists(db, collName)); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!collectionExists(db, collName)); - } - }, - createIndexes: { - testInTransaction: true, - explicitlyCreateCollection: true, - setUp: function(primaryDB, collName, testInTransaction) { - if (testInTransaction) { - // Drop the collection that was explicitly created above since inside transactions - // the index to create must either be on a non-existing collection, or on a new - // empty collection created earlier in the same transaction. - assert.commandWorked(primaryDB.runCommand({drop: collName})); - } - }, - command: function(dbName, collName) { - return { - createIndexes: collName, - indexes: [TenantMigrationConcurrentWriteUtil.kTestIndex] - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(indexExists(db, collName, TenantMigrationConcurrentWriteUtil.kTestIndex)); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!collectionExists(db, collName) || - !indexExists(db, collName, TenantMigrationConcurrentWriteUtil.kTestIndex)); - } - }, - createRole: {skip: isAuthCommand}, - createUser: {skip: isAuthCommand}, - currentOp: {skip: isNotRunOnUserDatabase}, - dataSize: {skip: isNotWriteCommand}, - dbCheck: {skip: isNotWriteCommand}, - dbHash: {skip: isNotWriteCommand}, - dbStats: {skip: isNotWriteCommand}, - delete: { - testInTransaction: true, - testAsRetryableWrite: true, - setUp: insertTestDoc, - command: function(dbName, collName) { - return { - delete: collName, - deletes: [{q: TenantMigrationConcurrentWriteUtil.kTestDoc, limit: 1}] - }; - }, - isBatchWrite: true, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 0); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 1); - } - }, - distinct: {skip: isNotWriteCommand}, - donorForgetMigration: {skip: isNotRunOnUserDatabase}, - donorStartMigration: {skip: isNotRunOnUserDatabase}, - donorWaitForMigrationToCommit: {skip: isNotRunOnUserDatabase}, - driverOIDTest: {skip: isNotRunOnUserDatabase}, - drop: { - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return {drop: collName}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(!collectionExists(db, collName)); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(collectionExists(db, collName)); - } - }, - dropAllRolesFromDatabase: {skip: isAuthCommand}, - dropAllUsersFromDatabase: {skip: isAuthCommand}, - dropConnections: {skip: isNotRunOnUserDatabase}, - dropDatabase: { - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return {dropDatabase: 1}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(!databaseExists(db, dbName)); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(databaseExists(db, dbName)); - } - }, - dropIndexes: { - explicitlyCreateCollection: true, - setUp: createTestIndex, - command: function(dbName, collName) { - return {dropIndexes: collName, index: "*"}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(!indexExists(db, collName, TenantMigrationConcurrentWriteUtil.kTestIndex)); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(indexExists(db, collName, TenantMigrationConcurrentWriteUtil.kTestIndex)); - } - }, - dropRole: {skip: isAuthCommand}, - dropUser: {skip: isAuthCommand}, - echo: {skip: isNotRunOnUserDatabase}, - emptycapped: { - explicitlyCreateCollection: true, - setUp: insertTestDoc, - command: function(dbName, collName) { - return {emptycapped: collName}; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 0); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 1); - } - }, - endSessions: {skip: isNotRunOnUserDatabase}, - explain: {skip: isNotRunOnUserDatabase}, - features: {skip: isNotRunOnUserDatabase}, - filemd5: {skip: isNotWriteCommand}, - find: {skip: isNotWriteCommand}, - findAndModify: { - testInTransaction: true, - testAsRetryableWrite: true, - setUp: insertTestDoc, - command: function(dbName, collName) { - return { - findAndModify: collName, - query: TenantMigrationConcurrentWriteUtil.kTestDoc, - remove: true - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 0); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 1); - } - }, - flushRouterConfig: {skip: isNotRunOnUserDatabase}, - fsync: {skip: isNotRunOnUserDatabase}, - fsyncUnlock: {skip: isNotRunOnUserDatabase}, - getCmdLineOpts: {skip: isNotRunOnUserDatabase}, - getDatabaseVersion: {skip: isNotRunOnUserDatabase}, - getDefaultRWConcern: {skip: isNotRunOnUserDatabase}, - getDiagnosticData: {skip: isNotRunOnUserDatabase}, - getLastError: {skip: isNotWriteCommand}, - getLog: {skip: isNotRunOnUserDatabase}, - getMore: {skip: isNotWriteCommand}, - getParameter: {skip: isNotRunOnUserDatabase}, - getShardMap: {skip: isNotRunOnUserDatabase}, - getShardVersion: {skip: isNotRunOnUserDatabase}, - getnonce: {skip: isNotRunOnUserDatabase}, - godinsert: {skip: isNotRunOnUserDatabase}, - grantPrivilegesToRole: {skip: isAuthCommand}, - grantRolesToRole: {skip: isAuthCommand}, - grantRolesToUser: {skip: isAuthCommand}, - hello: {skip: isNotRunOnUserDatabase}, - hostInfo: {skip: isNotRunOnUserDatabase}, - httpClientRequest: {skip: isNotRunOnUserDatabase}, - insert: { - testInTransaction: true, - testAsRetryableWrite: true, - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return {insert: collName, documents: [TenantMigrationConcurrentWriteUtil.kTestDoc]}; - }, - isBatchWrite: true, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 1); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, TenantMigrationConcurrentWriteUtil.kTestDoc), 0); - } - }, - internalRenameIfOptionsAndIndexesMatch: {skip: isNotRunOnUserDatabase}, - invalidateUserCache: {skip: isNotRunOnUserDatabase}, - killAllSessions: {skip: isNotRunOnUserDatabase}, - killAllSessionsByPattern: {skip: isNotRunOnUserDatabase}, - killCursors: {skip: isNotWriteCommand}, - killOp: {skip: isNotRunOnUserDatabase}, - killSessions: {skip: isNotRunOnUserDatabase}, - listCollections: {skip: isNotRunOnUserDatabase}, - listCommands: {skip: isNotRunOnUserDatabase}, - listDatabases: {skip: isNotRunOnUserDatabase}, - listIndexes: {skip: isNotWriteCommand}, - lockInfo: {skip: isNotRunOnUserDatabase}, - logRotate: {skip: isNotRunOnUserDatabase}, - logout: {skip: isNotRunOnUserDatabase}, - makeSnapshot: {skip: isNotRunOnUserDatabase}, - mapReduce: { - command: function(dbName, collName) { - return { - mapReduce: collName, - map: function mapFunc() { - emit(this.x, 1); - }, - reduce: function reduceFunc(key, values) { - return Array.sum(values); - }, - out: {replace: collName + "MrOut"}, - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(collectionExists(db, collName + "MrOut")); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(!collectionExists(db, collName + "MrOut")); - } - }, - mergeChunks: {skip: isNotRunOnUserDatabase}, - moveChunk: {skip: isNotRunOnUserDatabase}, - ping: {skip: isNotRunOnUserDatabase}, - planCacheClear: {skip: isNotWriteCommand}, - planCacheClearFilters: {skip: isNotWriteCommand}, - planCacheListFilters: {skip: isNotWriteCommand}, - planCacheSetFilter: {skip: isNotWriteCommand}, - prepareTransaction: {skip: isOnlySupportedOnShardedCluster}, - profile: {skip: isNotRunOnUserDatabase}, - reIndex: {skip: isOnlySupportedOnStandalone}, - reapLogicalSessionCacheNow: {skip: isNotRunOnUserDatabase}, - refreshLogicalSessionCacheNow: {skip: isNotRunOnUserDatabase}, - refreshSessions: {skip: isNotRunOnUserDatabase}, - recipientVoteImportedFiles: {skip: isNotRunOnUserDatabase}, - renameCollection: { - runAgainstAdminDb: true, - explicitlyCreateCollection: true, - command: function(dbName, collName) { - return { - renameCollection: dbName + "." + collName, - to: dbName + "." + collName + "Renamed" - }; - }, - assertCommandSucceeded: function(db, dbName, collName) { - assert(!collectionExists(db, collName)); - assert(collectionExists(db, collName + "Renamed")); - }, - assertCommandFailed: function(db, dbName, collName) { - assert(collectionExists(db, collName)); - assert(!collectionExists(db, collName + "Renamed")); - } - }, - replSetAbortPrimaryCatchUp: {skip: isNotRunOnUserDatabase}, - replSetFreeze: {skip: isNotRunOnUserDatabase}, - replSetGetConfig: {skip: isNotRunOnUserDatabase}, - replSetGetRBID: {skip: isNotRunOnUserDatabase}, - replSetGetStatus: {skip: isNotRunOnUserDatabase}, - replSetHeartbeat: {skip: isNotRunOnUserDatabase}, - replSetInitiate: {skip: isNotRunOnUserDatabase}, - replSetMaintenance: {skip: isNotRunOnUserDatabase}, - replSetReconfig: {skip: isNotRunOnUserDatabase}, - replSetRequestVotes: {skip: isNotRunOnUserDatabase}, - replSetResizeOplog: {skip: isNotRunOnUserDatabase}, - replSetStepDown: {skip: isNotRunOnUserDatabase}, - replSetStepUp: {skip: isNotRunOnUserDatabase}, - replSetSyncFrom: {skip: isNotRunOnUserDatabase}, - replSetTest: {skip: isNotRunOnUserDatabase}, - replSetTestEgress: {skip: isNotRunOnUserDatabase}, - replSetUpdatePosition: {skip: isNotRunOnUserDatabase}, - revokePrivilegesFromRole: {skip: isAuthCommand}, - revokeRolesFromRole: {skip: isAuthCommand}, - revokeRolesFromUser: {skip: isAuthCommand}, - rolesInfo: {skip: isNotWriteCommand}, - rotateCertificates: {skip: isAuthCommand}, - saslContinue: {skip: isAuthCommand}, - saslStart: {skip: isAuthCommand}, - sbe: {skip: isNotRunOnUserDatabase}, - serverStatus: {skip: isNotRunOnUserDatabase}, - setAllowMigrations: {skip: isNotRunOnUserDatabase}, - setCommittedSnapshot: {skip: isNotRunOnUserDatabase}, - setDefaultRWConcern: {skip: isNotRunOnUserDatabase}, - setFeatureCompatibilityVersion: {skip: isNotRunOnUserDatabase}, - setProfilingFilterGlobally: {skip: isNotRunOnUserDatabase}, - setIndexCommitQuorum: {skip: isNotRunOnUserDatabase}, - setParameter: {skip: isNotRunOnUserDatabase}, - setShardVersion: {skip: isNotRunOnUserDatabase}, - shardingState: {skip: isNotRunOnUserDatabase}, - shutdown: {skip: isNotRunOnUserDatabase}, - sleep: {skip: isNotRunOnUserDatabase}, - splitChunk: {skip: isNotRunOnUserDatabase}, - splitVector: {skip: isNotRunOnUserDatabase}, - stageDebug: {skip: isNotRunOnUserDatabase}, - startRecordingTraffic: {skip: isNotRunOnUserDatabase}, - startSession: {skip: isNotRunOnUserDatabase}, - stopRecordingTraffic: {skip: isNotRunOnUserDatabase}, - top: {skip: isNotRunOnUserDatabase}, - update: { - testInTransaction: true, - testAsRetryableWrite: true, - setUp: insertTestDoc, - command: function(dbName, collName) { - return { - update: collName, - updates: [{ - q: TenantMigrationConcurrentWriteUtil.kTestDoc, - u: {$set: {y: 0}}, - upsert: false, - multi: false - }] - }; - }, - isBatchWrite: true, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, - collName, - Object.assign({y: 0}, TenantMigrationConcurrentWriteUtil.kTestDoc)), - 1); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, - collName, - Object.assign({y: 0}, TenantMigrationConcurrentWriteUtil.kTestDoc)), - 0); - } - }, - multiUpdate: { - testInTransaction: true, - testAsRetryableWrite: false, - setUp: insertTwoTestDocs, - command: function(dbName, collName) { - return { - update: collName, - updates: [{q: {}, u: {$set: {y: 0}}, upsert: false, multi: true}] - }; - }, - isBatchWrite: true, - isMultiUpdate: true, - assertCommandSucceeded: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, Object.assign({y: 0})), 2); - }, - assertCommandFailed: function(db, dbName, collName) { - assert.eq(countDocs(db, collName, Object.assign({y: 0})), 0); - } - }, - updateRole: {skip: isAuthCommand}, - updateUser: {skip: isNotRunOnUserDatabase}, - usersInfo: {skip: isNotRunOnUserDatabase}, - validate: {skip: isNotWriteCommand}, - voteCommitIndexBuild: {skip: isNotRunOnUserDatabase}, - // TODO (SERVER-64296): Remove voteCommitMigrationProgress in 6.1. - voteCommitMigrationProgress: {skip: isNotRunOnUserDatabase}, - waitForFailPoint: {skip: isNotRunOnUserDatabase}, - waitForOngoingChunkSplits: {skip: isNotRunOnUserDatabase}, - whatsmysni: {skip: isNotRunOnUserDatabase}, - whatsmyuri: {skip: isNotRunOnUserDatabase} -}; - -function validateTestCase(testCase) { - assert(testCase.skip || testCase.command, - "must specify exactly one of 'skip' or 'command' for test case " + tojson(testCase)); - - if (testCase.skip) { - return; - } - - assert(testCase.command, "must specify 'command' for test case " + tojson(testCase)); - - // Check that all present fields are of the correct type. - assert(typeof (testCase.command) === "function"); - assert(typeof (testCase.assertCommandFailed) === "function"); - assert(testCase.setUp ? typeof (testCase.setUp) === "function" : true); - assert(testCase.runAgainstAdminDb ? typeof (testCase.runAgainstAdminDb) === "boolean" : true); - assert(testCase.explicitlyCreateCollection - ? typeof (testCase.explicitlyCreateCollection) === "boolean" - : true); - assert(testCase.testInTransaction ? typeof (testCase.testInTransaction) === "boolean" : true); - assert(testCase.testAsRetryableWrite ? typeof (testCase.testAsRetryableWrite) === "boolean" - : true); -} - -// Validate test cases for all commands. -for (let command of Object.keys(TenantMigrationConcurrentWriteUtil.testCases)) { - jsTestLog("calling the angels"); - validateTestCase(TenantMigrationConcurrentWriteUtil.testCases[command]); -} diff --git a/jstests/replsets/tenant_migration_donor_kill_op_retry.js b/jstests/replsets/tenant_migration_donor_kill_op_retry.js index dfea30c2665..1c9ef35fc98 100644 --- a/jstests/replsets/tenant_migration_donor_kill_op_retry.js +++ b/jstests/replsets/tenant_migration_donor_kill_op_retry.js @@ -43,7 +43,7 @@ function makeTenantId() { let fpNames = [ "pauseTenantMigrationBeforeInsertingDonorStateDoc", "pauseTenantMigrationDonorWhileUpdatingStateDoc", - "pauseTenantMigrationDonorBeforeStoringExternalClusterTimeKeyDocs" + "pauseTenantMigrationBeforeStoringExternalClusterTimeKeyDocs" ]; for (let fpName of fpNames) { jsTestLog("Setting failpoint \"" + fpName + diff --git a/jstests/replsets/tenant_migration_donor_resume_on_stepup_and_restart.js b/jstests/replsets/tenant_migration_donor_resume_on_stepup_and_restart.js new file mode 100644 index 00000000000..ffa0698b07d --- /dev/null +++ b/jstests/replsets/tenant_migration_donor_resume_on_stepup_and_restart.js @@ -0,0 +1,467 @@ +/** + * Tests that tenant migrations resume successfully on donor stepup and restart. + * + * Incompatible with shard merge, which can't handle restart. + * + * @tags: [ + * incompatible_with_eft, + * incompatible_with_macos, + * incompatible_with_shard_merge, + * incompatible_with_windows_tls, + * requires_majority_read_concern, + * requires_persistence, + * serverless, + * ] + */ + +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); +load("jstests/libs/parallelTester.js"); +load("jstests/libs/uuid_util.js"); +load("jstests/replsets/libs/tenant_migration_test.js"); +load("jstests/replsets/libs/tenant_migration_util.js"); + +const kMaxSleepTimeMS = 100; +const kTenantId = "testTenantId"; +const kMigrationFpNames = [ + "pauseTenantMigrationBeforeLeavingDataSyncState", + "pauseTenantMigrationBeforeLeavingBlockingState", + "abortTenantMigrationBeforeLeavingBlockingState", + "" +]; + +// Set the delay before a state doc is garbage collected to be short to speed up the test but long +// enough for the state doc to still be around after stepup or restart. +const kGarbageCollectionDelayMS = 30 * 1000; + +// Set the TTL monitor to run at a smaller interval to speed up the test. +const kTTLMonitorSleepSecs = 1; + +const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); + +/** + * Runs the donorStartMigration command to start a migration, and interrupts the migration on the + * donor using the 'interruptFunc', and asserts that migration eventually commits. + */ +function testDonorStartMigrationInterrupt(interruptFunc, + {donorRestarted = false, disableForShardMerge = true}) { + const donorRst = + new ReplSetTest({nodes: 3, name: "donorRst", nodeOptions: migrationX509Options.donor}); + + donorRst.startSet(); + donorRst.initiate(); + + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst}); + + let donorPrimary = tenantMigrationTest.getDonorPrimary(); + const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + + if (disableForShardMerge && + TenantMigrationUtil.isShardMergeEnabled(recipientPrimary.getDB("admin"))) { + jsTest.log("Skipping test for shard merge"); + tenantMigrationTest.stop(); + donorRst.stopSet(); + return; + } + + const migrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, + recipientConnString: tenantMigrationTest.getRecipientConnString(), + }; + const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); + + const runMigrationThread = new Thread(TenantMigrationUtil.runMigrationAsync, + migrationOpts, + donorRstArgs, + {retryOnRetryableErrors: true}); + runMigrationThread.start(); + + // Wait for donorStartMigration command to start. + assert.soon(() => donorPrimary.adminCommand({currentOp: true, desc: "tenant donor migration"}) + .inprog.length > 0); + + sleep(Math.random() * kMaxSleepTimeMS); + interruptFunc(donorRst); + + TenantMigrationTest.assertCommitted(runMigrationThread.returnData()); + tenantMigrationTest.waitForDonorNodesToReachState(donorRst.nodes, + migrationId, + migrationOpts.tenantId, + TenantMigrationTest.DonorState.kCommitted); + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + + donorPrimary = tenantMigrationTest.getDonorPrimary(); // Could change after interrupt. + const donorStats = tenantMigrationTest.getTenantMigrationStats(donorPrimary); + jsTestLog(`Stats at the donor primary: ${tojson(donorStats)}`); + if (donorRestarted) { + // If full restart happened the count could be lost completely. + assert.gte(1, donorStats.totalSuccessfulMigrationsDonated); + } else { + // The double counting happens when the failover happens after migration completes + // but before the state doc GC mark is persisted. While this test is targeting this + // scenario it is low probability in production. + assert(1 == donorStats.totalSuccessfulMigrationsDonated || + 2 == donorStats.totalSuccessfulMigrationsDonated); + } + // Skip checking the stats on the recipient since enableRecipientTesting is false + // so the recipient is forced to respond to recipientSyncData without starting the + // migration. + + tenantMigrationTest.stop(); + donorRst.stopSet(); +} + +/** + * Starts a migration and waits for it to commit, then runs the donorForgetMigration, and interrupts + * the donor using the 'interruptFunc', and asserts that the migration state is eventually garbage + * collected. + */ +function testDonorForgetMigrationInterrupt(interruptFunc) { + const donorRst = new ReplSetTest({ + nodes: 3, + name: "donorRst", + nodeOptions: Object.assign({}, migrationX509Options.donor, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: kTTLMonitorSleepSecs, + } + }) + }); + const recipientRst = new ReplSetTest({ + nodes: 1, + name: "recipientRst", + nodeOptions: Object.assign({}, migrationX509Options.recipient, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: kTTLMonitorSleepSecs, + } + }) + }); + + donorRst.startSet(); + donorRst.initiate(); + + recipientRst.startSet(); + recipientRst.initiate(); + + const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + + const migrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, + recipientConnString: recipientRst.getURL(), + }; + const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); + + TenantMigrationTest.assertCommitted( + tenantMigrationTest.runMigration(migrationOpts, {automaticForgetMigration: false})); + const forgetMigrationThread = new Thread(TenantMigrationUtil.forgetMigrationAsync, + migrationOpts.migrationIdString, + donorRstArgs, + true /* retryOnRetryableErrors */); + forgetMigrationThread.start(); + + // Wait for donorForgetMigration command to start. + assert.soon(() => { + const res = assert.commandWorked( + donorPrimary.adminCommand({currentOp: true, desc: "tenant donor migration"})); + return res.inprog[0].expireAt != null; + }); + sleep(Math.random() * kMaxSleepTimeMS); + interruptFunc(donorRst); + + assert.commandWorkedOrFailedWithCode( + tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString), + ErrorCodes.NoSuchTenantMigration); + + assert.commandWorked(forgetMigrationThread.returnData()); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, migrationOpts.tenantId); + + tenantMigrationTest.stop(); + donorRst.stopSet(); + recipientRst.stopSet(); +} + +/** + * Starts a migration and sets the passed in failpoint, then runs the donorAbortMigration, and + * interrupts the donor using the 'interruptFunc', and asserts that the migration state is + * eventually garbage collected. + */ +function testDonorAbortMigrationInterrupt(interruptFunc, fpName, isShutdown = false) { + const donorRst = new ReplSetTest({ + nodes: 3, + name: "donorRst", + nodeOptions: Object.assign({}, migrationX509Options.donor, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: kTTLMonitorSleepSecs, + } + }) + }); + const recipientRst = new ReplSetTest({ + nodes: 1, + name: "recipientRst", + nodeOptions: Object.assign({}, migrationX509Options.recipient, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: kTTLMonitorSleepSecs, + } + }) + }); + + donorRst.startSet(); + donorRst.initiate(); + + recipientRst.startSet(); + recipientRst.initiate(); + + const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + + const migrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, + recipientConnString: recipientRst.getURL(), + }; + const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); + let donorPrimary = tenantMigrationTest.getDonorPrimary(); + + // If we passed in a valid failpoint we set it, otherwise we let the migration run normally. + let fp; + if (fpName) { + fp = configureFailPoint(donorPrimary, fpName); + } + + assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); + + const tryAbortThread = new Thread(TenantMigrationUtil.tryAbortMigrationAsync, + {migrationIdString: migrationOpts.migrationIdString}, + donorRstArgs, + true /* retryOnRetryableErrors */); + tryAbortThread.start(); + + // Wait for donorAbortMigration command to start. + assert.soon(() => { + const res = assert.commandWorked( + donorPrimary.adminCommand({currentOp: true, desc: "tenant donor migration"})); + return res.inprog[0].receivedCancellation; + }); + + interruptFunc(donorRst); + + if (fp && !isShutdown) { + // Turn off failpoint in order to allow the migration to resume after stepup. + fp.off(); + } + + tryAbortThread.join(); + + let res = tryAbortThread.returnData(); + assert.commandWorkedOrFailedWithCode(res, ErrorCodes.TenantMigrationCommitted); + + donorPrimary = tenantMigrationTest.getDonorPrimary(); + let configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + let donorDoc = configDonorsColl.findOne({tenantId: kTenantId}); + + if (!res.ok) { + assert.eq(donorDoc.state, TenantMigrationTest.DonorState.kCommitted); + } else { + assert.eq(donorDoc.state, TenantMigrationTest.DonorState.kAborted); + } + + tenantMigrationTest.stop(); + donorRst.stopSet(); + recipientRst.stopSet(); +} + +/** + * Starts a migration and sets the passed in failpoint, then either waits for the failpoint or lets + * the migration run successfully and interrupts the donor using the 'interruptFunc'. After + * restarting, check the to see if the donorDoc data has persisted. + */ +function testStateDocPersistenceOnFailover(interruptFunc, fpName, isShutdown = false) { + const donorRst = + new ReplSetTest({nodes: 3, name: "donorRst", nodeOptions: migrationX509Options.donor}); + + donorRst.startSet(); + donorRst.initiate(); + + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst}); + + const migrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, + recipientConnString: tenantMigrationTest.getRecipientConnString(), + }; + let donorPrimary = tenantMigrationTest.getDonorPrimary(); + + // If we passed in a valid failpoint we set it, otherwise we let the migration run normally. + let fp; + if (fpName) { + fp = configureFailPoint(donorPrimary, fpName); + assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); + fp.wait(); + } else { + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); + } + + let configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + let donorDocBeforeFailover = configDonorsColl.findOne({tenantId: kTenantId}); + + interruptFunc(tenantMigrationTest.getDonorRst()); + + if (fp && !isShutdown) { + // Turn off failpoint in order to allow the migration to resume after stepup. + fp.off(); + } + + donorPrimary = tenantMigrationTest.getDonorPrimary(); + configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + let donorDocAfterFailover = configDonorsColl.findOne({tenantId: kTenantId}); + + // Check persisted fields in the donor doc. + assert.eq(donorDocBeforeFailover._id, donorDocAfterFailover._id); + assert.eq(donorDocBeforeFailover.recipientConnString, + donorDocAfterFailover.recipientConnString); + assert.eq(donorDocBeforeFailover.readPreference, donorDocAfterFailover.readPreference); + assert.eq(donorDocBeforeFailover.startMigrationDonorTimestamp, + donorDocAfterFailover.startMigrationDonorTimestamp); + assert.eq(donorDocBeforeFailover.migration, donorDocAfterFailover.migration); + assert.eq(donorDocBeforeFailover.tenantId, donorDocAfterFailover.tenantId); + assert.eq(donorDocBeforeFailover.donorCertificateForRecipient, + donorDocAfterFailover.donorCertificateForRecipient); + assert.eq(donorDocBeforeFailover.recipientCertificateForDonor, + donorDocAfterFailover.recipientCertificateForDonor); + assert.eq(donorDocBeforeFailover.migrationStart, donorDocAfterFailover.migrationStart); + + tenantMigrationTest.stop(); + donorRst.stopSet(); +} + +(() => { + jsTest.log("Test that the migration resumes on stepup"); + testDonorStartMigrationInterrupt((donorRst) => { + // Force the primary to step down but make it likely to step back up. + const donorPrimary = donorRst.getPrimary(); + assert.commandWorked( + donorPrimary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); + assert.commandWorked(donorPrimary.adminCommand({replSetFreeze: 0})); + }, {donorRestarted: false}); +})(); + +(() => { + jsTest.log("Test that the migration resumes after restart"); + testDonorStartMigrationInterrupt((donorRst) => { + // Skip validation on shutdown because the full validation can conflict with the tenant + // migration and cause it to fail. + donorRst.stopSet(null /* signal */, true /*forRestart */, {skipValidation: true}); + donorRst.startSet({restart: true}); + }, {donorRestarted: true, disableForShardMerge: true}); +})(); + +(() => { + jsTest.log("Test that the donorForgetMigration command can be retried on stepup"); + testDonorForgetMigrationInterrupt((donorRst) => { + // Force the primary to step down but make it likely to step back up. + const donorPrimary = donorRst.getPrimary(); + assert.commandWorked( + donorPrimary.adminCommand({replSetStepDown: ReplSetTest.kForeverSecs, force: true})); + assert.commandWorked(donorPrimary.adminCommand({replSetFreeze: 0})); + }); +})(); + +(() => { + jsTest.log("Test that the donorForgetMigration command can be retried after restart"); + testDonorForgetMigrationInterrupt((donorRst) => { + // Skip validation on shutdown because the full validation can conflict with the tenant + // migration and cause it to fail. + donorRst.stopSet(null /* signal */, true /*forRestart */, {skipValidation: true}); + donorRst.startSet({restart: true}); + }); +})(); + +(() => { + jsTest.log("Test that the donorAbortMigration command can be retried after restart"); + + kMigrationFpNames.forEach(fpName => { + if (!fpName) { + jsTest.log("Testing without setting a failpoint."); + } else { + jsTest.log("Testing with failpoint: " + fpName); + } + + testDonorAbortMigrationInterrupt((donorRst) => { + // Skip validation on shutdown because the full validation can conflict with the tenant + // migration and cause it to fail. + donorRst.stopSet(null /* signal */, true /*forRestart */, {skipValidation: true}); + donorRst.startSet({restart: true}); + }, fpName, true); + }); +})(); + +(() => { + jsTest.log("Test that the donorAbortMigration command can be retried on stepup"); + kMigrationFpNames.forEach(fpName => { + if (!fpName) { + jsTest.log("Testing without setting a failpoint."); + } else { + jsTest.log("Testing with failpoint: " + fpName); + } + + testDonorAbortMigrationInterrupt((donorRst) => { + // Force the primary to step down but make it likely to step back up. + const donorPrimary = donorRst.getPrimary(); + assert.commandWorked(donorPrimary.adminCommand( + {replSetStepDown: ReplSetTest.kForeverSecs, force: true})); + assert.commandWorked(donorPrimary.adminCommand({replSetFreeze: 0})); + }, fpName); + }); +})(); + +(() => { + jsTest.log("Test stateDoc data persistence on restart."); + kMigrationFpNames.forEach(fpName => { + if (!fpName) { + jsTest.log("Testing without setting a failpoint."); + } else { + jsTest.log("Testing with failpoint: " + fpName); + } + + testStateDocPersistenceOnFailover((donorRst) => { + // Skip validation on shutdown because the full validation can conflict with the tenant + // migration and cause it to fail. + donorRst.stopSet(null /* signal */, true /*forRestart */, {skipValidation: true}); + donorRst.startSet({restart: true}); + }, fpName, true); + }); +})(); + +(() => { + jsTest.log("Test stateDoc data persistence on stepup."); + kMigrationFpNames.forEach(fpName => { + if (!fpName) { + jsTest.log("Testing without setting a failpoint."); + } else { + jsTest.log("Testing with failpoint: " + fpName); + } + + testStateDocPersistenceOnFailover((donorRst) => { + // Force the primary to step down but make it likely to step back up. + const donorPrimary = donorRst.getPrimary(); + assert.commandWorked(donorPrimary.adminCommand( + {replSetStepDown: ReplSetTest.kForeverSecs, force: true})); + assert.commandWorked(donorPrimary.adminCommand({replSetFreeze: 0})); + }, fpName); + }); +})(); +})(); diff --git a/jstests/replsets/tenant_migration_donor_retry.js b/jstests/replsets/tenant_migration_donor_retry.js index 628ba16b927..281c7a22412 100644 --- a/jstests/replsets/tenant_migration_donor_retry.js +++ b/jstests/replsets/tenant_migration_donor_retry.js @@ -22,31 +22,30 @@ load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); +const kGarbageCollectionDelayMS = 5 * 1000; const kTenantIdPrefix = "testTenantId"; let testNum = 0; +const garbageCollectionOpts = { + // Set the delay before a donor state doc is garbage collected to be short to speed + // up the test. + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: 1 +}; + function setup() { const donorRst = new ReplSetTest({ name: "donorRst", nodes: 1, - nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, { - setParameter: { - // Set the delay before a donor state doc is garbage collected to be short to speed - // up the test. - tenantMigrationGarbageCollectionDelayMS: 0, - ttlMonitorSleepSecs: 1 - } - }) + nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, + {setParameter: garbageCollectionOpts}) }); donorRst.startSet(); donorRst.initiate(); - const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - donorRst: donorRst, - quickGarbageCollection: true, - }); + const tenantMigrationTest = new TenantMigrationTest( + {name: jsTestName(), donorRst: donorRst, quickGarbageCollection: true}); return { tenantMigrationTest, teardown: function() { @@ -95,6 +94,7 @@ function testDonorRetryRecipientSyncDataCmdOnError(tenantMigrationTest, errorCod TenantMigrationTest.assertCommitted( tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); return migrationId; } @@ -122,8 +122,7 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, e }, {times: 1}); - TenantMigrationTest.assertCommitted( - tenantMigrationTest.runMigration(migrationOpts, {automaticForgetMigration: false})); + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); // Verify that the initial recipientForgetMigration command failed. assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); @@ -146,7 +145,6 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, e tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); - assert.commandWorked(tenantMigrationTest.forgetMigration(extractUUIDFromObject(migrationId))); teardown(); })(); @@ -162,7 +160,6 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, e tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); - assert.commandWorked(tenantMigrationTest.forgetMigration(extractUUIDFromObject(migrationId))); teardown(); })(); @@ -193,7 +190,6 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, e tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); - assert.commandWorked(tenantMigrationTest.forgetMigration(extractUUIDFromObject(migrationId))); teardown(); })(); @@ -276,7 +272,7 @@ const kWriteErrorTimeMS = 50; jsTest.log("Test that the donor retries state doc update on retriable errors"); const {tenantMigrationTest, teardown} = setup(); - const tenantId = `${kTenantIdPrefix}RetryOnStateDocUpdateError`; + const tenantId = kTenantIdPrefix + "RetryOnStateDocUpdateError"; const migrationId = UUID(); const migrationOpts = { @@ -311,10 +307,17 @@ const kWriteErrorTimeMS = 50; fp.off(); migrationThread.join(); - // The state docs will only be completed and marked as garbage collectable if the - // update succeeds. - tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, tenantId); - + const donorStateDoc = tenantMigrationTest.getDonorPrimary() + .getCollection(TenantMigrationTest.kConfigDonorsNS) + .findOne({_id: migrationId}); + assert.eq(donorStateDoc.state, TenantMigrationTest.DonorState.kCommitted); + assert(donorStateDoc.expireAt); + + // Check that the recipient state doc is also correctly marked as garbage collectable. + const recipientStateDoc = tenantMigrationTest.getRecipientPrimary() + .getCollection(TenantMigrationTest.kConfigRecipientsNS) + .findOne({_id: migrationId}); + assert(recipientStateDoc.expireAt); teardown(); })(); })(); diff --git a/jstests/replsets/tenant_migration_donor_rollback_during_cloning.js b/jstests/replsets/tenant_migration_donor_rollback_during_cloning.js index d781c28330f..9385c37d4ca 100644 --- a/jstests/replsets/tenant_migration_donor_rollback_during_cloning.js +++ b/jstests/replsets/tenant_migration_donor_rollback_during_cloning.js @@ -145,7 +145,7 @@ function runTest(tenantId, // this situation. Allow replication once again. fpAfterListCall.wait(); const newDonorPrimary = otherNodes[0]; - donorRst.stepUp(newDonorPrimary, {awaitReplicationBeforeStepUp: false}); + newDonorPrimary.adminCommand({replSetStepUp: 1}); restartServerReplication(otherNodes); // Advance the cluster time by applying new operations on the new primary. We insert documents diff --git a/jstests/replsets/tenant_migration_donor_rollback_recovery.js b/jstests/replsets/tenant_migration_donor_rollback_recovery.js index 543b3f424c5..bccd2cd0c5f 100644 --- a/jstests/replsets/tenant_migration_donor_rollback_recovery.js +++ b/jstests/replsets/tenant_migration_donor_rollback_recovery.js @@ -237,20 +237,10 @@ function testRollBackMarkingStateGarbageCollectable() { true /* retryOnRetryableErrors */); forgetMigrationThread.start(); assert.soon(() => { - let docs = - donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).find().toArray(); - // There is a ttl index on `expireAt`. Thus we know the state doc is marked as garbage - // collectible either when: - // - // 1) It has an `expireAt`. - // 2) The document is deleted/the collection is empty. return 1 === donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).count({ _id: migrationId, expireAt: {$exists: 1} - }) || - donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).count({ - _id: migrationId - }) === 0; + }); }); }; diff --git a/jstests/replsets/tenant_migration_network_error_via_rollback.js b/jstests/replsets/tenant_migration_network_error_via_rollback.js index 1d2ad602941..cdb09f63577 100644 --- a/jstests/replsets/tenant_migration_network_error_via_rollback.js +++ b/jstests/replsets/tenant_migration_network_error_via_rollback.js @@ -124,7 +124,7 @@ function runTest({failPointName, failPointData = {}, batchSize = 10 * 1000}) { jsTestLog("Failing over to next primary"); assert.commandWorked( donorA.adminCommand({replSetStepDown: ReplSetTest.kDefaultTimeoutMS, force: true})); - donorRst.stepUp(nextPrimary, {awaitReplicationBeforeStepUp: false}); + assert.commandWorked(nextPrimary.adminCommand({replSetStepUp: ReplSetTest.kDefaultTimeoutMS})); assert.eq(nextPrimary, donorRst.getPrimary()); restartServerReplication(nextPrimary); restartServerReplication(donorD); diff --git a/jstests/replsets/tenant_migration_recipient_aborts_merge_on_donor_failure.js b/jstests/replsets/tenant_migration_recipient_aborts_merge_on_donor_failure.js index 77656281600..95d9710ef0f 100644 --- a/jstests/replsets/tenant_migration_recipient_aborts_merge_on_donor_failure.js +++ b/jstests/replsets/tenant_migration_recipient_aborts_merge_on_donor_failure.js @@ -68,9 +68,7 @@ load("jstests/replsets/libs/tenant_migration_util.js"); // step up a secondary so that the migration will complete and the // waitForMigrationToComplete call to the donor primary succeeds - assert.soonNoExcept(() => { - return assert.commandWorked(donorSecondary.adminCommand({replSetStepUp: 1})); - }); + assert.commandWorked(donorSecondary.adminCommand({replSetStepUp: 1})); hangBeforeTaskCompletion.off(); TenantMigrationTest.assertAborted( diff --git a/jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js b/jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js index 96430683846..6ab2a79746e 100644 --- a/jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js +++ b/jstests/replsets/tenant_migration_recipient_access_blocker_rollback.js @@ -89,14 +89,14 @@ function runRollbackAfterMigrationCommitted(tenantId) { // Stepping up one of the secondaries should cause the original primary to rollback. jsTestLog("Stepping up one of the secondaries."); const newRecipientPrimary = secondaries[0]; - recipientRst.stepUp(newRecipientPrimary, {awaitReplicationBeforeStepUp: false}); + assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1})); jsTestLog("Restarting server replication."); restartServerReplication(secondaries); recipientRst.awaitReplication(); jsTestLog("Stepping up the original primary back to primary."); - recipientRst.stepUp(originalPrimary, {awaitReplicationBeforeStepUp: false}); + assert.commandWorked(originalPrimary.adminCommand({replSetStepUp: 1})); jsTestLog("Perform a read against the original primary on the tenant collection."); assert.eq(numDocs, originalPrimary.getDB(dbName)[collName].find().itcount()); @@ -188,7 +188,7 @@ function runRollbackAfterLoneRecipientForgetMigrationCommand(tenantId) { // Stepping up one of the secondaries should cause the original primary to rollback. jsTestLog("Stepping up one of the secondaries."); - recipientRst.stepUp(newPrimary, {awaitReplicationBeforeStepUp: false}); + assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1})); assert.commandFailedWithCode(recipientForgetMigrationThread.returnData(), ErrorCodes.InterruptedDueToReplStateChange); @@ -203,13 +203,13 @@ function runRollbackAfterLoneRecipientForgetMigrationCommand(tenantId) { jsTestLog("Stepping up the original primary back to primary."); const fpOriginalPrimaryBeforeStarting = configureFailPoint(originalPrimary, "pauseBeforeRunTenantMigrationRecipientInstance"); - fpOriginalPrimary.off(); - recipientRst.stepUp(originalPrimary, {awaitReplicationBeforeStepUp: false}); + assert.commandWorked(originalPrimary.adminCommand({replSetStepUp: 1})); jsTestLog("Perform another read against the original primary on the tenant collection."); assert.eq(1, originalPrimary.getDB(dbName)[collName].find().itcount()); fpOriginalPrimaryBeforeStarting.off(); + fpOriginalPrimary.off(); fpNewPrimary.off(); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_recipient_current_op.js b/jstests/replsets/tenant_migration_recipient_current_op.js index 5dabe5dad5e..c5d88734a00 100644 --- a/jstests/replsets/tenant_migration_recipient_current_op.js +++ b/jstests/replsets/tenant_migration_recipient_current_op.js @@ -24,12 +24,7 @@ load("jstests/libs/parallelTester.js"); // For the Thread(). load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - // This test relies on a large awaitData timeout keeping a window open such that failpoints - // configured for hanging are hit. - optimizeMigrations: false, -}); +const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); const kMigrationId = UUID(); const kTenantId = 'testTenantId'; diff --git a/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js b/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js index b970f20cce1..62c899fa87c 100644 --- a/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js +++ b/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js @@ -102,7 +102,8 @@ assert.soon(() => recipientColl.find().itcount() === batchSize); verifySyncSource(recipientPrimary, migrationId, donorPrimary.host); // Steps down the current donor's primary and wait for the new primary to be discovered. -donorRst.stepUp(donorRst.getSecondary()); +donorRst.awaitLastOpCommitted(); +assert.commandWorked(donorRst.getSecondary().adminCommand({replSetStepUp: 1})); const newDonorPrimary = donorRst.getPrimary(); assert.neq(newDonorPrimary.host, donorPrimary.host); diff --git a/jstests/replsets/tenant_migration_recipient_failover_before_creating_oplog_buffer.js b/jstests/replsets/tenant_migration_recipient_failover_before_creating_oplog_buffer.js index be99a6ffacb..c4ac48d46c8 100644 --- a/jstests/replsets/tenant_migration_recipient_failover_before_creating_oplog_buffer.js +++ b/jstests/replsets/tenant_migration_recipient_failover_before_creating_oplog_buffer.js @@ -48,8 +48,8 @@ jsTestLog("Waiting until the recipient primary is about to create an oplog buffe fpBeforeCreatingOplogBuffer.wait(); jsTestLog("Stepping a new primary up."); -tenantMigrationTest.getRecipientRst().stepUp( - tenantMigrationTest.getRecipientRst().getSecondaries()[0]); +assert.commandWorked(tenantMigrationTest.getRecipientRst().getSecondaries()[0].adminCommand( + {replSetStepUp: ReplSetTest.kForeverSecs, force: true})); fpBeforeCreatingOplogBuffer.off(); diff --git a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js index 61535095919..e532419c76f 100644 --- a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js +++ b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js @@ -90,7 +90,7 @@ function restartNodeAndCheckStateWithoutOplogApplication( jsTestLog("Stepping up the new node."); // Now step up the new node - tenantMigrationTest.getRecipientRst().stepUp(initialSyncNode); + assert.commandWorked(initialSyncNode.adminCommand({"replSetStepUp": 1})); fpOnRecipient.off(); } @@ -119,7 +119,7 @@ function restartNodeAndCheckStateDuringOplogApplication( jsTestLog("Stepping up the new node."); // Now step up the new node - tenantMigrationTest.getRecipientRst().stepUp(initialSyncNode); + assert.commandWorked(initialSyncNode.adminCommand({"replSetStepUp": 1})); fpPauseOplogApplierOnBatch.off(); fpOnRecipient.off(); } diff --git a/jstests/replsets/tenant_migration_recipient_resumes_on_donor_failover.js b/jstests/replsets/tenant_migration_recipient_resumes_on_donor_failover.js index c917fb6cc29..d47dd189d2c 100644 --- a/jstests/replsets/tenant_migration_recipient_resumes_on_donor_failover.js +++ b/jstests/replsets/tenant_migration_recipient_resumes_on_donor_failover.js @@ -112,9 +112,7 @@ function runTest(failPoint) { 'fpAfterStartingOplogFetcherMigrationRecipientInstance', {action: "hang"}); // Step up a new donor primary. - assert.soonNoExcept(() => { - return assert.commandWorked(donorSecondary.adminCommand({replSetStepUp: 1})); - }); + assert.commandWorked(donorSecondary.adminCommand({replSetStepUp: 1})); hangOnRetry.wait(); res = recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"}); currOp = res.inprog[0]; diff --git a/jstests/replsets/tenant_migration_recipient_retry_forget_migration.js b/jstests/replsets/tenant_migration_recipient_retry_forget_migration.js index 3023a2fb61f..da694d8dc3d 100644 --- a/jstests/replsets/tenant_migration_recipient_retry_forget_migration.js +++ b/jstests/replsets/tenant_migration_recipient_retry_forget_migration.js @@ -85,7 +85,7 @@ const newRecipientPrimary = tenantMigrationTest.getRecipientRst().getSecondary() const newPrimaryFp = configureFailPoint(newRecipientPrimary, "hangBeforeTaskCompletion"); // Step up a new recipient primary before the state doc is truly marked as garbage collectable. -tenantMigrationTest.getRecipientRst().stepUp(newRecipientPrimary); +assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1})); fp.off(); // The new primary should skip all tenant migration steps but wait for another diff --git a/jstests/replsets/tenant_migration_recipient_retryable_writes_failover.js b/jstests/replsets/tenant_migration_recipient_retryable_writes_failover.js index 68ceae0b709..4ecdac15965 100644 --- a/jstests/replsets/tenant_migration_recipient_retryable_writes_failover.js +++ b/jstests/replsets/tenant_migration_recipient_retryable_writes_failover.js @@ -94,7 +94,9 @@ const recipientSecondary = recipientRst.getSecondary(); const fpAfterFetchingRetryableWritesEntries = configureFailPoint( recipientSecondary, "fpAfterFetchingRetryableWritesEntriesBeforeStartOpTime", {action: "hang"}); -recipientRst.stepUp(recipientSecondary); +recipientRst.awaitLastOpCommitted(); +assert.commandWorked( + recipientSecondary.adminCommand({replSetStepUp: ReplSetTest.kForeverSecs, force: true})); fpPauseAfterRetrievingRetryableWritesBatch.off(); const newRecipientPrimary = recipientRst.getPrimary(); diff --git a/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js b/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js index b25df0e9d9b..7e2ca21b321 100644 --- a/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js +++ b/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js @@ -55,8 +55,8 @@ forgetMigrationThread.start(); fpBeforeDroppingOplogBufferCollection.wait(); jsTestLog("Step up a new recipient primary."); -tenantMigrationTest.getRecipientRst().stepUp( - tenantMigrationTest.getRecipientRst().getSecondaries()[0]); +assert.commandWorked(tenantMigrationTest.getRecipientRst().getSecondaries()[0].adminCommand( + {replSetStepUp: ReplSetTest.kForeverSecs, force: true})); fpBeforeDroppingOplogBufferCollection.off(); diff --git a/jstests/replsets/tenant_migration_recipient_ttl.js b/jstests/replsets/tenant_migration_recipient_ttl.js index 304ddd1fdab..f0f2de2281f 100644 --- a/jstests/replsets/tenant_migration_recipient_ttl.js +++ b/jstests/replsets/tenant_migration_recipient_ttl.js @@ -18,18 +18,8 @@ load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const kGarbageCollectionParams = { - // Set the delay to 20s so that we can see the `expireAt` set prior to the document vanishing. - tenantMigrationGarbageCollectionDelayMS: 20 * 1000, - - // Set the TTL monitor to run at a smaller interval to speed up the test. - ttlMonitorSleepSecs: 1 -}; - -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: {setParameter: kGarbageCollectionParams}, -}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); const kRecipientTTLIndexName = "TenantMigrationRecipientTTLIndex"; @@ -81,7 +71,8 @@ assert(stateDocQuery[0].hasOwnProperty("expireAt"), tojson(stateDocQuery)); // Sleep past the garbage collection delay time, and then make sure the state document for our // migration does not exist. -jsTestLog("Waiting for the state document to have been deleted."); +jsTestLog("Sleeping and then expecting the state document to have been deleted."); +sleep(30000); // The garbage collection delay is 30s. tenantMigrationTest.waitForMigrationGarbageCollection(kMigrationId, kTenantId); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover.js b/jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover.js index 6a25e0e0c6b..26fb31e41e8 100644 --- a/jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover.js +++ b/jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover.js @@ -90,7 +90,8 @@ const tenantMigrationFailoverTest = function(isTimeSeries, createCollFn, docs) { // Step up a new node in the recipient set and trigger a failover. The new primary should resume // cloning starting from the third document. const newRecipientPrimary = recipientRst.getSecondaries()[0]; - recipientRst.stepUp(newRecipientPrimary); + recipientRst.awaitLastOpCommitted(); + assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1})); hangDuringCollectionClone.off(); recipientRst.getPrimary(); diff --git a/jstests/replsets/tenant_migration_resume_collection_cloner_after_rename.js b/jstests/replsets/tenant_migration_resume_collection_cloner_after_rename.js index 45284f9f358..2fd5aeec36c 100644 --- a/jstests/replsets/tenant_migration_resume_collection_cloner_after_rename.js +++ b/jstests/replsets/tenant_migration_resume_collection_cloner_after_rename.js @@ -95,7 +95,7 @@ const fpPauseAtStartOfMigration = // Step up a new node in the recipient set and trigger a failover. The new primary should resume // cloning starting from the third document. -recipientRst.stepUp(newRecipientPrimary); +assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1})); hangDuringCollectionClone.off(); recipientRst.getPrimary(); diff --git a/jstests/replsets/tenant_migration_resume_oplog_application.js b/jstests/replsets/tenant_migration_resume_oplog_application.js index 532908d960b..530c9314574 100644 --- a/jstests/replsets/tenant_migration_resume_oplog_application.js +++ b/jstests/replsets/tenant_migration_resume_oplog_application.js @@ -94,7 +94,7 @@ if (appliedNoOps.count() === 2) { // Step up a new node in the recipient set and trigger a failover. The new primary should resume // fetching starting from the unapplied documents. const newRecipientPrimary = recipientRst.getSecondaries()[0]; -recipientRst.stepUp(newRecipientPrimary); +assert.commandWorked(newRecipientPrimary.adminCommand({replSetStepUp: 1})); waitAfterDatabaseClone.off(); waitInOplogApplier.off(); recipientRst.getPrimary(); diff --git a/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js b/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js index ad823e098a4..7b7292f85a3 100644 --- a/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js +++ b/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js @@ -264,7 +264,8 @@ testRecipientRetryableWrites(recipientDb, beforeWrites); testRecipientRetryableWrites(recipientDb, duringWrites); jsTestLog("Step up secondary"); const recipientRst = tenantMigrationTest.getRecipientRst(); -recipientRst.stepUp(recipientRst.getSecondary()); +recipientRst.awaitReplication(); +assert.commandWorked(recipientRst.getSecondary().adminCommand({replSetStepUp: 1})); jsTestLog("Run retryable write on secondary after the migration"); testRecipientRetryableWrites(recipientRst.getPrimary().getDB(kDbName), beforeWrites); testRecipientRetryableWrites(recipientRst.getPrimary().getDB(kDbName), duringWrites); diff --git a/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js b/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js index 0e7a4a3fcf8..5fc0eb8b9ee 100644 --- a/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js +++ b/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js @@ -127,7 +127,8 @@ function testRetryOnRecipient(ordered) { jsTestLog("Step up secondary"); const recipientRst = tenantMigrationTest.getRecipientRst(); - recipientRst.stepUp(recipientRst.getSecondary()); + recipientRst.awaitReplication(); + assert.commandWorked(recipientRst.getSecondary().adminCommand({replSetStepUp: 1})); jsTestLog("Run retryable write on secondary after the migration"); testRecipientRetryableWrites(recipientRst.getPrimary().getDB(kDbName), beforeWrites); testRecipientRetryableWrites(recipientRst.getPrimary().getDB(kDbName), duringWrites); diff --git a/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js b/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js deleted file mode 100644 index c1031db50d2..00000000000 --- a/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Tests initial sync with a time-series collection that contains mixed-schema buckets. - * - * @tags: [ - * requires_fcv_60, - * ] - */ -(function() { -"use strict"; - -load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. - -TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; - -const replTest = new ReplSetTest({nodes: 1}); -replTest.startSet(); -replTest.initiate(); - -const primary = replTest.getPrimary(); -const db = primary.getDB(jsTestName()); -const coll = db.coll; -const bucketsColl = db["system.buckets." + coll.getName()]; - -const bucket = { - _id: ObjectId("65a6eb806ffc9fa4280ecac4"), - control: { - version: NumberInt(1), - min: { - _id: ObjectId("65a6eba7e6d2e848e08c3750"), - t: ISODate("2024-01-16T20:48:00Z"), - a: 1, - }, - max: { - _id: ObjectId("65a6eba7e6d2e848e08c3751"), - t: ISODate("2024-01-16T20:48:39.448Z"), - a: "a", - }, - }, - meta: 0, - data: { - _id: { - 0: ObjectId("65a6eba7e6d2e848e08c3750"), - 1: ObjectId("65a6eba7e6d2e848e08c3751"), - }, - t: { - 0: ISODate("2024-01-16T20:48:39.448Z"), - 1: ISODate("2024-01-16T20:48:39.448Z"), - }, - a: { - 0: "a", - 1: 1, - }, - } -}; - -assert.commandWorked( - db.createCollection(coll.getName(), {timeseries: {timeField: "t", metaField: "m"}})); -assert.commandWorked( - db.runCommand({collMod: coll.getName(), timeseriesBucketsMayHaveMixedSchemaData: true})); -assert.commandWorked(bucketsColl.insert(bucket)); - -const secondary = replTest.add(); -replTest.reInitiate(); -replTest.waitForState(secondary, ReplSetTest.State.SECONDARY); -replTest.awaitReplication(); - -const primaryColl = primary.getDB(db.getName())[bucketsColl.getName()]; -const secondaryColl = secondary.getDB(db.getName())[bucketsColl.getName()]; - -assert(TimeseriesTest.bucketsMayHaveMixedSchemaData(primaryColl)); -assert(TimeseriesTest.bucketsMayHaveMixedSchemaData(secondaryColl)); - -replTest.stopSet(); -})(); diff --git a/jstests/replsets/wtimeout_too_large.js b/jstests/replsets/wtimeout_too_large.js deleted file mode 100644 index 3beef1ed578..00000000000 --- a/jstests/replsets/wtimeout_too_large.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Test that the server rejects extremely large values for write concern wtimeout. - */ - -(function() { -"use strict"; - -const rst = new ReplSetTest({name: jsTestName(), nodes: 2}); -rst.startSet(); -rst.initiateWithHighElectionTimeout(); - -const dbName = "testdb"; -const collName = "testcoll"; - -const primary = rst.getPrimary(); -const primaryDB = primary.getDB(dbName); -const primaryColl = primaryDB.getCollection(collName); - -jsTestLog("Issuing a write within accepted wtimeout bounds"); -assert.commandWorked( - primaryColl.insert({a: 1}, {writeConcern: {w: 2, wtimeout: ReplSetTest.kDefaultTimeoutMS}})); - -jsTestLog("Issuing a high wtimeout write and confirming that it gets rejected"); - -// Outside int32 bounds. -const oneTrillionMS = 1000 * 1000 * 1000 * 1000; -assert.commandFailedWithCode( - primaryColl.insert({b: 2}, {writeConcern: {w: 2, wtimeout: oneTrillionMS}}), - ErrorCodes.FailedToParse); - -rst.stopSet(); -})(); |