diff options
Diffstat (limited to 'jstests/replsets')
17 files changed, 887 insertions, 210 deletions
diff --git a/jstests/replsets/backwards_compatible_timeseries_catalog_options.js b/jstests/replsets/backwards_compatible_timeseries_catalog_options.js new file mode 100644 index 00000000000..af65d49ae8a --- /dev/null +++ b/jstests/replsets/backwards_compatible_timeseries_catalog_options.js @@ -0,0 +1,72 @@ +/* + * Test that timeseriesBucketsMayHaveMixedSchemaData and timeseriesBucketingParametersHaveChanged + * collection options are correctly applied by secondaries and cloned on initial sync according to + * the format introduced under SERVER-91195 (relying on storageEngine.wiredTiger.configString). + * + * @tags: [multiversion_incompatible] + */ +load("jstests/libs/feature_flag_util.js"); // For "FeatureFlagUtil" + +const dbName = "testdb"; +const collName = "testcoll"; +const bucketCollName = 'system.buckets.' + collName; +const bucketNs = dbName + '.' + bucketCollName; + +const rst = new ReplSetTest({name: 'rs', nodes: 2}); +rst.startSet(); +rst.initiateWithHighElectionTimeout(); + +// Create collection on the primary node +const primary = rst.getPrimary(); +const primaryDb = primary.getDB(dbName); +const primaryColl = primaryDb.getCollection(collName); + +assert.commandWorked(primaryDb.createCollection(collName, {timeseries: {timeField: "t"}})); + +// Execute collMod to change the timeseries catalog options under testing +assert.commandWorked(primaryDb.runCommand({ + collMod: collName, + timeseriesBucketsMayHaveMixedSchemaData: true, +})); + +// Double check that the option has been correctly applied on the primary node +const expectedAppMetadata = "app_metadata=(timeseriesBucketsMayHaveMixedSchemaData=true)"; + +const configStringAfterCollMod = + primaryDb.runCommand({listCollections: 1, filter: {name: bucketCollName}}) + .cursor.firstBatch[0] + .options.storageEngine.wiredTiger.configString; + +assert.eq(configStringAfterCollMod, expectedAppMetadata); + +// Add a new node and wait for it to complete initial sync +rst.add({rsConfig: {priority: 0}}); + +rst.reInitiate(); +rst.awaitSecondaryNodes(); +rst.awaitReplication(); + +function assertSameOutputFromDifferentNodes(func) { + let outputs = []; + rst.nodes.forEach(function(node) { + outputs.push(func(node)); + }); + assert.eq(outputs[0], outputs[1]); + assert.eq(outputs[1], outputs[2]); +} + +// Assert that collection options for the view and for the buckets namespace +// are the same on primary, secondary and initial-synced secondary. +assertSameOutputFromDifferentNodes(node => { + return node.getDB(dbName) + .runCommand({listCollections: 1, filter: {name: collName}}) + .cursor.firstBatch[0]; +}); + +assertSameOutputFromDifferentNodes(node => { + return node.getDB(dbName) + .runCommand({listCollections: 1, filter: {name: bucketCollName}}) + .cursor.firstBatch[0]; +}); + +rst.stopSet(); diff --git a/jstests/replsets/catchup_ignores_old_heartbeats.js b/jstests/replsets/catchup_ignores_old_heartbeats.js new file mode 100644 index 00000000000..9f72adbc86a --- /dev/null +++ b/jstests/replsets/catchup_ignores_old_heartbeats.js @@ -0,0 +1,70 @@ +// Makes sure an old heartbeat that is being processed when primary catchup starts does not cause +// primary catchup to think we're already caught up. + +(function() { + +load('jstests/libs/fail_point_util.js'); +load('jstests/replsets/rslib.js'); + +var name = TestData.testName; +var rst = new ReplSetTest({ + name: name, + nodes: 3, + // We're not testing catchup takeover in this test, and in the case where primary catchup fails, + // catchup takeover may cause an additional election and muddle the results. Setting + // catchUpTakeoverDelayMillis to -1 disables catchup takeover. + settings: {chainingAllowed: true, catchUpTakeoverDelayMillis: -1}, + nodeOptions: { + "setParameter": { + "logComponentVerbosity": tojsononeline({"replication": {"verbosity": 2}}), + }, + }, + useBridge: true, + waitForKeys: true +}); + +rst.startSet(); +rst.initiate(); +rst.awaitSecondaryNodes(); + +var primary = rst.getPrimary(); +var primaryColl = primary.getDB("test").coll; + +// The default WC is majority and this test can't test catchup properly if it used majority writes. +assert.commandWorked(primary.adminCommand( + {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}})); + +assert(primary.host == rst.nodes[0].host); +// Make us chain node 1 (the node which will become the new primary) from node 2. Don't allow +// node 1 to switch back. +const forceSyncSource = configureFailPoint( + rst.nodes[1], "forceSyncSourceCandidate", {"hostAndPort": rst.nodes[2].host}); +syncFrom(rst.nodes[2], rst.nodes[0], rst); +syncFrom(rst.nodes[1], rst.nodes[2], rst); +const RBIDBeforeStepUp = assert.commandWorked(primary.adminCommand({replSetGetRBID: 1})); + +// Disconnect the primary from the node syncing from it. +primary.disconnect(rst.nodes[2]); +// Get a heartbeat from the original primary "stuck" in the new primary. +const newPrimary = rst.nodes[1]; +let hbfp = + configureFailPoint(newPrimary, "pauseInHandleHeartbeatResponse", {"target": primary.host}); +hbfp.wait(); +// Put the original primary ahead of the secondaries. +assert.commandWorked(primaryColl.insert({_id: 1})); +jsTestLog("Stepping up new primary"); +assert.commandWorked(newPrimary.adminCommand({replSetStepUp: 1})); +// Allow the "stuck" heartbeat to proceed. +hbfp.off(); +// The step-up command waits for the election to complete, but not catch-up. Reconnect the old +// primary to the new primary's sync source to allow replication. +primary.reconnect(rst.nodes[2]); +rst.awaitReplication(); +// The new primary should still be primary. +assert.eq(newPrimary.host, rst.getPrimary().host); +// No rollbacks should have happened. +const RBIDAfterStepUp = assert.commandWorked(primary.adminCommand({replSetGetRBID: 1})); +assert.eq(RBIDBeforeStepUp.rbid, RBIDAfterStepUp.rbid); +forceSyncSource.off(); +rst.stopSet(); +})(); diff --git a/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js b/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js index 014dd8fe568..b0e1f887b60 100644 --- a/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js +++ b/jstests/replsets/change_stream_pit_pre_image_deletion_asymmetric.js @@ -47,7 +47,8 @@ assert.commandWorked(coll.insert({_id: 1, v: 1}, {writeConcern: {w: 2}})); // data situation during oplog application. const initialSyncNode = replTest.add({ rsConfig: {priority: 0}, - setParameter: {'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'})} + setParameter: {'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'})}, + oplogSize: oplogSizeMB }); // Wait until the new node starts and pauses on the fail point. @@ -90,6 +91,10 @@ function oplogIsRolledOver() { } while (!oplogIsRolledOver()) { + // Reducing the number of documents inserted to prevent using too much memory. + // The oplog rollover depends on timestamp, sleeping between inserts can significantly reduce + // the total number of documents. + sleep(20); // Insert a large document with a write concern that ensures that before proceeding the // operation gets replicated to all 3 nodes in the replica set, since, otherwise, the node that // is being initial synced may not be able to catchup due to a small size of the oplog. diff --git a/jstests/replsets/dbcheck_fails_on_write_concern_error.js b/jstests/replsets/dbcheck_fails_on_write_concern_error.js new file mode 100644 index 00000000000..4e76e9ef1c6 --- /dev/null +++ b/jstests/replsets/dbcheck_fails_on_write_concern_error.js @@ -0,0 +1,103 @@ +/** + * Tests dbCheck fails on write concern error. + * + */ +load("jstests/libs/fail_point_util.js"); +load("jstests/libs/write_concern_util.js"); +load("jstests/replsets/libs/dbcheck_utils.js"); + +const dbName = jsTestName(); +const collName = jsTestName(); + +// There should be multiple batches in the dbcheck run, but dbcheck should stop after the first +// batch fails when waiting for write concern. +const nDocs = 100; +const batchSize = 10; + +function runTest(validateMode, writeConcern) { + const rst = new ReplSetTest({ + name: jsTestName(), + nodes: 2, + nodeOptions: { + setParameter: + {logComponentVerbosity: tojson({command: 3}), dbCheckHealthLogEveryNBatches: 1}, + } + }); + rst.startSet(); + rst.initiateWithHighElectionTimeout(); + const primary = rst.getPrimary(); + const secondary = rst.getSecondary(); + const primaryHealthLog = primary.getDB("local").system.healthlog; + const secondaryHealthLog = secondary.getDB("local").system.healthlog; + const primaryDB = primary.getDB(dbName); + const secondaryDB = secondary.getDB(dbName); + + resetAndInsert(rst, primaryDB, collName, nDocs); + assert.commandWorked(primaryDB.runCommand({ + createIndexes: collName, + indexes: [{key: {a: 1}, name: 'a_1'}], + })); + rst.awaitReplication(); + + assert.eq(primaryDB.getCollection(collName).find({}).count(), nDocs); + assert.eq(secondaryDB.getCollection(collName).find({}).count(), nDocs); + clearHealthLog(rst); + + const hangBeforeProcessingDbCheckRunFp = + configureFailPoint(primary, "hangBeforeProcessingDbCheckRun"); + + const hangBeforeAddingDBCheckBatchToOplogFp = + configureFailPoint(primary, "hangBeforeAddingDBCheckBatchToOplog"); + + if (validateMode == "dataConsistencyAndMissingIndexKeysCheck") { + jsTestLog("Running dbCheck dataConsistencyAndMissingIndexKeysCheck"); + runDbCheck(rst, primary.getDB(dbName), collName, { + // validateMode: "dataConsistencyAndMissingIndexKeysCheck", + maxDocsPerBatch: batchSize, + batchWriteConcern: writeConcern, + }); + } + + // TODO SERVER-89817: uncomment when write concern erroring is backported for extra index keys + // check. else if (validateMode == "extraIndexKeysCheck") { + // jsTestLog("Running dbCheck extraIndexKeysCheck"); + // runDbCheck(rst, primary.getDB(dbName), collName, { + // validateMode: "extraIndexKeysCheck", + // secondaryIndex: "a_1", + // maxDocsPerBatch: batchSize, + // batchWriteConcern: writeConcern, + // }); + // } + + hangBeforeProcessingDbCheckRunFp.wait(); + stopReplicationOnSecondaries(rst); + + hangBeforeProcessingDbCheckRunFp.off(); + hangBeforeAddingDBCheckBatchToOplogFp.wait(); + hangBeforeAddingDBCheckBatchToOplogFp.off(); + + // Verify that dbCheck stopped after write concern error. + checkHealthLog(primaryHealthLog, logQueries.writeConcernErrorQuery, 1); + checkHealthLog(primaryHealthLog, logQueries.startStopQuery, 2); + // 1 for start, 1 for the batch, 1 for write concern error, 1 for stop + checkHealthLog(primaryHealthLog, {}, 4); + checkHealthLog(secondaryHealthLog, logQueries.startStopQuery, 1); + checkHealthLog(secondaryHealthLog, {}, 1); + + restartReplicationOnSecondaries(rst); + rst.stopSet(); +} + +["dataConsistencyAndMissingIndexKeysCheck", + // TODO SERVER-89817: uncomment when write concern erroring is backported for extra index keys + // check. "extraIndexKeysCheck" +].forEach((failpointName) => { + runTest(failpointName, { + w: 'majority', + wtimeout: 100, + }); + runTest(failpointName, { + w: 3, + wtimeout: 100, + }); +}); diff --git a/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js b/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js new file mode 100644 index 00000000000..e74611ffd1e --- /dev/null +++ b/jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js @@ -0,0 +1,91 @@ +/** + * Tests the skipApplyingDbCheckBatchOnSecondary parameter. + */ +load("jstests/libs/feature_flag_util.js"); +load("jstests/replsets/libs/dbcheck_utils.js"); + +const dbName = jsTestName(); +const collName = jsTestName(); + +const nDocs = 20; +const batchSize = 4; + +function runTest(validateMode) { + const rst = new ReplSetTest({ + name: jsTestName(), + nodes: 2, + nodeOptions: { + setParameter: + {logComponentVerbosity: tojson({command: 3}), dbCheckHealthLogEveryNBatches: 1}, + } + }); + rst.startSet(); + rst.initiateWithHighElectionTimeout(); + const primary = rst.getPrimary(); + + if (validateMode == "extraIndexKeysCheck" && + !FeatureFlagUtil.isEnabled(primary.getDB("admin"), "SecondaryIndexChecksInDbCheck")) { + rst.stopSet(); + return; + } + const secondary = rst.getSecondary(); + const primaryHealthLog = primary.getDB("local").system.healthlog; + const secondaryHealthLog = secondary.getDB("local").system.healthlog; + const primaryDB = primary.getDB(dbName); + const secondaryDB = secondary.getDB(dbName); + + assert.commandWorked( + secondary.adminCommand({"setParameter": 1, "skipApplyingDbCheckBatchOnSecondary": true})); + const writeConcern = {w: 'majority'}; + + resetAndInsert(rst, primaryDB, collName, nDocs); + assert.commandWorked(primaryDB.runCommand({ + createIndexes: collName, + indexes: [{key: {a: 1}, name: 'a_1'}], + })); + rst.awaitReplication(); + + assert.eq(primaryDB.getCollection(collName).find({}).count(), nDocs); + assert.eq(secondaryDB.getCollection(collName).find({}).count(), nDocs); + clearHealthLog(rst); + + if (validateMode == "dataConsistencyCheck") { + jsTestLog("Running dbCheck dataConsistencyCheck"); + runDbCheck(rst, + primary.getDB(dbName), + collName, + { + maxDocsPerBatch: batchSize, + batchWriteConcern: writeConcern, + }, + true /*awaitCompletion*/); + } else if (validateMode == "extraIndexKeysCheck") { + jsTestLog("Running dbCheck extraIndexKeysCheck"); + runDbCheck(rst, + primary.getDB(dbName), + collName, + { + validateMode: "extraIndexKeysCheck", + secondaryIndex: "a_1", + maxDocsPerBatch: batchSize, + batchWriteConcern: writeConcern, + }, + true /*awaitCompletion*/); + } + + checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, 0); + checkHealthLog(primaryHealthLog, logQueries.infoBatchQuery, nDocs / batchSize); + + checkHealthLog(secondaryHealthLog, logQueries.startStopQuery, 2); + checkHealthLog( + secondaryHealthLog, logQueries.skipApplyingBatchOnSecondaryQuery, nDocs / batchSize); + checkHealthLog(secondaryHealthLog, logQueries.allErrorsOrWarningsQuery, nDocs / batchSize); + checkHealthLog(secondaryHealthLog, logQueries.infoBatchQuery, 0); + rst.stopSet(); +} + +["extraIndexKeysCheck", + "dataConsistencyCheck", +].forEach((validateMode) => { + runTest(validateMode); +}); diff --git a/jstests/replsets/dbcheck_write_concern.js b/jstests/replsets/dbcheck_write_concern.js index 4e2a233a2b3..6e102d7bd17 100644 --- a/jstests/replsets/dbcheck_write_concern.js +++ b/jstests/replsets/dbcheck_write_concern.js @@ -99,87 +99,5 @@ const healthlog = db.getSiblingDB('local').system.healthlog; assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); })(); -// Validate that dbCheck completes with w:majority even when the secondary is down and a wtimeout is -// specified. -(function testWMajorityUnavailable() { - clearLog(); - coll.drop(); - - // Insert 1000 docs and run a few small batches to ensure we wait for write concern between - // each one. - const nDocs = 1000; - const maxDocsPerBatch = 100; - assert.commandWorked(coll.insertMany([...Array(nDocs).keys()].map(x => ({a: x})))); - replSet.awaitReplication(); - - // Stop the secondary and expect that the dbCheck batches still complete on the primary. - const secondaryConn = replSet.getSecondary(); - const secondaryNodeId = replSet.getNodeId(secondaryConn); - replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */}); - - assert.commandWorked(db.runCommand({ - dbCheck: coll.getName(), - maxDocsPerBatch: maxDocsPerBatch, - batchWriteConcern: {w: 'majority', wtimeout: 10}, - })); - - // Confirm dbCheck logs the expected number of batches. - assert.soon(function() { - return (healthlog.find({operation: "dbCheckBatch", severity: "info"}).itcount() == - nDocs / maxDocsPerBatch); - }, "dbCheck doesn't seem to complete", 60 * 1000); - - // Confirm dbCheck logs a warning for every batch. - assert.soon(function() { - return (healthlog.find({operation: "dbCheckBatch", severity: "warning"}).itcount() == - nDocs / maxDocsPerBatch); - }, "dbCheck did not log writeConcern warnings", 60 * 1000); - // There should be no errors. - assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); - - replSet.start(secondaryNodeId, {}, true /*restart*/); - replSet.awaitNodesAgreeOnPrimaryNoAuth(); - replSet.awaitReplication(); -})(); - -// Validate that an invalid 'w' setting still allows dbCheck to succeed when presented with a -// wtimeout. -(function testW3Unavailable() { - clearLog(); - coll.drop(); - - // Insert 1000 docs and run a few small batches to ensure we wait for write concern between - // each one. - const nDocs = 1000; - const maxDocsPerBatch = 100; - assert.commandWorked(coll.insertMany([...Array(nDocs).keys()].map(x => ({a: x})))); - replSet.awaitReplication(); - - // Stop the secondary and expect that the dbCheck batches still complete on the primary. - const secondaryConn = replSet.getSecondary(); - const secondaryNodeId = replSet.getNodeId(secondaryConn); - replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */}); - - assert.commandWorked(db.runCommand({ - dbCheck: coll.getName(), - maxDocsPerBatch: maxDocsPerBatch, - batchWriteConcern: {w: 3, wtimeout: 10}, - })); - - // Confirm dbCheck logs the expected number of batches. - assert.soon(function() { - return (healthlog.find({operation: "dbCheckBatch", severity: "info"}).itcount() == - nDocs / maxDocsPerBatch); - }, "dbCheck doesn't seem to complete", 60 * 1000); - - // Confirm dbCheck logs a warning for every batch. - assert.soon(function() { - return (healthlog.find({operation: "dbCheckBatch", severity: "warning"}).itcount() == - nDocs / maxDocsPerBatch); - }, "dbCheck did not log writeConcern warnings", 60 * 1000); - // There should be no errors. - assert.eq(healthlog.find({operation: "dbCheckBatch", severity: "error"}).itcount(), 0); -})(); - replSet.stopSet(); })(); diff --git a/jstests/replsets/empty_ts_repl.js b/jstests/replsets/empty_ts_repl.js new file mode 100644 index 00000000000..3a2052beb45 --- /dev/null +++ b/jstests/replsets/empty_ts_repl.js @@ -0,0 +1,159 @@ +/** + * Tests how replication handles inserts and updates with "Timestamp(0,0)" values. + * + * @tags: [ + * multiversion_incompatible, + * ] + */ +(function() { +"use strict"; + +const dbName = "test"; +const collName = "empty_ts_repl"; + +const rst = new ReplSetTest({ + name: jsTestName(), + nodes: 2, +}); +rst.startSet(); +rst.initiate(); + +const primaryColl = rst.getPrimary().getDB(dbName).getCollection(collName); +const secondaryColl = rst.getSecondary().getDB(dbName).getCollection(collName); +const emptyTs = Timestamp(0, 0); + +// Insert several documents. For the first document inserted (_id=101), the empty timestamp value +// in field "a" should get replaced with the current timestamp. +assert.commandWorked(primaryColl.insert({_id: 101, a: emptyTs})); +assert.commandWorked(primaryColl.insert({_id: 102, a: 1})); +assert.commandWorked(primaryColl.insert({_id: 103, a: 2})); +assert.commandWorked(primaryColl.insert({_id: 106, a: 3})); +assert.commandWorked(primaryColl.insert({_id: 107, a: 4})); +assert.commandWorked(primaryColl.insert({_id: 108, a: 5})); +assert.commandWorked(primaryColl.insert({_id: 109, a: 6})); +assert.commandWorked(primaryColl.insert({_id: 110, a: 7})); +assert.commandWorked(primaryColl.insert({_id: 111, a: 8})); + +// Wait for all the inserted documents to replicate to the secondaries. +rst.awaitReplication(); + +// Use a replacement-style update to update _id=102. This should result in field "a" being set to +// the current timestamp. +assert.commandWorked(primaryColl.update({_id: 102}, {a: emptyTs})); + +// Use a replacement-style findAndModify to update _id=103. This should result in field "a" being +// set to the current timestamp. +let findAndModifyResult = primaryColl.findAndModify({query: {_id: 103}, update: {a: emptyTs}}); +assert.eq(findAndModifyResult, {_id: 103, a: 2}); + +// Do a replacement-style update to add a new document with _id=104. This should result in field "a" +// being set to the current timestamp. +assert.commandWorked(primaryColl.update({_id: 104}, {a: emptyTs}, {upsert: true})); + +// Do a replacement-style findAndModify to add a new document with _id=105. This should result in +// field "a" being set to the current timestamp. +findAndModifyResult = + primaryColl.findAndModify({query: {_id: 105}, update: {a: emptyTs}, upsert: true}); +assert.eq(findAndModifyResult, null); + +// For the rest of the commands below, the empty timestamp values stored in field "a" should be +// preserved as-is. + +// Do an update-operator-style update to update _id=106. +assert.commandWorked(primaryColl.update({_id: 106}, {$set: {a: emptyTs}})); + +// Do an update-operator-style findAndModify to update _id=107. +findAndModifyResult = primaryColl.findAndModify({query: {_id: 107}, update: {$set: {a: emptyTs}}}); +assert.eq(findAndModifyResult, {_id: 107, a: 4}); + +// Do a pipeline-style update to update _id=108. +assert.commandWorked(primaryColl.update({_id: 108}, [{$addFields: {a: emptyTs}}])); + +// Do a pipeline-style findAndModify to update _id=109. +findAndModifyResult = + primaryColl.findAndModify({query: {_id: 109}, update: [{$addFields: {a: emptyTs}}]}); +assert.eq(findAndModifyResult, {_id: 109, a: 6}); + +// Do a pipeline-style update with $internalApplyOplogUpdate to update _id=110. +assert.commandWorked(primaryColl.update( + {_id: 110}, [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}])); + +// Do a pipeline-style findAndModify with $internalApplyOplogUpdate to update _id=111. +findAndModifyResult = primaryColl.findAndModify({ + query: {_id: 111}, + update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}] +}); +assert.eq(findAndModifyResult, {_id: 111, a: 8}); + +// Do an update-operator-style update to add a new document with _id=112. +assert.commandWorked(primaryColl.update({_id: 112}, {$set: {a: emptyTs}}, {upsert: true})); + +// Do an update-operator-style findAndModify to add a new document with _id=113. +findAndModifyResult = + primaryColl.findAndModify({query: {_id: 113}, update: {$set: {a: emptyTs}}, upsert: true}); +assert.eq(findAndModifyResult, null); + +// Do a pipeline-style update to add a new document with _id=114. +assert.commandWorked(primaryColl.update({_id: 114}, [{$addFields: {a: emptyTs}}], {upsert: true})); + +// Do a pipeline-style findAndModify to add a new document with _id=115. +findAndModifyResult = primaryColl.findAndModify( + {query: {_id: 115}, update: [{$addFields: {a: emptyTs}}], upsert: true}); +assert.eq(findAndModifyResult, null); + +// Do a pipeline-style update with $internalApplyOplogUpdate to add a new document _id=116. +assert.commandWorked(primaryColl.update( + {_id: 116}, + [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + {upsert: true})); + +// Do a pipeline-style findAndModify with $internalApplyOplogUpdate to add a new document _id=117. +findAndModifyResult = primaryColl.findAndModify({ + query: {_id: 117}, + update: [{$_internalApplyOplogUpdate: {oplogUpdate: {$v: 2, diff: {i: {a: emptyTs}}}}}], + upsert: true +}); +assert.eq(findAndModifyResult, null); + +rst.awaitReplication(); + +// Verify that all the insert, update, and findAndModify commands behaved the way we expect and that +// they all were replicated correctly to the secondaries. +for (let i = 101; i <= 117; ++i) { + let result = primaryColl.findOne({_id: i}); + let secondaryResult = secondaryColl.findOne({_id: i}); + + assert.eq(tojson(result), tojson(secondaryResult), "_id=" + i); + + if (i >= 106) { + assert.eq(tojson(result.a), tojson(emptyTs), "_id=" + i); + } else { + assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + i); + } +} + +// Insert a document with _id=Timestamp(0,0). +assert.commandWorked(primaryColl.insert({_id: emptyTs, a: 9})); + +// Verify the document we just inserted can be retrieved using the filter "{_id: Timestamp(0,0)}". +let result = primaryColl.findOne({_id: emptyTs}); +assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); +assert.eq(tojson(result.a), tojson(9), "_id=" + tojson(emptyTs)); + +// Do a replacement-style update on the document. +assert.commandWorked(primaryColl.update({_id: emptyTs}, {_id: emptyTs, a: emptyTs})); + +// Verify the document we just updated can still be retrieved using "{_id: Timestamp(0,0)}" and +// verify that field "a" was set to the current timestamp. +result = primaryColl.findOne({_id: emptyTs}); +assert.eq(tojson(result._id), tojson(emptyTs), "_id=" + tojson(emptyTs)); +assert.neq(tojson(result.a), tojson(emptyTs), "_id=" + tojson(emptyTs)); + +rst.awaitReplication(); + +// Verify that the document was replicated correctly to the secondaries. +let secondaryResult = secondaryColl.findOne({_id: emptyTs}); +assert.eq(tojson(result), tojson(secondaryResult), "_id=" + tojson(emptyTs)); + +rst.stopSet(); +}()); diff --git a/jstests/replsets/ignore_dbcheck_in_initial_sync.js b/jstests/replsets/ignore_dbcheck_in_initial_sync.js index 4f1557e4d85..8efc8dcabf4 100644 --- a/jstests/replsets/ignore_dbcheck_in_initial_sync.js +++ b/jstests/replsets/ignore_dbcheck_in_initial_sync.js @@ -57,6 +57,7 @@ replSet.reInitiate(); initialSyncHangBeforeSplittingControlFlowFailPoint.wait(); +// TODO SERVER-89921: Uncomment validateMode once the relevant tickets are backported. runDbCheck(replSet, primaryDb, collName, @@ -65,11 +66,17 @@ runDbCheck(replSet, maxDocsPerBatch //, validateMode: "dataConsistencyAndMissingIndexKeysCheck" }); +// Wait until the primary finished dbcheck before turning off the initial sync failpoint so that the +// initial sync node would need to apply (ignore) dbCheckStop in initial sync. +checkHealthLog(primaryHealthlog, {"operation": "dbCheckStop"}, 1); + assert.commandWorked(initialSyncNode.adminCommand( {configureFailPoint: 'initialSyncHangBeforeSplittingControlFlow', mode: 'off'})); replSet.waitForState(initialSyncNode, ReplSetTest.State.SECONDARY); +// TODO SERVER-89921: Uncomment all of the following checkHealthLog once the relevant tickets are +// backported. // Check that the primary logged an error health log entry for each document with missing index key. // checkHealthLog(primaryHealthlog, logQueries.missingIndexKeysQuery, nDocs); // Check that the primary does not have other error/warning entries. diff --git a/jstests/replsets/ignore_dbcheck_in_rollback.js b/jstests/replsets/ignore_dbcheck_in_rollback.js new file mode 100644 index 00000000000..239bf0bfc94 --- /dev/null +++ b/jstests/replsets/ignore_dbcheck_in_rollback.js @@ -0,0 +1,104 @@ +/* + * This test makes sure the 'dbcheck' command is ignored during rollback and a warning health log + * entry is logged. + * @tags: [ + * featureFlagSecondaryIndexChecksInDbCheck + * ] + */ + +load("jstests/libs/fail_point_util.js"); +load("jstests/replsets/libs/rollback_test.js"); +load("jstests/replsets/libs/dbcheck_utils.js"); + +// This test injects inconsistencies between replica set members; do not fail because of expected +// dbHash differences. +TestData.skipCollectionAndIndexValidation = true; +TestData.skipCheckDBHashes = true; + +const dbName = "ignore_dbcheck_in_rollback"; +const collName = "ignore_dbcheck_in_rollback-collection"; + +const replSet = new ReplSetTest({ + name: jsTestName(), + nodes: [{}, {}, {rsConfig: {priority: 0}}], + useBridge: true, + settings: {chainingAllowed: false} +}); +replSet.startSet(); +replSet.initiateWithHighElectionTimeout(); + +let primary = replSet.getPrimary(); +let primaryDB = primary.getDB(dbName); +const secondary = replSet.getSecondary(); +const secondaryDb = secondary.getDB(dbName); +const primaryColl = primaryDB.getCollection(collName); + +const nDocs = 200; +resetAndInsert(replSet, primaryDB, collName, nDocs); +assert.commandWorked( + primaryDB.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); +replSet.awaitReplication(); +assert.eq(primaryColl.find({}).count(), nDocs); + +// Set up inconsistency. +const skipUnindexingDocumentWhenDeleted = + configureFailPoint(primaryDB, "skipUnindexingDocumentWhenDeleted", {indexName: "a_1"}); +jsTestLog("Deleting docs"); +const stableTimestamp = assert.commandWorked(primaryColl.deleteMany({})); + +replSet.awaitReplication(); +assert.eq(primaryColl.find({}).count(), 0); +assert.eq(secondaryDb.getCollection(collName).find({}).count(), 0); + +const rollbackTest = new RollbackTest(jsTestName(), replSet); +primary = rollbackTest.getPrimary(); +primaryDB = primary.getDB(dbName); + +// Hold stable timestamp. +const stableTimestampFailPoint = configureFailPoint( + primary, "holdStableTimestampAtSpecificTimestamp", {timestamp: stableTimestamp}); + +// TODO SERVER-89921: Uncomment validateMode and secondaryIndex once the relevant tickets are +// backported. +runDbCheck(rollbackTest, + primaryDB, + collName, + { + maxDocsPerBatch: 20 //, validateMode: "extraIndexKeysCheck", secondaryIndex: "a_1" + }); + +// TODO SERVER-89921: Uncomment checkHealthLog once the relevant tickets are backported. +// Check that the old primary prior to transitioning to rollback has start, batch, and stop entries. +const oldPrimaryHealthLog = primary.getDB("local").system.healthlog; +// checkHealthLog(oldPrimaryHealthLog, logQueries.recordNotFoundQuery, nDocs); +checkHealthLog(oldPrimaryHealthLog, logQueries.startStopQuery, 2); + +const rollbackNode = rollbackTest.transitionToRollbackOperations(); +rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); +rollbackTest.transitionToSyncSourceOperationsDuringRollback(); +rollbackTest.transitionToSteadyStateOperations({skipDataConsistencyChecks: true}); + +primary = rollbackTest.getPrimary(); +const primaryHealthLog = primary.getDB("local").system.healthlog; +const rollbackNodeHealthLog = rollbackNode.getDB("local").system.healthlog; + +// TODO SERVER-89921: Change the # of logs to check once the relevant tickets are backported. +// Check that the start, batch (10 batches), and stop entries on the rollback node are all warning +// logs. +checkHealthLog(rollbackNodeHealthLog, logQueries.duringStableRecovery, 3); // 12 +// The rollback node will contain start, batch, and stop entries from when it was primary. Check +// that there are no extra error logs other than recordNotFound. +checkHealthLog(rollbackNodeHealthLog, logQueries.startStopQuery, 2); +// checkHealthLog(rollbackNodeHealthLog, logQueries.recordNotFoundQuery, nDocs); +checkHealthLog(rollbackNodeHealthLog, logQueries.allErrorsOrWarningsQuery, 3); // nDocs + 12 + +// TODO SERVER-89921: Uncomment the following checkHealthLog once the relevant tickets are +// backported. +// Check that the primary only has the batch inconsistent entries from when it was secondary. +// checkHealthLog(primaryHealthLog, logQueries.inconsistentBatchQuery, 10); +// Check that the primary does not have other error/warning entries. +// checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, 10); + +skipUnindexingDocumentWhenDeleted.off(); +stableTimestampFailPoint.off(); +rollbackTest.stop(null /* checkDataConsistencyOptions */, true /* skipDataConsistencyCheck */); diff --git a/jstests/replsets/ignore_dbcheck_in_startup_recovery.js b/jstests/replsets/ignore_dbcheck_in_startup_recovery.js new file mode 100644 index 00000000000..ff770e38ec4 --- /dev/null +++ b/jstests/replsets/ignore_dbcheck_in_startup_recovery.js @@ -0,0 +1,101 @@ +/** + * Test that dbcheck command will be ignored during startup recovery. + * + * @tags: [ + * featureFlagSecondaryIndexChecksInDbCheck, requires_persistence + * ] + */ + +load("jstests/libs/fail_point_util.js"); +load("jstests/replsets/libs/dbcheck_utils.js"); + +// This test injects inconsistencies between replica set members; do not fail because of expected +// dbHash differences. +TestData.skipCollectionAndIndexValidation = true; +TestData.skipCheckDBHashes = true; + +const dbName = "ignore_dbcheck_in_startup_recovery"; +const collName = "ignore_dbcheck_in_startup_recovery-collection"; + +const replSet = new ReplSetTest({ + name: jsTestName(), + nodes: 2, + nodeOptions: {setParameter: {dbCheckHealthLogEveryNBatches: 1}} +}); +replSet.startSet(); +replSet.initiateWithHighElectionTimeout(); + +let primary = replSet.getPrimary(); +const primaryDb = primary.getDB(dbName); +let secondary = replSet.getSecondary(); +const secondaryDb = secondary.getDB(dbName); +const primaryColl = primaryDb.getCollection(collName); + +const nDocs = 200; +resetAndInsert(replSet, primaryDb, collName, nDocs); +assert.commandWorked( + primaryDb.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); +replSet.awaitReplication(); +assert.eq(primaryColl.find({}).count(), nDocs); + +// Set up inconsistency. +const skipUnindexingDocumentWhenDeleted = + configureFailPoint(primaryDb, "skipUnindexingDocumentWhenDeleted", {indexName: "a_1"}); +jsTestLog("Deleting docs"); +const stableTimestamp = assert.commandWorked(primaryColl.deleteMany({})); + +replSet.awaitReplication(); +assert.eq(primaryColl.find({}).count(), 0); +assert.eq(secondaryDb.getCollection(collName).find({}).count(), 0); + +configureFailPoint(primary, "holdStableTimestampAtSpecificTimestamp", {timestamp: stableTimestamp}); + +// TODO SERVER-89921: Uncomment minKey, maxKey, validateMode and secondaryIndex once the relevant +// tickets are backported. +runDbCheck(replSet, + primaryDb, + collName, + { + maxDocsPerBatch: 20, + batchWriteConcern: {w: 2}, + // minKey: {a: "0"}, + // maxKey: {a: "199"}, + // , validateMode: "extraIndexKeysCheck", secondaryIndex: "a_1" + }, + true /*awaitCompletion*/); + +// Wait to make sure that the secondary applied the stop entry before the ungraceful shutdown. +replSet.awaitNodesAgreeOnAppliedOpTime(); + +// Perform ungraceful shutdown of the secondary node and do not clean the db path directory. +replSet.stop( + 1, 9, {allowedExitCode: MongoRunner.EXIT_SIGKILL}, {forRestart: true, skipValidation: true}); + +secondary = replSet.start(secondary, + { + setParameter: { + "failpoint.stopReplProducer": tojson({"mode": "alwaysOn"}), + }, + }, + {noCleanData: true}); + +primary = replSet.getPrimary(); + +const primaryHealthLog = primary.getDB("local").system.healthlog; +const secondaryHealthLog = secondary.getDB("local").system.healthlog; + +// Check that the start and batch entries are warning logs since the node skips dbcheck during +// startup recovery. +checkHealthLog(secondaryHealthLog, logQueries.duringStableRecovery, 3); +// Check that the initial sync node does not have other info or error/warning entries. +checkHealthLog(secondaryHealthLog, logQueries.infoBatchQuery, 0); +checkHealthLog(secondaryHealthLog, logQueries.allErrorsOrWarningsQuery, 3); + +// TODO SERVER-89921: Uncomment checkHealthLog once the relevant tickets are backported. +// Check that the primary logged an error health log entry for each document with missing index key. +// checkHealthLog(primaryHealthLog, logQueries.recordNotFoundQuery, nDocs); +// Check that the primary does not have other error/warning entries. +// checkHealthLog(primaryHealthLog, logQueries.allErrorsOrWarningsQuery, nDocs); + +skipUnindexingDocumentWhenDeleted.off(); +replSet.stopSet(); diff --git a/jstests/replsets/libs/dbcheck_utils.js b/jstests/replsets/libs/dbcheck_utils.js index 5bf564b54b0..b138322a07e 100644 --- a/jstests/replsets/libs/dbcheck_utils.js +++ b/jstests/replsets/libs/dbcheck_utils.js @@ -32,12 +32,26 @@ const logQueries = { }, duringInitialSyncQuery: {severity: "warning", "msg": "cannot execute dbcheck due to ongoing initial sync"}, + duringStableRecovery: + {severity: "warning", "msg": "cannot execute dbcheck due to ongoing stable recovering"}, errorQuery: {"severity": "error"}, warningQuery: {"severity": "warning"}, infoOrErrorQuery: {$or: [{"severity": "info", "operation": "dbCheckBatch"}, {"severity": "error"}]}, infoBatchQuery: {"severity": "info", "operation": "dbCheckBatch"}, inconsistentBatchQuery: {"severity": "error", "msg": "dbCheck batch inconsistent"}, + startStopQuery: { + $or: [ + {"operation": "dbCheckStart", "severity": "info"}, + {"operation": "dbCheckStop", "severity": "info"} + ] + }, + writeConcernErrorQuery: {severity: "error", "msg": "dbCheck failed waiting for writeConcern"}, + skipApplyingBatchOnSecondaryQuery: { + severity: "warning", + "msg": + "skipping applying dbcheck batch because the 'skipApplyingDbCheckBatchOnSecondary' parameter is on", + }, }; // Apply function on all secondary nodes except arbiters. @@ -430,7 +444,7 @@ const assertForDbCheckErrorsForAllNodes = * Utility for checking if the featureFlagSecondaryIndexChecksInDbCheck is on. */ function checkSecondaryIndexChecksInDbCheckFeatureFlagEnabled(conn) { - return FeatureFlagUtil.isPresentAndEnabled(conn, 'SecondaryIndexChecksInDbCheck'); + return FeatureFlagUtil.isEnabled(conn.getDB("admin"), 'SecondaryIndexChecksInDbCheck'); } function checkNumSnapshots(debugBuild, expectedNumSnapshots) { diff --git a/jstests/replsets/libs/rollback_test.js b/jstests/replsets/libs/rollback_test.js index 40b602e5b61..d6838deffcf 100644 --- a/jstests/replsets/libs/rollback_test.js +++ b/jstests/replsets/libs/rollback_test.js @@ -543,11 +543,11 @@ function RollbackTest(name = "RollbackTest", replSet) { return curPrimary; }; - this.stop = function(checkDataConsistencyOptions) { + this.stop = function(checkDataConsistencyOptions, skipDataConsistencyCheck = false) { const start = new Date(); restartServerReplication(tiebreakerNode); rst.awaitReplication(); - if (!doneConsistencyChecks) { + if (!doneConsistencyChecks && !skipDataConsistencyCheck) { this.checkDataConsistency(checkDataConsistencyOptions); } transitionIfAllowed(State.kStopped); diff --git a/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js b/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js new file mode 100644 index 00000000000..d7a9d8858de --- /dev/null +++ b/jstests/replsets/log_wt_stats_during_secondary_oplog_application.js @@ -0,0 +1,46 @@ +/** + * Tests that the server prints storage statistics along with slow secondary oplog application + * logs. + * + * @tags: [requires_wiredtiger, requires_persistence] + */ + +load("jstests/libs/fail_point_util.js"); +load("jstests/replsets/rslib.js"); + +const name = "log_wt_stats_during_secondary_oplog_application"; +const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0}}]}); +rst.startSet(); +rst.initiate(); + +let primary = rst.getPrimary(); +let secondary = rst.getSecondary(); + +// Create a collection and write some data to it. +assert.commandWorked(primary.getDB(name).createCollection("readFromDisk")); +assert.commandWorked(primary.getDB(name)["readFromDisk"].insert({x: "value"})); + +// Cleanly shut down the secondary and restart it. This will clear the local cache and ensure +// that we need to read from disk the next time we write to the collection, resulting in +// actual storage statistics instead of empty storage statistics. +rst.stop(secondary, undefined /* signal */, {} /* opts */, {forRestart: true}); +rst.start(secondary, {} /* options */, true /* restart */, false /* waitForHealth */); + +// Ensure secondary completes startup recovery. +secondary = rst.getSecondary(); + +// Set profiling level to 2 so that we log all operations. +assert.commandWorked(secondary.getDB(name).setProfilingLevel(2, 0)); + +// Issue a new write to the primary that will require reading from disk. +assert.commandWorked(primary.getDB(name)["readFromDisk"].insert({x: "sloth"})); +rst.awaitReplication(); + +// Make sure we log that insert op. +const slowLogLine = checkLog.containsLog(secondary, "sloth"); +jsTestLog(slowLogLine); + +// Make sure we've logged a storage statistic as well. +assert(slowLogLine.includes("bytesRead")); + +rst.stopSet(); diff --git a/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js b/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js index 8ba24bdaa9b..c29e7705614 100644 --- a/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js +++ b/jstests/replsets/nodes_eventually_sync_from_closer_data_center.js @@ -105,7 +105,13 @@ assert.soon(() => { const syncSourcePingTime = replSetGetStatus.members[0].pingMs; const receivedSyncSourceHb = (syncSourcePingTime > 60); - return (receivedCentralHb && receivedSyncSourceHb); + // Wait for enough heartbeat's from the desired sync source so that our understanding of the + // ping time to that node is at least 'changeSyncSourceThresholdMillis' less than the ping time + // to our current sync source. + const centralSecondaryPingTime = replSetGetStatus.members[1].pingMs; + const exceedsChangeSyncSourceThreshold = (syncSourcePingTime - centralSecondaryPingTime > 5); + + return (receivedCentralHb && receivedSyncSourceHb && exceedsChangeSyncSourceThreshold); }); const replSetGetStatus = assert.commandWorked(testNode.adminCommand({replSetGetStatus: 1})); diff --git a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js index 99018d864c5..56169ee402c 100644 --- a/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js +++ b/jstests/replsets/reconstruct_prepared_transactions_initial_sync.js @@ -59,6 +59,10 @@ assert.commandWorked(sessionColl1.insert({_id: 1})); assert.commandWorked(sessionColl2.insert({_id: 2})); assert.commandWorked(sessionColl3.insert({_id: 3})); assert.commandWorked(sessionColl3.insert({_id: 4})); +assert.commandWorked(sessionColl3.insert({_id: 5})); + +// Add a validator so we can make sure validation doesn't cause initial sync to fail. +assert.commandWorked(testDB.runCommand({collMod: collName, validator: {b: {$exists: false}}})); jsTestLog("Preparing three transactions"); @@ -111,7 +115,7 @@ jsTestLog("Running operations while collection cloning is paused"); // Perform writes while collection cloning is paused so that we know they must be applied during // the oplog application stage of initial sync. -assert.commandWorked(testColl.insert({_id: 5})); +assert.commandWorked(testColl.insert({_id: 999})); let session4 = primary.startSession(); let sessionDB4 = session4.getDatabase(dbName); @@ -125,6 +129,22 @@ session4.startTransaction(); assert.commandWorked(sessionColl4.update({_id: 4}, {_id: 4, a: 1})); const prepareTimestamp4 = PrepareHelpers.prepareTransaction(session4, {w: 1}); +let session5 = primary.startSession(); +let sessionDB5 = session5.getDatabase(dbName); +const sessionColl5 = sessionDB5.getCollection(collName); + +jsTestLog("Preparing the fifth transaction"); + +// Prepare a transaction that would fail validation while collection cloning is paused so that its +// oplog entry must be applied during the oplog application phase of initial sync. +session5.startTransaction(); +assert.commandWorked(sessionDB5.runCommand({ + update: collName, + updates: [{q: {_id: 5}, u: {_id: 5, a: 1, b: 2}}], + bypassDocumentValidation: true +})); +const prepareTimestamp5 = PrepareHelpers.prepareTransaction(session5, {w: 1}); + jsTestLog("Resuming initial sync"); // Resume initial sync. @@ -145,7 +165,7 @@ const secondaryColl = secondary.getDB(dbName).getCollection(collName); // changes to the documents from any of the prepared transactions after initial sync. Also, make // sure that the writes that happened when collection cloning was paused happened. const res = secondaryColl.find().sort({_id: 1}).toArray(); -assert.eq(res, [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}], res); +assert.eq(res, [{_id: 1}, {_id: 2}, {_id: 3}, {_id: 4}, {_id: 5}, {_id: 999}], res); jsTestLog("Checking that the first transaction is properly prepared"); @@ -173,9 +193,15 @@ jsTestLog("Committing the fourth transaction"); assert.commandWorked(PrepareHelpers.commitTransaction(session4, prepareTimestamp4)); replTest.awaitReplication(); +jsTestLog("Committing the fifth transaction"); + +assert.commandWorked(PrepareHelpers.commitTransaction(session5, prepareTimestamp5)); +replTest.awaitReplication(); + // Make sure that we can see the data from a committed transaction on the secondary if it was // applied during secondary oplog application. -assert.docEq(secondaryColl.findOne({_id: 4}), {_id: 4, a: 1}); +assert.docEq({_id: 4, a: 1}, secondaryColl.findOne({_id: 4})); +assert.docEq({_id: 5, a: 1, b: 2}, secondaryColl.findOne({_id: 5})); jsTestLog("Stepping up the secondary"); @@ -271,4 +297,4 @@ assert.commandWorked(sessionDB3.adminCommand( assert.eq(testColl.find({_id: 3}).toArray(), [{_id: 3}]); replTest.stopSet(); -})();
\ No newline at end of file +})(); diff --git a/jstests/replsets/tenant_migration_abort_forget_retry.js b/jstests/replsets/tenant_migration_abort_forget_retry.js deleted file mode 100644 index 21913140ba6..00000000000 --- a/jstests/replsets/tenant_migration_abort_forget_retry.js +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Starts a tenant migration that aborts, either due to the - * abortTenantMigrationBeforeLeavingBlockingState failpoint or due to receiving donorAbortMigration, - * and then issues a donorForgetMigration command. Finally, starts a second tenant migration with - * the same tenantId as the aborted migration, and expects this second migration to go through. - * - * @tags: [ - * incompatible_with_eft, - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_majority_read_concern, - * requires_persistence, - * serverless, - * ] - */ - -(function() { -"use strict"; - -load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallelTester.js"); -load("jstests/libs/uuid_util.js"); -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); - -const kTenantIdPrefix = "testTenantId"; -let testNum = 0; - -function makeTenantId() { - return kTenantIdPrefix + testNum++; -} - -const tenantMigrationTest = - new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); - -(() => { - const migrationId1 = extractUUIDFromObject(UUID()); - const migrationId2 = extractUUIDFromObject(UUID()); - const tenantId = makeTenantId(); - - // Start a migration with the "abortTenantMigrationBeforeLeavingBlockingState" failPoint - // enabled. The migration will abort as a result, and a status of "kAborted" should be returned. - jsTestLog( - "Starting a migration that is expected to abort due to setting abortTenantMigrationBeforeLeavingBlockingState failpoint. migrationId: " + - migrationId1 + ", tenantId: " + tenantId); - const donorPrimary = tenantMigrationTest.getDonorPrimary(); - const abortFp = - configureFailPoint(donorPrimary, "abortTenantMigrationBeforeLeavingBlockingState"); - TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration( - {migrationIdString: migrationId1, tenantId: tenantId}, {automaticForgetMigration: false})); - abortFp.off(); - - // Forget the aborted migration. - jsTestLog("Forgetting aborted migration with migrationId: " + migrationId1); - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationId1)); - - // Try running a new migration with the same tenantId. It should succeed, since the previous - // migration with the same tenantId was aborted. - jsTestLog("Attempting to run a new migration with the same tenantId. New migrationId: " + - migrationId2 + ", tenantId: " + tenantId); - TenantMigrationTest.assertCommitted( - tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); -})(); - -(() => { - const migrationId1 = extractUUIDFromObject(UUID()); - const migrationId2 = extractUUIDFromObject(UUID()); - const tenantId = makeTenantId(); - - jsTestLog( - "Starting a migration that is expected to abort in blocking state due to receiving donorAbortMigration. migrationId: " + - migrationId1 + ", tenantId: " + tenantId); - - const donorPrimary = tenantMigrationTest.getDonorPrimary(); - let fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState"); - assert.commandWorked( - tenantMigrationTest.startMigration({migrationIdString: migrationId1, tenantId: tenantId})); - - fp.wait(); - - const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()); - const tryAbortThread = new Thread(TenantMigrationUtil.tryAbortMigrationAsync, - {migrationIdString: migrationId1, tenantId: tenantId}, - donorRstArgs, - TenantMigrationUtil.runTenantMigrationCommand); - tryAbortThread.start(); - - // Wait for donorAbortMigration command to start. - assert.soon(() => { - const res = assert.commandWorked( - donorPrimary.adminCommand({currentOp: true, desc: "tenant donor migration"})); - const op = res.inprog.find(op => extractUUIDFromObject(op.instanceID) === migrationId1); - return op.receivedCancellation; - }); - - fp.off(); - - tryAbortThread.join(); - assert.commandWorked(tryAbortThread.returnData()); - - TenantMigrationTest.assertAborted(tenantMigrationTest.waitForMigrationToComplete( - {migrationIdString: migrationId1, tenantId: tenantId})); - - // Forget the aborted migration. - jsTestLog("Forgetting aborted migration with migrationId: " + migrationId1); - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationId1)); - - // Try running a new migration with the same tenantId. It should succeed, since the previous - // migration with the same tenantId was aborted. - jsTestLog("Attempting to run a new migration with the same tenantId. New migrationId: " + - migrationId2 + ", tenantId: " + tenantId); - TenantMigrationTest.assertCommitted( - tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); -})(); - -tenantMigrationTest.stop(); -})(); diff --git a/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js b/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js new file mode 100644 index 00000000000..c1031db50d2 --- /dev/null +++ b/jstests/replsets/timeseries_mixed_schema_bucket_initial_sync.js @@ -0,0 +1,74 @@ +/** + * Tests initial sync with a time-series collection that contains mixed-schema buckets. + * + * @tags: [ + * requires_fcv_60, + * ] + */ +(function() { +"use strict"; + +load("jstests/core/timeseries/libs/timeseries.js"); // For 'TimeseriesTest'. + +TestData.skipEnforceTimeseriesBucketsAreAlwaysCompressedOnValidate = true; + +const replTest = new ReplSetTest({nodes: 1}); +replTest.startSet(); +replTest.initiate(); + +const primary = replTest.getPrimary(); +const db = primary.getDB(jsTestName()); +const coll = db.coll; +const bucketsColl = db["system.buckets." + coll.getName()]; + +const bucket = { + _id: ObjectId("65a6eb806ffc9fa4280ecac4"), + control: { + version: NumberInt(1), + min: { + _id: ObjectId("65a6eba7e6d2e848e08c3750"), + t: ISODate("2024-01-16T20:48:00Z"), + a: 1, + }, + max: { + _id: ObjectId("65a6eba7e6d2e848e08c3751"), + t: ISODate("2024-01-16T20:48:39.448Z"), + a: "a", + }, + }, + meta: 0, + data: { + _id: { + 0: ObjectId("65a6eba7e6d2e848e08c3750"), + 1: ObjectId("65a6eba7e6d2e848e08c3751"), + }, + t: { + 0: ISODate("2024-01-16T20:48:39.448Z"), + 1: ISODate("2024-01-16T20:48:39.448Z"), + }, + a: { + 0: "a", + 1: 1, + }, + } +}; + +assert.commandWorked( + db.createCollection(coll.getName(), {timeseries: {timeField: "t", metaField: "m"}})); +assert.commandWorked( + db.runCommand({collMod: coll.getName(), timeseriesBucketsMayHaveMixedSchemaData: true})); +assert.commandWorked(bucketsColl.insert(bucket)); + +const secondary = replTest.add(); +replTest.reInitiate(); +replTest.waitForState(secondary, ReplSetTest.State.SECONDARY); +replTest.awaitReplication(); + +const primaryColl = primary.getDB(db.getName())[bucketsColl.getName()]; +const secondaryColl = secondary.getDB(db.getName())[bucketsColl.getName()]; + +assert(TimeseriesTest.bucketsMayHaveMixedSchemaData(primaryColl)); +assert(TimeseriesTest.bucketsMayHaveMixedSchemaData(secondaryColl)); + +replTest.stopSet(); +})(); |