summaryrefslogtreecommitdiff
path: root/jstests/replsets/libs
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /jstests/replsets/libs
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'jstests/replsets/libs')
-rw-r--r--jstests/replsets/libs/dbcheck_utils.js462
-rw-r--r--jstests/replsets/libs/oplog_rollover_test.js204
-rw-r--r--jstests/replsets/libs/rollback_resumable_index_build.js6
-rw-r--r--jstests/replsets/libs/rollback_test.js8
-rw-r--r--jstests/replsets/libs/secondary_reads_test.js4
-rw-r--r--jstests/replsets/libs/tenant_migration_test.js11
6 files changed, 8 insertions, 687 deletions
diff --git a/jstests/replsets/libs/dbcheck_utils.js b/jstests/replsets/libs/dbcheck_utils.js
deleted file mode 100644
index b138322a07e..00000000000
--- a/jstests/replsets/libs/dbcheck_utils.js
+++ /dev/null
@@ -1,462 +0,0 @@
-/**
- * Contains helper functions for testing dbCheck.
- */
-load("jstests/libs/fail_point_util.js");
-load("jstests/libs/feature_flag_util.js");
-
-const logQueries = {
- allErrorsOrWarningsQuery: {$or: [{"severity": "warning"}, {"severity": "error"}]},
- recordNotFoundQuery: {
- "severity": "error",
- "msg": "found extra index key entry without corresponding document",
- "data.context.indexSpec": {$exists: true}
- },
- missingIndexKeysQuery: {
- "severity": "error",
- "msg": "Document has missing index keys",
- "data.context.missingIndexKeys": {$exists: true},
- },
- recordDoesNotMatchQuery: {
- "severity": "error",
- "msg":
- "found index key entry with corresponding document/keystring set that does not contain the expected key string",
- "data.context.indexSpec": {$exists: true}
- },
- collNotFoundWarningQuery: {
- severity: "warning",
- "msg": "abandoning dbCheck extra index keys check because collection no longer exists"
- },
- indexNotFoundWarningQuery: {
- severity: "warning",
- "msg": "abandoning dbCheck extra index keys check because index no longer exists"
- },
- duringInitialSyncQuery:
- {severity: "warning", "msg": "cannot execute dbcheck due to ongoing initial sync"},
- duringStableRecovery:
- {severity: "warning", "msg": "cannot execute dbcheck due to ongoing stable recovering"},
- errorQuery: {"severity": "error"},
- warningQuery: {"severity": "warning"},
- infoOrErrorQuery:
- {$or: [{"severity": "info", "operation": "dbCheckBatch"}, {"severity": "error"}]},
- infoBatchQuery: {"severity": "info", "operation": "dbCheckBatch"},
- inconsistentBatchQuery: {"severity": "error", "msg": "dbCheck batch inconsistent"},
- startStopQuery: {
- $or: [
- {"operation": "dbCheckStart", "severity": "info"},
- {"operation": "dbCheckStop", "severity": "info"}
- ]
- },
- writeConcernErrorQuery: {severity: "error", "msg": "dbCheck failed waiting for writeConcern"},
- skipApplyingBatchOnSecondaryQuery: {
- severity: "warning",
- "msg":
- "skipping applying dbcheck batch because the 'skipApplyingDbCheckBatchOnSecondary' parameter is on",
- },
-};
-
-// Apply function on all secondary nodes except arbiters.
-const forEachNonArbiterSecondary = (replSet, f) => {
- for (let secondary of replSet.getSecondaries()) {
- if (!secondary.adminCommand({isMaster: 1}).arbiterOnly) {
- f(secondary);
- }
- }
-};
-
-// Apply function on primary and all secondary nodes.
-const forEachNonArbiterNode = (replSet, f) => {
- f(replSet.getPrimary());
- forEachNonArbiterSecondary(replSet, f);
-};
-
-// Clear local.system.healthlog.
-const clearHealthLog = (replSet) => {
- forEachNonArbiterNode(replSet, conn => conn.getDB("local").system.healthlog.drop());
- replSet.awaitReplication();
-};
-
-const logEveryBatch = (replSet) => {
- forEachNonArbiterNode(replSet, conn => {
- conn.adminCommand({setParameter: 1, "dbCheckHealthLogEveryNBatches": 1});
- });
-};
-
-const dbCheckCompleted = (db) => {
- const inprog = db.getSiblingDB("admin").currentOp().inprog;
- return inprog == undefined || inprog.filter(x => x["desc"] == "dbCheck")[0] === undefined;
-};
-
-// Wait for dbCheck to complete (on both primaries and secondaries).
-const awaitDbCheckCompletion =
- (replSet, db, waitForHealthLogDbCheckStop = true, awaitCompletionTimeoutMs = null) => {
- assert.soon(
- () => dbCheckCompleted(db),
- "dbCheck timed out for database: " + db.getName() + " for RS: " + replSet.getURL(),
- awaitCompletionTimeoutMs);
-
- const tokens = replSet.nodes.map(node => node._securityToken);
- try {
- // This function might be called with a security token (to specify a tenant) on a
- // connection. Calling tenant agnostic commands to await replication conflict with this
- // token so temporarily remove it.
- replSet.nodes.forEach(node => node._setSecurityToken(undefined));
- replSet.awaitSecondaryNodes();
- replSet.awaitReplication();
-
- if (waitForHealthLogDbCheckStop) {
- forEachNonArbiterNode(replSet, function(node) {
- const healthlog = node.getDB('local').system.healthlog;
- assert.soon(
- function() {
- return (healthlog.find({"operation": "dbCheckStop"}).itcount() == 1);
- },
- "dbCheck command didn't complete for database: " + db.getName() +
- " for RS: " + replSet.getURL());
- });
- }
- } finally {
- replSet.nodes.forEach((node, idx) => {
- node._setSecurityToken(tokens[idx]);
- });
- }
- };
-
-// Clear health log and insert nDocs documents.
-const resetAndInsert = (replSet, db, collName, nDocs, docSuffix = null) => {
- db[collName].drop();
- clearHealthLog(replSet);
-
- if (docSuffix) {
- assert.commandWorked(db[collName].insertMany(
- [...Array(nDocs).keys()].map(x => ({a: x.toString() + docSuffix})), {ordered: false}));
- } else {
- assert.commandWorked(
- db[collName].insertMany([...Array(nDocs).keys()].map(x => ({a: x})), {ordered: false}));
- }
-
- replSet.awaitReplication();
- assert.eq(db.getCollection(collName).find({}).count(), nDocs);
-};
-
-// Clear health log and insert nDocs documents with two fields `a` and `b`.
-const resetAndInsertTwoFields = (replSet, db, collName, nDocs, docSuffix = null) => {
- db[collName].drop();
- clearHealthLog(replSet);
-
- if (docSuffix) {
- assert.commandWorked(db[collName].insertMany(
- [...Array(nDocs).keys()].map(
- x => ({a: x.toString() + docSuffix, b: x.toString() + docSuffix})),
- {ordered: false}));
- } else {
- assert.commandWorked(db[collName].insertMany(
- [...Array(nDocs).keys()].map(x => ({a: x, b: x})), {ordered: false}));
- }
-
- replSet.awaitReplication();
- assert.eq(db.getCollection(collName).find({}).count(), nDocs);
-};
-
-// Clear health log and insert nDocs documents with identical 'a' field
-const resetAndInsertIdentical = (replSet, db, collName, nDocs) => {
- db[collName].drop();
- clearHealthLog(replSet);
-
- assert.commandWorked(db[collName].insertMany(
- [...Array(nDocs).keys()].map(x => ({_id: x, a: 0})), {ordered: false}));
-
- replSet.awaitReplication();
- assert.eq(db.getCollection(collName).find({}).count(), nDocs);
-};
-
-// Insert numDocs documents with missing index keys for testing.
-const insertDocsWithMissingIndexKeys =
- (replSet, dbName, collName, doc, numDocs = 1, doPrimary = true, doSecondary = true) => {
- const primaryDb = replSet.getPrimary().getDB(dbName);
- const secondaryDb = replSet.getSecondary().getDB(dbName);
-
- assert.commandWorked(primaryDb.createCollection(collName));
-
- // Create an index for every key in the document.
- let index = {};
- for (let key in doc) {
- index[key] = 1;
- assert.commandWorked(primaryDb[collName].createIndex(index));
- index = {};
- }
- replSet.awaitReplication();
-
- // dbCheck requires the _id index to iterate through documents in a batch.
- let skipIndexNewRecordsExceptIdPrimary;
- let skipIndexNewRecordsExceptIdSecondary;
- if (doPrimary) {
- skipIndexNewRecordsExceptIdPrimary =
- configureFailPoint(primaryDb, "skipIndexNewRecords", {skipIdIndex: false});
- }
- if (doSecondary) {
- skipIndexNewRecordsExceptIdSecondary =
- configureFailPoint(secondaryDb, "skipIndexNewRecords", {skipIdIndex: false});
- }
- for (let i = 0; i < numDocs; i++) {
- assert.commandWorked(primaryDb[collName].insert(doc));
- }
- replSet.awaitReplication();
- if (doPrimary) {
- skipIndexNewRecordsExceptIdPrimary.off();
- }
- if (doSecondary) {
- skipIndexNewRecordsExceptIdSecondary.off();
- }
-
- // Verify that index has been replicated to all nodes, including _id index.
- forEachNonArbiterNode(replSet, function(node) {
- assert.eq(Object.keys(doc).length + 1,
- node.getDB(dbName)[collName].getIndexes().length);
- });
- };
-
-// Run dbCheck with given parameters and potentially wait for completion.
-const runDbCheck = (replSet,
- db,
- collName,
- parameters = {},
- awaitCompletion = false,
- waitForHealthLogDbCheckStop = true,
- allowedErrorCodes = []) => {
- if (!parameters.hasOwnProperty('maxBatchTimeMillis')) {
- // Make this huge because stalls and pauses sometimes break this test.
- parameters['maxBatchTimeMillis'] = 20000;
- }
- let dbCheckCommand = {dbCheck: collName};
- for (let parameter in parameters) {
- dbCheckCommand[parameter] = parameters[parameter];
- }
-
- let res =
- assert.commandWorkedOrFailedWithCode(db.runCommand(dbCheckCommand), allowedErrorCodes);
- if (res.ok && awaitCompletion) {
- awaitDbCheckCompletion(replSet, db, waitForHealthLogDbCheckStop);
- }
-};
-
-const checkHealthLog = (healthlog, query, numExpected, timeout = 60 * 1000) => {
- let query_count;
- assert.soon(
- function() {
- query_count = healthlog.find(query).count();
- if (query_count != numExpected) {
- jsTestLog("health log query returned " + query_count + " entries, expected " +
- numExpected + " query: " + tojson(query) +
- " found: " + tojson(healthlog.find(query).toArray()));
- }
- return query_count == numExpected;
- },
- "health log query returned " + query_count + " entries, expected " + numExpected +
- " query: " + tojson(query) + " found: " + tojson(healthlog.find(query).toArray()) +
- " HealthLog: " + tojson(healthlog.find().toArray()),
- timeout);
-};
-
-// Temporarily restart the secondary as a standalone, inject an inconsistency and
-// restart it back as a secondary.
-const injectInconsistencyOnSecondary = (replSet, dbName, cmd, noCleanData = true) => {
- const secondaryConn = replSet.getSecondary();
- const secondaryNodeId = replSet.getNodeId(secondaryConn);
- replSet.stop(secondaryNodeId, {forRestart: true /* preserve dbPath */});
-
- const standaloneConn = MongoRunner.runMongod({
- dbpath: secondaryConn.dbpath,
- noCleanData: noCleanData,
- });
-
- const standaloneDB = standaloneConn.getDB(dbName);
- assert.commandWorked(standaloneDB.runCommand(cmd));
-
- // Shut down the secondary and restart it as a member of the replica set.
- MongoRunner.stopMongod(standaloneConn);
- replSet.start(secondaryNodeId, {}, true /*restart*/);
- replSet.awaitNodesAgreeOnPrimaryNoAuth();
-};
-
-// Returns a list of all collections in a given database excluding views.
-function listCollectionsWithoutViews(database) {
- var failMsg = "'listCollections' command failed";
- // Some tests adds an invalid view, resulting in a failure of the 'listCollections' operation
- // with an 'InvalidViewDefinition' error.
- let res = assert.commandWorkedOrFailedWithCode(
- database.runCommand("listCollections"), ErrorCodes.InvalidViewDefinition, failMsg);
- if (res.ok) {
- return res.cursor.firstBatch.filter(c => c.type == "collection");
- }
- return [];
-}
-
-// Returns a list of names of all indexes.
-function getIndexNames(db, collName, allowedErrorCodes) {
- var failMsg = "'listIndexes' command failed";
- let res = assert.commandWorkedOrFailedWithCode(
- db[collName].runCommand("listIndexes"), allowedErrorCodes, failMsg);
- if (res.ok) {
- return new DBCommandCursor(db, res).toArray().map(spec => spec.name);
- }
- return [];
-}
-
-// List of collection names that are ignored from dbcheck.
-const collNamesIgnoredFromDBCheck = [
- "operationalLatencyHistogramTest_coll_temp",
- "top_coll_temp",
-];
-
-// Run dbCheck for all collections in the database with given parameters and potentially wait for
-// completion.
-const runDbCheckForDatabase =
- (replSet, db, awaitCompletion = false, awaitCompletionTimeoutMs = null) => {
- const secondaryIndexCheckEnabled =
- checkSecondaryIndexChecksInDbCheckFeatureFlagEnabled(replSet.getPrimary());
- let collDbCheckParameters = {};
- if (secondaryIndexCheckEnabled) {
- collDbCheckParameters = {validateMode: "dataConsistencyAndMissingIndexKeysCheck"};
- }
-
- const allowedErrorCodes = [
- ErrorCodes.NamespaceNotFound /* collection got dropped. */,
- ErrorCodes.CommandNotSupportedOnView /* collection got dropped and a view
- got created with the same name. */
- ,
- 40619 /* collection is not replicated error. */,
- // Some tests adds an invalid view, resulting in a failure of the 'dbcheck'
- // operation with an 'InvalidViewDefinition' error.
- ErrorCodes.InvalidViewDefinition,
- // Might hit stale shardVersion response from shard config while racing with
- // 'dropCollection' command.
- ErrorCodes.StaleConfig
- ];
-
- listCollectionsWithoutViews(db).map(c => c.name).forEach(collName => {
- if (collNamesIgnoredFromDBCheck.includes(collName)) {
- jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is skipped on ns: " +
- db.getName() + "." + collName + " for RS: " + replSet.getURL());
- return;
- }
-
- jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is starting on ns: " +
- db.getName() + "." + collName + " for RS: " + replSet.getURL());
- runDbCheck(replSet,
- db,
- collName,
- collDbCheckParameters /* parameters */,
- false /* awaitCompletion */,
- false /* waitForHealthLogDbCheckStop */,
- allowedErrorCodes);
- jsTestLog("dbCheck (" + tojson(collDbCheckParameters) + ") is done on ns: " +
- db.getName() + "." + collName + " for RS: " + replSet.getURL());
-
- if (!secondaryIndexCheckEnabled) {
- return;
- }
-
- getIndexNames(db, collName, allowedErrorCodes).forEach(indexName => {
- let extraIndexDbCheckParameters = {
- validateMode: "extraIndexKeysCheck",
- secondaryIndex: indexName
- };
- jsTestLog("dbCheck (" + tojson(extraIndexDbCheckParameters) +
- ") is starting on ns: " + db.getName() + "." + collName +
- " for RS: " + replSet.getURL());
- runDbCheck(replSet,
- db,
- collName,
- extraIndexDbCheckParameters /* parameters */,
- false /* awaitCompletion */,
- false /* waitForHealthLogDbCheckStop */,
- allowedErrorCodes);
- jsTestLog("dbCheck (" + tojson(extraIndexDbCheckParameters) + ") is done on ns: " +
- db.getName() + "." + collName + " for RS: " + replSet.getURL());
- });
- });
-
- if (awaitCompletion) {
- awaitDbCheckCompletion(
- replSet, db, false /*waitForHealthLogDbCheckStop*/, awaitCompletionTimeoutMs);
- }
- };
-
-// Assert no errors/warnings (i.e., found inconsistencies). Tolerate
-// SnapshotTooOld errors, as they can occur if the primary is slow enough processing a
-// batch that the secondary is unable to obtain the timestamp the primary used.
-const assertForDbCheckErrors = (node,
- assertForErrors = true,
- assertForWarnings = false,
- errorsFound = []) => {
- let severityValues = [];
- if (assertForErrors == true) {
- severityValues.push("error");
- }
-
- if (assertForWarnings == true) {
- severityValues.push("warning");
- }
-
- const healthlog = node.getDB('local').system.healthlog;
- // Regex matching strings that start without "SnapshotTooOld"
- const regexStringWithoutSnapTooOld = /^((?!^SnapshotTooOld).)*$/;
-
- // healthlog is a capped collection, truncation during scan might cause cursor
- // invalidation. Truncated data is most likely from previous tests in the fixture, so we
- // should still be able to catch errors by retrying.
- assert.soon(() => {
- try {
- let errs = healthlog.find(
- {"severity": {$in: severityValues}, "data.error": regexStringWithoutSnapTooOld});
- if (errs.hasNext()) {
- const errMsg = "dbCheck found inconsistency on " + node.host;
- jsTestLog(errMsg + ". Errors/Warnings: ");
- let err;
- for (let count = 0; errs.hasNext() && count < 20; count++) {
- err = errs.next();
- errorsFound.push(err);
- jsTestLog(tojson(err));
- }
- assert(false, errMsg);
- }
- return true;
- } catch (e) {
- if (e.code !== ErrorCodes.CappedPositionLost) {
- throw e;
- }
- jsTestLog(`Retrying on CappedPositionLost error: ${tojson(e)}`);
- return false;
- }
- }, "healthlog scan could not complete.", 60000);
-
- jsTestLog("Checked health log for on " + node.host);
-};
-
-// Check for dbcheck errors for all nodes in a replica set and ignoring arbiters.
-const assertForDbCheckErrorsForAllNodes =
- (rst, assertForErrors = true, assertForWarnings = false) => {
- forEachNonArbiterNode(
- rst, node => assertForDbCheckErrors(node, assertForErrors, assertForWarnings));
- };
-
-/**
- * Utility for checking if the featureFlagSecondaryIndexChecksInDbCheck is on.
- */
-function checkSecondaryIndexChecksInDbCheckFeatureFlagEnabled(conn) {
- return FeatureFlagUtil.isEnabled(conn.getDB("admin"), 'SecondaryIndexChecksInDbCheck');
-}
-
-function checkNumSnapshots(debugBuild, expectedNumSnapshots) {
- if (debugBuild) {
- const actualNumSnapshots =
- rawMongoProgramOutput()
- .split(/7844808.*Catalog snapshot for reverse lookup check ending/)
- .length -
- 1;
- assert.eq(actualNumSnapshots,
- expectedNumSnapshots,
- "expected " + expectedNumSnapshots +
- " catalog snapshots during reverse lookup, found " + actualNumSnapshots);
- }
-}
diff --git a/jstests/replsets/libs/oplog_rollover_test.js b/jstests/replsets/libs/oplog_rollover_test.js
deleted file mode 100644
index 63439f2849f..00000000000
--- a/jstests/replsets/libs/oplog_rollover_test.js
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Test that oplog (on both primary and secondary) rolls over when its size exceeds the configured
- * maximum, with parameters for setting the initial sync method and the storage engine.
- */
-
-"use strict";
-
-load("jstests/libs/fail_point_util.js");
-
-function oplogRolloverTest(storageEngine, initialSyncMethod) {
- jsTestLog("Testing with storageEngine: " + storageEngine);
- if (initialSyncMethod) {
- jsTestLog(" and initial sync method: " + initialSyncMethod);
- }
-
- // Pause the oplog cap maintainer thread for this test until oplog truncation is needed. The
- // truncation thread can hold a mutex for a short period of time which prevents new oplog stones
- // from being created during an insertion if the mutex cannot be obtained immediately. Instead,
- // the next insertion will attempt to create a new oplog stone, which this test does not do.
- let parameters = {
- logComponentVerbosity: tojson({storage: 2}),
- 'failpoint.hangOplogCapMaintainerThread': tojson({mode: 'alwaysOn'})
- };
- if (initialSyncMethod) {
- parameters = Object.merge(parameters, {initialSyncMethod: initialSyncMethod});
- }
- const replSet = new ReplSetTest({
- // Set the syncdelay to 1s to speed up checkpointing.
- nodeOptions: {
- syncdelay: 1,
- setParameter: parameters,
- },
- nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]
- });
- // Set max oplog size to 1MB.
- replSet.startSet({storageEngine: storageEngine, oplogSize: 1});
- replSet.initiate();
-
- const primary = replSet.getPrimary();
- const primaryOplog = primary.getDB("local").oplog.rs;
- const secondary = replSet.getSecondary();
- const secondaryOplog = secondary.getDB("local").oplog.rs;
-
- // Verify that the oplog cap maintainer thread is paused.
- assert.commandWorked(primary.adminCommand({
- waitForFailPoint: "hangOplogCapMaintainerThread",
- timesEntered: 1,
- maxTimeMS: kDefaultWaitForFailPointTimeout
- }));
- assert.commandWorked(secondary.adminCommand({
- waitForFailPoint: "hangOplogCapMaintainerThread",
- timesEntered: 1,
- maxTimeMS: kDefaultWaitForFailPointTimeout
- }));
-
- const coll = primary.getDB("test").foo;
- // 400KB each so that oplog can keep at most two insert oplog entries.
- const longString = new Array(400 * 1024).join("a");
-
- function numInsertOplogEntry(oplog) {
- print(`Oplog times for ${oplog.getMongo().host}: ${
- tojsononeline(oplog.find().projection({ts: 1, t: 1, op: 1, ns: 1}).toArray())}`);
- return oplog.find({op: "i", "ns": "test.foo"}).itcount();
- }
-
- // Insert the first document.
- const firstInsertTimestamp =
- assert
- .commandWorked(coll.runCommand(
- "insert", {documents: [{_id: 0, longString: longString}], writeConcern: {w: 2}}))
- .operationTime;
- jsTestLog("First insert timestamp: " + tojson(firstInsertTimestamp));
-
- // Test that oplog entry of the first insert exists on both primary and secondary.
- assert.eq(1, numInsertOplogEntry(primaryOplog));
- assert.eq(1, numInsertOplogEntry(secondaryOplog));
-
- // Insert the second document.
- const secondInsertTimestamp =
- assert
- .commandWorked(coll.runCommand(
- "insert", {documents: [{_id: 1, longString: longString}], writeConcern: {w: 2}}))
- .operationTime;
- jsTestLog("Second insert timestamp: " + tojson(secondInsertTimestamp));
-
- // Test that oplog entries of both inserts exist on both primary and secondary.
- assert.eq(2, numInsertOplogEntry(primaryOplog));
- assert.eq(2, numInsertOplogEntry(secondaryOplog));
-
- // Have a more fine-grained test for enableMajorityReadConcern=true to also test oplog
- // truncation happens at the time we expect it to happen. When
- // enableMajorityReadConcern=false the lastStableRecoveryTimestamp is not available, so
- // switch to a coarser-grained mode to only test that oplog truncation will eventually
- // happen when oplog size exceeds the configured maximum.
- if (primary.getDB('admin').serverStatus().storageEngine.supportsCommittedReads) {
- const awaitCheckpointer = function(timestamp) {
- assert.soon(
- () => {
- const primaryReplSetStatus =
- assert.commandWorked(primary.adminCommand({replSetGetStatus: 1}));
- const primaryRecoveryTimestamp =
- primaryReplSetStatus.lastStableRecoveryTimestamp;
- const primaryDurableTimestamp = primaryReplSetStatus.optimes.durableOpTime.ts;
- const secondaryReplSetStatus =
- assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
- const secondaryRecoveryTimestamp =
- secondaryReplSetStatus.lastStableRecoveryTimestamp;
- const secondaryDurableTimestamp =
- secondaryReplSetStatus.optimes.durableOpTime.ts;
- jsTestLog(
- "Awaiting durable & last stable recovery timestamp " +
- `(primary last stable recovery: ${tojson(primaryRecoveryTimestamp)}, ` +
- `primary durable: ${tojson(primaryDurableTimestamp)}, ` +
- `secondary last stable recovery: ${tojson(secondaryRecoveryTimestamp)}, ` +
- `secondary durable: ${tojson(secondaryDurableTimestamp)}) ` +
- `target: ${tojson(timestamp)}`);
- return ((timestampCmp(primaryRecoveryTimestamp, timestamp) >= 0) &&
- (timestampCmp(primaryDurableTimestamp, timestamp) >= 0) &&
- (timestampCmp(secondaryDurableTimestamp, timestamp) >= 0) &&
- (timestampCmp(secondaryRecoveryTimestamp, timestamp) >= 0));
- },
- "Timeout waiting for checkpointing to catch up",
- ReplSetTest.kDefaultTimeoutMS,
- 2000);
- };
-
- // Wait for checkpointing/stable timestamp to catch up with the second insert so oplog
- // entry of the first insert is allowed to be deleted by the oplog cap maintainer thread
- // when a new oplog stone is created. "inMemory" WT engine does not run checkpoint
- // thread and lastStableRecoveryTimestamp is the stable timestamp in this case.
- awaitCheckpointer(secondInsertTimestamp);
-
- // Insert the third document which will trigger a new oplog stone to be created. The
- // oplog cap maintainer thread will then be unblocked on the creation of the new oplog
- // stone and will start truncating oplog entries. The oplog entry for the first
- // insert will be truncated after the oplog cap maintainer thread finishes.
- const thirdInsertTimestamp =
- assert
- .commandWorked(coll.runCommand(
- "insert",
- {documents: [{_id: 2, longString: longString}], writeConcern: {w: 2}}))
- .operationTime;
- jsTestLog("Third insert timestamp: " + tojson(thirdInsertTimestamp));
-
- // There is a race between how we calculate the pinnedOplog and checkpointing. The timestamp
- // of the pinnedOplog could be less than the actual stable timestamp used in a checkpoint.
- // Wait for the checkpointer to run for another round to make sure the first insert oplog is
- // not pinned.
- awaitCheckpointer(thirdInsertTimestamp);
-
- // Verify that there are three oplog entries while the oplog cap maintainer thread is
- // paused.
- assert.eq(3, numInsertOplogEntry(primaryOplog));
- assert.eq(3, numInsertOplogEntry(secondaryOplog));
-
- // Let the oplog cap maintainer thread start truncating the oplog.
- assert.commandWorked(primary.adminCommand(
- {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
- assert.commandWorked(secondary.adminCommand(
- {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
-
- // Test that oplog entry of the initial insert rolls over on both primary and secondary.
- // Use assert.soon to wait for oplog cap maintainer thread to run.
- assert.soon(() => {
- return numInsertOplogEntry(primaryOplog) === 2;
- }, "Timeout waiting for oplog to roll over on primary");
- assert.soon(() => {
- return numInsertOplogEntry(secondaryOplog) === 2;
- }, "Timeout waiting for oplog to roll over on secondary");
-
- const res = primary.getDB("test").runCommand({serverStatus: 1});
- assert.commandWorked(res);
- assert.eq(res.oplogTruncation.truncateCount, 1, tojson(res.oplogTruncation));
- assert.gt(res.oplogTruncation.totalTimeTruncatingMicros, 0, tojson(res.oplogTruncation));
- } else {
- // Let the oplog cap maintainer thread start truncating the oplog.
- assert.commandWorked(primary.adminCommand(
- {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
- assert.commandWorked(secondary.adminCommand(
- {configureFailPoint: "hangOplogCapMaintainerThread", mode: "off"}));
-
- // Only test that oplog truncation will eventually happen.
- let numInserted = 2;
- assert.soon(function() {
- // Insert more documents.
- assert.commandWorked(
- coll.insert({_id: numInserted++, longString: longString}, {writeConcern: {w: 2}}));
- const numInsertOplogEntryPrimary = numInsertOplogEntry(primaryOplog);
- const numInsertOplogEntrySecondary = numInsertOplogEntry(secondaryOplog);
- // Oplog has been truncated if the number of insert oplog entries is less than
- // number of inserted.
- if (numInsertOplogEntryPrimary < numInserted &&
- numInsertOplogEntrySecondary < numInserted)
- return true;
- jsTestLog("Awaiting oplog truncation: number of oplog entries: " +
- `(primary: ${tojson(numInsertOplogEntryPrimary)}, ` +
- `secondary: ${tojson(numInsertOplogEntrySecondary)}) ` +
- `number inserted: ${numInserted}`);
- return false;
- }, "Timeout waiting for oplog to roll over", ReplSetTest.kDefaultTimeoutMS, 1000);
- }
-
- replSet.stopSet();
-}
diff --git a/jstests/replsets/libs/rollback_resumable_index_build.js b/jstests/replsets/libs/rollback_resumable_index_build.js
index 2bd3c28e99d..5420b45ddb0 100644
--- a/jstests/replsets/libs/rollback_resumable_index_build.js
+++ b/jstests/replsets/libs/rollback_resumable_index_build.js
@@ -79,10 +79,8 @@ const RollbackResumableIndexBuildTest = class {
rollbackTest.awaitLastOpCommitted();
- assert.commandWorked(originalPrimary.adminCommand({
- setParameter: 1,
- logComponentVerbosity: {index: 1, replication: {election: 0, heartbeats: 0}},
- }));
+ assert.commandWorked(originalPrimary.adminCommand(
+ {setParameter: 1, logComponentVerbosity: {index: 1, replication: {heartbeats: 0}}}));
// Set internalQueryExecYieldIterations to 0, internalIndexBuildBulkLoadYieldIterations to
// 1, and maxIndexBuildDrainBatchSize to 1 so that the index builds are guaranteed to yield
diff --git a/jstests/replsets/libs/rollback_test.js b/jstests/replsets/libs/rollback_test.js
index d6838deffcf..5e70cd96614 100644
--- a/jstests/replsets/libs/rollback_test.js
+++ b/jstests/replsets/libs/rollback_test.js
@@ -294,10 +294,6 @@ function RollbackTest(name = "RollbackTest", replSet) {
return rst.getPrimary(ReplSetTest.kDefaultTimeoutMS, kRetryIntervalMS);
}
- this.stepUpNode = function(conn) {
- stepUp(conn);
- };
-
function oplogTop(conn) {
return conn.getDB("local").oplog.rs.find().limit(1).sort({$natural: -1}).next();
}
@@ -543,11 +539,11 @@ function RollbackTest(name = "RollbackTest", replSet) {
return curPrimary;
};
- this.stop = function(checkDataConsistencyOptions, skipDataConsistencyCheck = false) {
+ this.stop = function(checkDataConsistencyOptions) {
const start = new Date();
restartServerReplication(tiebreakerNode);
rst.awaitReplication();
- if (!doneConsistencyChecks && !skipDataConsistencyCheck) {
+ if (!doneConsistencyChecks) {
this.checkDataConsistency(checkDataConsistencyOptions);
}
transitionIfAllowed(State.kStopped);
diff --git a/jstests/replsets/libs/secondary_reads_test.js b/jstests/replsets/libs/secondary_reads_test.js
index 9c91c871b84..4840708dba2 100644
--- a/jstests/replsets/libs/secondary_reads_test.js
+++ b/jstests/replsets/libs/secondary_reads_test.js
@@ -99,8 +99,8 @@ function SecondaryReadsTest(name = "secondary_reads_test") {
assert.gt(readers.length, 0, "no readers to stop");
assert.commandWorked(primaryDB.getCollection(signalColl).insert({_id: testDoneId}));
for (let i = 0; i < readers.length; i++) {
- const awaitReader = readers[i];
- awaitReader();
+ const await = readers[i];
+ await ();
print("reader " + i + " done");
}
readers = [];
diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js
index b12512b2b8d..b7b81e3f01d 100644
--- a/jstests/replsets/libs/tenant_migration_test.js
+++ b/jstests/replsets/libs/tenant_migration_test.js
@@ -38,7 +38,6 @@ function TenantMigrationTest({
initiateRstWithHighElectionTimeout = true,
quickGarbageCollection = false,
insertDataForTenant,
- optimizeMigrations = true,
}) {
const donorPassedIn = (donorRst !== undefined);
const recipientPassedIn = (recipientRst !== undefined);
@@ -48,15 +47,9 @@ function TenantMigrationTest({
const nodes = sharedOptions.nodes || 2;
const setParameterOpts = sharedOptions.setParameter || {};
- if (optimizeMigrations) {
- // A tenant migration recipient's `OplogFetcher` uses aggregation which does not support
- // tailable awaitdata cursors. For aggregation commands `OplogFetcher` will default to half
- // the election timeout (e.g: 5 seconds) between getMores. That wait is largely unnecessary.
- setParameterOpts["failpoint.setSmallOplogGetMoreMaxTimeMS"] = tojson({"mode": "alwaysOn"});
- }
if (quickGarbageCollection) {
- setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 0;
- setParameterOpts.ttlMonitorSleepSecs = 1;
+ setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 3 * 1000;
+ setParameterOpts.ttlMonitorSleepSecs = 3;
}
donorRst = donorPassedIn ? donorRst : performSetUp(true /* isDonor */);