diff options
Diffstat (limited to 'jstests/disk')
| -rw-r--r-- | jstests/disk/directoryperdb.js | 8 | ||||
| -rw-r--r-- | jstests/disk/repair_clustered_collection.js | 87 | ||||
| -rw-r--r-- | jstests/disk/wt_size_storer_cleanup_replica_set.js | 78 | ||||
| -rw-r--r-- | jstests/disk/wt_size_storer_cleanup_standalone.js | 70 | ||||
| -rw-r--r-- | jstests/disk/wt_table_checks.js | 164 | ||||
| -rw-r--r-- | jstests/disk/wt_table_checks_read_only.js | 51 |
6 files changed, 135 insertions, 323 deletions
diff --git a/jstests/disk/directoryperdb.js b/jstests/disk/directoryperdb.js index 84183eae7df..49c2c89bb0f 100644 --- a/jstests/disk/directoryperdb.js +++ b/jstests/disk/directoryperdb.js @@ -19,11 +19,6 @@ assertDocumentCount = function(db, count) { * deleted. MongoDB does not always delete data immediately with a catalog change. */ const waitForDatabaseDirectoryRemoval = function(dbName, dbDirPath) { - // The periodic task to drop data tables for two-phase commit runs once per second and - // in standalone mode, without timestamps, can execute table drops immediately. It should - // only take a couple seconds for the periodic task to start removing any data tables. - // However, slow disk access may delay the actual removal of the directory. - // Therefore, we should be conservative and use the default timeout in assert.soon(). assert.soon( function() { const files = listFiles(dbDirPath).filter(function(path) { @@ -36,7 +31,8 @@ const waitForDatabaseDirectoryRemoval = function(dbName, dbDirPath) { } }, "dbpath contained '" + dbName + - "' directory when it should have been removed: " + tojson(listFiles(dbDirPath))); + "' directory when it should have been removed: " + tojson(listFiles(dbDirPath)), + 10 * 1000); // The periodic task to run data table cleanup runs once a second. }; /** diff --git a/jstests/disk/repair_clustered_collection.js b/jstests/disk/repair_clustered_collection.js deleted file mode 100644 index 001ac2dbeb2..00000000000 --- a/jstests/disk/repair_clustered_collection.js +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Tests that --repair on WiredTiger correctly and gracefully handles a missing _mdb_catalog when - * a clustered collection exists on the server instance. - * - * @tags: [requires_wiredtiger] - */ -(function() { - -load('jstests/disk/libs/wt_file_helper.js'); -load("jstests/libs/collection_drop_recreate.js"); - -const dbName = jsTestName(); -const collName = "test"; -const dbpath = MongoRunner.dataPath + dbName + "/"; - -const runRepairTest = function runRepairTestOnMongoDInstance( - collectionOptions, docToInsert, isTimeseries) { - let mongod = startMongodOnExistingPath(dbpath); - let db = mongod.getDB(dbName); - - assertDropCollection(db, collName); - assertCreateCollection(db, collName, collectionOptions); - - let testColl = db[collName]; - let testCollUri = getUriForColl(testColl); - let testCollFile = dbpath + testCollUri + ".wt"; - - assert.commandWorked(testColl.insert(docToInsert)); - - // A document repaired from a timeseries collection will be in a different format than the - // original document. This is because the timeseries's system.views collection will be not be - // associated with the orphaned clustered collection. Thus, the data will show up as it would - // have in the raw system.buckets collection for the timeseries collection. - const expectedOrphanDoc = - isTimeseries ? db["system.buckets." + collName].findOne() : testColl.findOne(); - - MongoRunner.stopMongod(mongod); - - // Delete the _mdb_catalog. - let mdbCatalogFile = dbpath + "_mdb_catalog.wt"; - jsTestLog("deleting catalog file: " + mdbCatalogFile); - removeFile(mdbCatalogFile); - - assertRepairSucceeds(dbpath, mongod.port); - - // Verify that repair succeeds in creating an empty catalog and MongoDB starts up normally with - // no data. - mongod = startMongodOnExistingPath(dbpath); - db = mongod.getDB(dbName); - testColl = db[collName]; - assert.isnull(testColl.exists()); - assert.eq(testColl.find(docToInsert).itcount(), 0); - assert.eq(testColl.count(), 0); - - // Ensure the orphaned collection is valid and the document is preserved. - const orphanedImportantCollName = "orphan." + testCollUri.replace(/-/g, "_"); - const localDb = mongod.getDB("local"); - orphanedCollection = localDb[orphanedImportantCollName]; - assert(orphanedCollection.exists()); - assert.eq(orphanedCollection.count(expectedOrphanDoc), - 1, - `Expected to find document ${tojson(expectedOrphanDoc)} but collection has contents ${ - tojson(orphanedCollection.find().toArray())}`); - - const validateResult = orphanedCollection.validate(); - assert(validateResult.valid); - MongoRunner.stopMongod(mongod); -}; - -// Standard clustered collection test. -let isTimeseries = false; -let clusteredCollOptions = {clusteredIndex: {key: {_id: 1}, unique: true}}; -let docToInsert = {_id: 1}; -runRepairTest(clusteredCollOptions, docToInsert, isTimeseries); - -// Timeseries test since all timeseries collections are implicitly clustered. -isTimeseries = true; -clusteredCollOptions = { - timeseries: {timeField: "timestamp", metaField: "metadata", granularity: "hours"} -}; -docToInsert = { - "metadata": {"sensorId": 5578, "type": "temperature"}, - "timestamp": ISODate("2021-05-18T00:00:00.000Z"), - "temp": 12 -}; -runRepairTest(clusteredCollOptions, docToInsert, isTimeseries); -})(); diff --git a/jstests/disk/wt_size_storer_cleanup_replica_set.js b/jstests/disk/wt_size_storer_cleanup_replica_set.js deleted file mode 100644 index 624d29f9a13..00000000000 --- a/jstests/disk/wt_size_storer_cleanup_replica_set.js +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Tests that the size storer entry for a collection gets cleaned up when that collection is - * dropped. - * - * @tags: [ - * requires_replication, - * requires_wiredtiger, - * ] - */ -(function() { -"use strict"; - -load("jstests/disk/libs/wt_file_helper.js"); - -const replTest = new ReplSetTest({nodes: 1}); -replTest.startSet(); -replTest.initiate(); - -let primary = replTest.getPrimary(); -const dbpath = primary.dbpath; - -const coll = function() { - return primary.getDB(jsTestName()).test; -}; - -// TODO (SERVER-82902): Use JSON-formatted size storer data. -// const getSizeStorerData = function() { -// const filePath = dbpath + (_isWindows() ? "\\" : "/") + jsTestName(); -// runWiredTigerTool("-r", "-h", dbpath, "dump", "-j", "-f", filePath, "sizeStorer"); -// return JSON.parse(cat(filePath))["table:sizeStorer"][1].data; -// }; -const getSizeStorerData = function() { - const filePath = dbpath + (_isWindows() ? "\\" : "/") + jsTestName(); - runWiredTigerTool("-r", "-h", dbpath, "dump", "-f", filePath, "sizeStorer"); - return cat(filePath); -}; - -assert.commandWorked(coll().insert({a: 1})); -assert.eq(coll().count(), 1); -const uri = coll().stats().wiredTiger.uri.split("statistics:")[1]; - -replTest.stop(primary, undefined, {}, {forRestart: true}); - -let sizeStorerData = getSizeStorerData(); -// TODO (SERVER-82902): Use JSON-formatted size storer data. -// assert(sizeStorerData.find(entry => entry.key0 === uri), -// "Size storer unexpectedly does not contain entry for " + uri + ": " + -// tojson(sizeStorerData)); -assert(sizeStorerData.includes(uri), - "Size storer unexpectedly does not contain entry for " + uri + ": " + sizeStorerData); - -replTest.start( - primary, {setParameter: {minSnapshotHistoryWindowInSeconds: 0}}, true /* forRestart */); -primary = replTest.getPrimary(); - -const collIdent = getUriForColl(coll()); -const indexIdent = getUriForIndex(coll(), "_id_"); - -assert.eq(coll().count(), 1); -assert(coll().drop()); -assert.commandWorked(primary.adminCommand({appendOplogNote: 1, data: {msg: "advance timestamp"}})); -assert.commandWorked(primary.adminCommand({fsync: 1})); - -checkLog.containsJson(primary, 22237, {ident: collIdent}); -checkLog.containsJson(primary, 22237, {ident: indexIdent}); - -replTest.stop(primary, undefined, {}, {forRestart: true}); - -sizeStorerData = getSizeStorerData(); -// TODO (SERVER-82902): Use JSON-formatted size storer data. -// assert(!sizeStorerData.find(entry => entry.key0 === uri), -// "Size storer unexpectedly contains entry for " + uri + ": " + tojson(sizeStorerData)); -assert(!sizeStorerData.includes(uri), - "Size storer unexpectedly contains entry for " + uri + ": " + sizeStorerData); - -replTest.start(primary, {}, true /* forRestart */); -replTest.stopSet(); -})(); diff --git a/jstests/disk/wt_size_storer_cleanup_standalone.js b/jstests/disk/wt_size_storer_cleanup_standalone.js deleted file mode 100644 index 9c3aff6e8b9..00000000000 --- a/jstests/disk/wt_size_storer_cleanup_standalone.js +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Tests that the size storer entry for a collection gets cleaned up when that collection is - * dropped. - * - * @tags: [ - * requires_wiredtiger, - * ] - */ -(function() { -"use strict"; - -load("jstests/disk/libs/wt_file_helper.js"); - -const runTest = function(insertAfterRestart) { - let conn = MongoRunner.runMongod(); - const dbpath = conn.dbpath; - - const coll = function() { - return conn.getDB(jsTestName()).test; - }; - - // TODO (SERVER-82902): Use JSON-formatted size storer data. - // const getSizeStorerData = function() { - // const filePath = dbpath + (_isWindows() ? "\\" : "/") + jsTestName(); - // runWiredTigerTool("-r", "-h", dbpath, "dump", "-j", "-f", filePath, "sizeStorer"); - // return JSON.parse(cat(filePath))["table:sizeStorer"][1].data; - // }; - const getSizeStorerData = function() { - const filePath = dbpath + (_isWindows() ? "\\" : "/") + jsTestName(); - runWiredTigerTool("-r", "-h", dbpath, "dump", "-f", filePath, "sizeStorer"); - return cat(filePath); - }; - - assert.commandWorked(coll().insert({a: 1})); - assert.eq(coll().count(), 1); - const uri = coll().stats().wiredTiger.uri.split("statistics:")[1]; - - MongoRunner.stopMongod(conn); - - let sizeStorerData = getSizeStorerData(); - // TODO (SERVER-82902): Use JSON-formatted size storer data. - // assert(sizeStorerData.find(entry => entry.key0 === uri), - // "Size storer unexpectedly does not contain entry for " + uri + ": " + - // tojson(sizeStorerData)); - assert(sizeStorerData.includes(uri), - "Size storer unexpectedly does not contain entry for " + uri + ": " + sizeStorerData); - - conn = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true, setParameter: {syncdelay: 0}}); - - if (insertAfterRestart) { - assert.commandWorked(coll().insert({a: 2})); - } - assert.eq(coll().count(), insertAfterRestart ? 2 : 1); - assert(coll().drop()); - assert.commandWorked(conn.adminCommand({setParameter: 1, syncdelay: 1})); - checkLog.containsJson(conn, 6776600, {ident: uri.split("table:")[1]}); - - MongoRunner.stopMongod(conn); - - sizeStorerData = getSizeStorerData(); - // TODO (SERVER-82902): Use JSON-formatted size storer data. - // assert(!sizeStorerData.find(entry => entry.key0 === uri), - // "Size storer unexpectedly contains entry for " + uri + ": " + tojson(sizeStorerData)); - assert(!sizeStorerData.includes(uri), - "Size storer unexpectedly contains entry for " + uri + ": " + sizeStorerData); -}; - -runTest(false); -runTest(true); -})(); diff --git a/jstests/disk/wt_table_checks.js b/jstests/disk/wt_table_checks.js index 5fb9c0c9fb8..47a16e02f7a 100644 --- a/jstests/disk/wt_table_checks.js +++ b/jstests/disk/wt_table_checks.js @@ -8,43 +8,6 @@ load('jstests/disk/libs/wt_file_helper.js'); -function checkTableLogSettings(conn, enabled) { - conn.getDBNames().forEach(function(d) { - let collNames = - conn.getDB(d) - .runCommand({listCollections: 1, nameOnly: true, filter: {type: "collection"}}) - .cursor.firstBatch; - - collNames.forEach(function(c) { - let stats = conn.getDB(d).runCommand({collStats: c.name}); - - let logStr = "log=(enabled=" + (enabled ? "true" : "false") + ")"; - if (d == "local") { - if (c.name == "replset.minvalid" && !enabled) { - // This collection is never logged in a replica set. - logStr = "log=(enabled=false)"; - } else { - // All other collections and indexes in the 'local' database have table - // logging enabled always. - logStr = "log=(enabled=true)"; - } - } - - assert.eq(true, stats.wiredTiger.creationString.includes(logStr)); - Object.keys(stats.indexDetails).forEach(function(i) { - assert.eq(true, stats.indexDetails[i].creationString.includes(logStr)); - }); - }); - }); -} - -function checkTableChecksFileRemoved(dbpath) { - let files = listFiles(dbpath); - for (file of files) { - assert.eq(false, file.name.includes("_wt_table_checks")); - } -} - // Create a bunch of collections under various database names. let conn = MongoRunner.runMongod({}); const dbpath = conn.dbpath; @@ -53,75 +16,112 @@ for (let i = 0; i < 10; i++) { assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); } -checkTableLogSettings(conn, /*enabled=*/true); MongoRunner.stopMongod(conn); /** - * Test 1. Change into a single node replica set, which requires all of the table logging settings - * to be updated. Write the '_wt_table_checks' file and check that it gets removed. + * Test 1. The regular case, where no table logging setting modifications are needed. */ jsTest.log("Test 1."); -writeFile(dbpath + "/_wt_table_checks", ""); -conn = startMongodOnExistingPath( - dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); -checkTableChecksFileRemoved(dbpath); - -// Changing table logging settings. -checkLog.containsJson(conn, 22432); +conn = startMongodOnExistingPath(dbpath, {}); +checkLog.containsJson(conn, 4366408, {loggingEnabled: true}); MongoRunner.stopMongod(conn); /** - * Test 2. Restart in standalone mode with wiredTigerSkipTableLoggingChecksOnStartup. No table log - * settings are updated. Write the '_wt_table_checks' file and check that it gets removed. + * Test 2. Repair checks all of the table logging settings. */ jsTest.log("Test 2."); -writeFile(dbpath + "/_wt_table_checks", ""); -conn = startMongodOnExistingPath(dbpath, { - setParameter: { - wiredTigerSkipTableLoggingChecksOnStartup: true, - logComponentVerbosity: tojson({verbosity: 1}) - } -}); -checkTableChecksFileRemoved(dbpath); -// Skipping table logging checks. -checkLog.containsJson(conn, 5548302); +assertRepairSucceeds(dbpath, conn.port, {}); -// Changing table logging settings. -assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0)); -checkTableLogSettings(conn, /*enabled=*/false); -MongoRunner.stopMongod(conn); +// Cannot use checkLog here as the server is no longer running. +let logContents = rawMongoProgramOutput(); +assert(logContents.indexOf( + "Modifying the table logging settings for all existing WiredTiger tables") > 0); /** - * Test 3. Change into a single node replica set again. Table log settings are checked but none are - * changed. Write the '_wt_table_checks' file and check that it gets removed. + * Test 3. Explicitly create the '_wt_table_checks' file to force all of the table logging setting + * modifications to be made. */ -jsTestLog("Test 3."); +jsTest.log("Test 3."); + +let files = listFiles(dbpath); +for (f in files) { + assert(!files[f].name.includes("_wt_table_checks")); +} + writeFile(dbpath + "/_wt_table_checks", ""); -conn = startMongodOnExistingPath( - dbpath, {replSet: "mySet", setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); -checkTableChecksFileRemoved(dbpath); -// Changing table logging settings. -assert(checkLog.checkContainsWithCountJson(conn, 22432, undefined, 0)); +// Cannot skip table logging checks on startup when there are previously incomplete table checks. +assert.throws(() => startMongodOnExistingPath( + dbpath, {setParameter: "wiredTigerSkipTableLoggingChecksOnStartup=true"})); + +conn = startMongodOnExistingPath(dbpath, {}); +checkLog.containsJson( + conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true}); MongoRunner.stopMongod(conn); /** - * Test 4. Back to standalone. Check that the table log settings are enabled. Write the - * '_wt_table_checks' file and check that it gets removed. + * Test 4. Change into a single replica set, which requires all of the table logging settings to be + * updated. But simulate an interruption/crash while starting up during the table logging check + * phase. + * + * The next start up will detect an unclean shutdown causing all of the table logging settings to be + * updated. */ jsTest.log("Test 4."); -writeFile(dbpath + "/_wt_table_checks", ""); -conn = startMongodOnExistingPath(dbpath, - {setParameter: {logComponentVerbosity: tojson({verbosity: 1})}}); -checkTableChecksFileRemoved(dbpath); -// Changing table logging settings. -checkLog.containsJson(conn, 22432); +assert.throws(() => startMongodOnExistingPath(dbpath, { + replSet: "mySet", + setParameter: "failpoint.crashAfterUpdatingFirstTableLoggingSettings=" + + tojson({"mode": "alwaysOn"}) + })); + +// Cannot use checkLog here as the server is no longer running. +logContents = rawMongoProgramOutput(); +assert(logContents.indexOf( + "Crashing due to 'crashAfterUpdatingFirstTableLoggingSettings' fail point") > 0); + +// The '_wt_table_checks' still exists, so all table logging settings should be modified. +conn = startMongodOnExistingPath(dbpath, {}); +checkLog.containsJson( + conn, 4366405, {loggingEnabled: true, repair: false, hasPreviouslyIncompleteTableChecks: true}); +MongoRunner.stopMongod(conn); + +/** + * Test 5. Change into a single node replica set, which requires all of the table logging settings + * to be updated as the node was successfully started up as a standalone the last time. + */ +jsTest.log("Test 5."); + +conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); +checkLog.containsJson(conn, 4366406, {loggingEnabled: false}); +MongoRunner.stopMongod(conn); + +/** + * Test 6. Restart as a standalone and skip table logging checks on startup. Verify that restarting + * as a replica set again does not require any table logging modifications. + */ +jsTest.log("Test 6."); + +conn = startMongodOnExistingPath(dbpath, { + setParameter: { + wiredTigerSkipTableLoggingChecksOnStartup: true, + logComponentVerbosity: tojson({verbosity: 1}) + } +}); + +// Skipping table logging checks for all existing tables. +checkLog.containsJson(conn, 5548301, {wiredTigerSkipTableLoggingChecksOnStartup: true}); + +// Log level 1 prints each individual table it skips table logging checks for. +checkLog.containsJson(conn, 5548302); + +MongoRunner.stopMongod(conn); + +conn = startMongodOnExistingPath(dbpath, {replSet: "mySet"}); -// Skipping table logging checks. -assert(checkLog.checkContainsWithCountJson(conn, 5548302, undefined, 0)); -checkTableLogSettings(conn, /*enabled=*/true); +// No table logging settings modifications are required. +checkLog.containsJson(conn, 4366408); MongoRunner.stopMongod(conn); }()); diff --git a/jstests/disk/wt_table_checks_read_only.js b/jstests/disk/wt_table_checks_read_only.js new file mode 100644 index 00000000000..6d6519f2c85 --- /dev/null +++ b/jstests/disk/wt_table_checks_read_only.js @@ -0,0 +1,51 @@ +/** + * Tests that the table logging settings are not changed during read only mode. + * + * @tags: [requires_wiredtiger] + */ +(function() { + +load('jstests/disk/libs/wt_file_helper.js'); + +// Create a bunch of collections under various database names. +let conn = MongoRunner.runMongod({}); +const dbpath = conn.dbpath; + +for (let i = 0; i < 10; i++) { + assert.commandWorked(conn.getDB(i.toString()).createCollection(i.toString())); +} + +MongoRunner.stopMongod(conn); + +// Option for read only mode. +let options = {queryableBackupMode: ""}; + +// Verifies that setTableLogging() does not get called in read only mode, otherwise the invariant +// would fire. +conn = startMongodOnExistingPath(dbpath, options); +assert(conn); +MongoRunner.stopMongod(conn); + +// Create the '_wt_table_checks' file in the dbpath and ensure it doesn't get removed while in read +// only mode. +let files = listFiles(dbpath); +for (f in files) { + assert(!files[f].name.includes("_wt_table_checks")); +} + +writeFile(dbpath + "/_wt_table_checks", ""); + +conn = startMongodOnExistingPath(dbpath, options); +assert(conn); +MongoRunner.stopMongod(conn); + +let hasWTTableChecksFile = false; +files = listFiles(dbpath); +for (f in files) { + if (files[f].name.includes("_wt_table_checks")) { + hasWTTableChecksFile = true; + } +} + +assert(hasWTTableChecksFile); +}()); |