diff options
| author | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-03-22 12:04:55 +0200 |
|---|---|---|
| committer | Apollon Oikonomopoulos <apoikos@debian.org> | 2018-03-22 12:04:55 +0200 |
| commit | c49e99631589113663b1a3ac691870421965a315 (patch) | |
| tree | ac8127a5a6816f169a6656511bf131a35af2f74a /jstests | |
| parent | d982a88efa79f510c03f1c6c8c63360680ebbf88 (diff) | |
New upstream version 3.4.14upstream/3.4.14
Diffstat (limited to 'jstests')
156 files changed, 3598 insertions, 733 deletions
diff --git a/jstests/aggregation/extras/utils.js b/jstests/aggregation/extras/utils.js index 2dd1038388c..68b2597c533 100644 --- a/jstests/aggregation/extras/utils.js +++ b/jstests/aggregation/extras/utils.js @@ -276,7 +276,7 @@ function assertErrorCode(coll, pipe, code, errmsg) { var cursorRes = coll.runCommand("aggregate", cmd); if (cursorRes.ok) { var followupBatchSize = 0; // default - var cursor = new DBCommandCursor(coll.getMongo(), cursorRes, followupBatchSize); + var cursor = new DBCommandCursor(cursorRes._mongo, cursorRes, followupBatchSize); var error = assert.throws(function() { cursor.itcount(); diff --git a/jstests/auth/mongos_cache_invalidation.js b/jstests/auth/mongos_cache_invalidation.js index e36445cac5e..b53656de3a1 100644 --- a/jstests/auth/mongos_cache_invalidation.js +++ b/jstests/auth/mongos_cache_invalidation.js @@ -61,7 +61,7 @@ db3.auth('spencer', 'pwd'); * At this point we have 3 handles to the "test" database, each of which are on connections to * different mongoses. "db1", "db2", and "db3" are all auth'd as spencer@test and will be used * to verify that user and role data changes get propaged to their mongoses. - * "db2" is connected to a mongos with a 10 second user cache invalidation interval, + * "db2" is connected to a mongos with a 5 second user cache invalidation interval, * while "db3" is connected to a mongos with a 10 minute cache invalidation interval. */ @@ -202,12 +202,9 @@ db3.auth('spencer', 'pwd'); assert.commandFailedWithCode(db1.foo.runCommand("collStats"), authzErrorCode); // s1/db2 should update its cache in 10 seconds. - assert.soon( - function() { - return db2.foo.runCommand("collStats").code == authzErrorCode; - }, - "Mongos did not update its user cache after 10 seconds", - 6 * 1000); // Give an extra 1 second to avoid races + assert.soon(function() { + return db2.foo.runCommand("collStats").code == authzErrorCode; + }, "Mongos did not update its user cache after 10 seconds", 10 * 1000); // We manually invalidate the cache on s2/db3. db3.adminCommand("invalidateUserCache"); diff --git a/jstests/auth/scram-credentials-invalid.js b/jstests/auth/scram-credentials-invalid.js new file mode 100644 index 00000000000..16c0c204d12 --- /dev/null +++ b/jstests/auth/scram-credentials-invalid.js @@ -0,0 +1,45 @@ +// Ensure that attempting to use SCRAM-SHA-1 auth on a +// user with invalid SCRAM-SHA-1 credentials fails gracefully. + +(function() { + 'use strict'; + + function runTest(mongod) { + assert(mongod); + const admin = mongod.getDB('admin'); + const test = mongod.getDB('test'); + + admin.createUser({user: 'admin', pwd: 'pass', roles: jsTest.adminUserRoles}); + assert(admin.auth('admin', 'pass')); + + test.createUser({user: 'user', pwd: 'pass', roles: jsTest.basicUserRoles}); + + // Give the test user an invalid set of SCRAM-SHA-1 credentials. + assert.eq(admin.system.users + .update({_id: "test.user"}, { + $set: { + "credentials.SCRAM-SHA-1": { + salt: "AAAA", + storedKey: "AAAA", + serverKey: "AAAA", + iterationCount: 10000 + } + } + }) + .nModified, + 1, + "Should have updated one document for user@test"); + admin.logout(); + + assert(!test.auth({user: 'user', pwd: 'pass'})); + + assert.soon(function() { + const log = cat(mongod.fullOptions.logFile); + return /Unable to perform SCRAM-SHA-1 auth.* invalid SCRAM credentials/.test(log); + }, "No warning issued for invalid SCRAM-SHA-1 credendials doc", 30 * 1000, 5 * 1000); + } + + const mongod = MongoRunner.runMongod({auth: "", useLogFiles: true}); + runTest(mongod); + MongoRunner.stopMongod(mongod); +})(); diff --git a/jstests/auth/system_authorization_indexes.js b/jstests/auth/system_authorization_indexes.js new file mode 100644 index 00000000000..496f78578a4 --- /dev/null +++ b/jstests/auth/system_authorization_indexes.js @@ -0,0 +1,66 @@ +/** Ensure that authorization system collections' indexes are correctly generated. + * + * This test requires users to persist across a restart. + * @tags: [requires_persistence] + */ + +(function() { + let conn = MongoRunner.runMongod({smallfiles: ""}); + let db = conn.getDB("admin"); + + // TEST: User and role collections start off with no indexes + assert.eq(0, db.system.users.getIndexes().length); + assert.eq(0, db.system.roles.getIndexes().length); + + // TEST: User and role creation generates indexes + db.createUser({user: "user", pwd: "pwd", roles: []}); + assert.eq(2, db.system.users.getIndexes().length); + + db.createRole({role: "role", privileges: [], roles: []}); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying admin.system.users index and restarting will recreate it + assert.commandWorked(db.system.users.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying admin.system.roles index and restarting will recreate it + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying both authorization indexes and restarting will recreate them + assert.commandWorked(db.system.users.dropIndexes()); + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + assert.eq(2, db.system.roles.getIndexes().length); + + // TEST: Destroying the admin.system.users index and restarting will recreate it, even if + // admin.system.roles does not exist + db.dropDatabase(); + db.createUser({user: "user", pwd: "pwd", roles: []}); + assert.commandWorked(db.system.users.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.users.getIndexes().length); + + // TEST: Destroying the admin.system.roles index and restarting will recreate it, even if + // admin.system.users does not exist + db.dropDatabase(); + db.createRole({role: "role", privileges: [], roles: []}); + assert.commandWorked(db.system.roles.dropIndexes()); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({restart: conn, cleanData: false}); + db = conn.getDB("admin"); + assert.eq(2, db.system.roles.getIndexes().length); +})(); diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js index a5705a409af..f3e2429f898 100644 --- a/jstests/concurrency/fsm_libs/cluster.js +++ b/jstests/concurrency/fsm_libs/cluster.js @@ -573,6 +573,25 @@ var Cluster = function(options) { return data; }; + + this.isRunningWiredTigerLSM = function isRunningWiredTigerLSM() { + var adminDB = this.getDB('admin'); + + if (this.isSharded()) { + // Get the storage engine the sharded cluster is configured to use from one of the + // shards since mongos won't report it. + adminDB = st.shard0.getDB('admin'); + } + + var res = adminDB.runCommand({getCmdLineOpts: 1}); + assert.commandWorked(res, 'failed to get command line options'); + + var wiredTigerOptions = res.parsed.storage.wiredTiger || {}; + var wiredTigerCollectionConfig = wiredTigerOptions.collectionConfig || {}; + var wiredTigerConfigString = wiredTigerCollectionConfig.configString || ''; + + return wiredTigerConfigString === 'type=lsm'; + }; }; /** diff --git a/jstests/concurrency/fsm_workloads/compact.js b/jstests/concurrency/fsm_workloads/compact.js index 8f91f52bf5e..e86e6ef555f 100644 --- a/jstests/concurrency/fsm_workloads/compact.js +++ b/jstests/concurrency/fsm_workloads/compact.js @@ -87,12 +87,27 @@ var $config = (function() { dropCollections(db, pattern); }; + var skip = function skip(cluster) { + if (cluster.isRunningWiredTigerLSM()) { + // There is a known hang during concurrent FSM workloads with the compact command used + // with wiredTiger LSM variants. Bypass this command for the wiredTiger LSM variant + // until a fix is available for WT-2523. + return { + skip: true, + msg: 'WT-2523: compact command can cause hang using WT LSM index during ' + + 'concurrent workloads' + }; + } + return {skip: false}; + }; + return { threadCount: 15, iterations: 10, states: states, transitions: transitions, teardown: teardown, - data: data + data: data, + skip: skip }; })(); diff --git a/jstests/concurrency/fsm_workloads/remove_multiple_documents.js b/jstests/concurrency/fsm_workloads/remove_multiple_documents.js index bfd64cd6790..1461ca0cc03 100644 --- a/jstests/concurrency/fsm_workloads/remove_multiple_documents.js +++ b/jstests/concurrency/fsm_workloads/remove_multiple_documents.js @@ -34,8 +34,17 @@ var $config = (function() { } }; + var skip = function skip(cluster) { + // When the balancer is enabled, the nRemoved result may be inaccurate as + // a chunk migration may be active, causing the count function to assert. + if (cluster.isBalancerEnabled()) { + return {skip: true, msg: 'does not run when balancer is enabled.'}; + } + return {skip: false}; + }; + var transitions = {init: {count: 1}, count: {remove: 1}, remove: {remove: 0.825, count: 0.125}}; - return {threadCount: 10, iterations: 20, states: states, transitions: transitions}; + return {threadCount: 10, iterations: 20, states: states, transitions: transitions, skip: skip}; })(); diff --git a/jstests/concurrency/fsm_workloads/remove_where.js b/jstests/concurrency/fsm_workloads/remove_where.js index f9c0e6a2c03..3ef214c769b 100644 --- a/jstests/concurrency/fsm_workloads/remove_where.js +++ b/jstests/concurrency/fsm_workloads/remove_where.js @@ -38,5 +38,14 @@ var $config = extendWorkload($config, function($config, $super) { /* no-op to prevent index from being created */ }; + $config.skip = function skip(cluster) { + // When the balancer is enabled, the nRemoved result may be inaccurate as + // a chunk migration may be active, causing the count function to assert. + if (cluster.isBalancerEnabled()) { + return {skip: true, msg: 'does not run when balancer is enabled.'}; + } + return {skip: false}; + }; + return $config; }); diff --git a/jstests/core/apply_ops1.js b/jstests/core/apply_ops1.js index 8ce15b1df22..7088145c718 100644 --- a/jstests/core/apply_ops1.js +++ b/jstests/core/apply_ops1.js @@ -286,9 +286,54 @@ 'applyOps should fail on unknown operation type "x" with valid "ns" value'); assert.eq(0, t.find().count(), "Non-zero amount of documents in collection to start"); + + /** + * Test function for running CRUD operations on non-existent namespaces using various + * combinations of invalid namespaces (collection/database), allowAtomic and alwaysUpsert. + * + * Leave 'expectedErrorCode' undefined if this command is expected to run successfully. + */ + function testCrudOperationOnNonExistentNamespace(optype, o, o2, expectedErrorCode) { + expectedErrorCode = expectedErrorCode || ErrorCodes.OK; + const t2 = db.getSiblingDB('apply_ops1_no_such_db').getCollection('t'); + [t, t2].forEach(coll => { + const op = {op: optype, ns: coll.getFullName(), o: o, o2: o2}; + [false, true].forEach(allowAtomic => { + [false, true].forEach(alwaysUpsert => { + const cmd = { + applyOps: [op], + allowAtomic: allowAtomic, + alwaysUpsert: alwaysUpsert + }; + jsTestLog('Testing applyOps on non-existent namespace: ' + tojson(cmd)); + if (expectedErrorCode === ErrorCodes.OK) { + assert.commandWorked(db.adminCommand(cmd)); + } else { + assert.commandFailedWithCode(db.adminCommand(cmd), expectedErrorCode); + } + }); + }); + }); + } + + // Insert and update operations on non-existent collections/databases should return + // NamespaceNotFound. + testCrudOperationOnNonExistentNamespace('i', {_id: 0}, {}, ErrorCodes.NamespaceNotFound); + testCrudOperationOnNonExistentNamespace('u', {x: 0}, {_id: 0}, ErrorCodes.NamespaceNotFound); + + // Delete operations on non-existent collections/databases should return OK for idempotency + // reasons. + testCrudOperationOnNonExistentNamespace('d', {_id: 0}, {}); + assert.commandFailed( - db.adminCommand({applyOps: [{"op": "i", "ns": t.getFullName(), "o": {_id: 5, x: 17}}]}), - "Applying an insert operation on a non-existent collection should fail"); + db.adminCommand({ + applyOps: [{ + "op": "c", + "ns": "admin.$cmd", + "o": {applyOps: [{"op": "i", "ns": t.getFullName(), "o": {_id: 5, x: 17}}]} + }] + }), + "Applying a nested insert operation on a non-existent collection should fail"); assert.commandWorked(db.createCollection(t.getName())); var a = assert.commandWorked( diff --git a/jstests/core/apply_ops_atomicity.js b/jstests/core/apply_ops_atomicity.js index 911ce32a311..e704815a889 100644 --- a/jstests/core/apply_ops_atomicity.js +++ b/jstests/core/apply_ops_atomicity.js @@ -29,10 +29,12 @@ var newDBName = "apply_ops_atomicity"; var newDB = db.getSiblingDB(newDBName); assert.commandWorked(newDB.dropDatabase()); - // Do an update on a non-existent database, since only 'u' ops can implicitly create - // collections. - assert.commandWorked(newDB.runCommand( - {applyOps: [{op: "u", ns: newDBName + ".foo", o: {_id: 5, x: 17}, o2: {_id: 5, x: 16}}]})); + // Updates on a non-existent database no longer implicitly create collections and will fail with + // a NamespaceNotFound error. + assert.commandFailedWithCode(newDB.runCommand({ + applyOps: [{op: "u", ns: newDBName + ".foo", o: {_id: 5, x: 17}, o2: {_id: 5, x: 16}}] + }), + ErrorCodes.NamespaceNotFound); var sawTooManyLocksError = false; diff --git a/jstests/core/autocomplete.js b/jstests/core/autocomplete.js new file mode 100644 index 00000000000..6eb6e21a7a3 --- /dev/null +++ b/jstests/core/autocomplete.js @@ -0,0 +1,42 @@ +/** + * Validate auto complete works for various javascript types implemented by C++. + */ +(function() { + 'use strict'; + + function testAutoComplete(prefix) { + // This method updates a global object with an array of strings on success. + shellAutocomplete(prefix); + return __autocomplete__; + } + + // Create a collection + db.auto_complete_coll.insert({}); + + // Validate DB auto completion + const db_stuff = testAutoComplete('db.'); + + // Verify we enumerate built-in methods + assert.contains('db.prototype', db_stuff); + assert.contains('db.hasOwnProperty', db_stuff); + assert.contains('db.toString(', db_stuff); + + // Verify we have some methods we added + assert.contains('db.adminCommand(', db_stuff); + assert.contains('db.runCommand(', db_stuff); + + // Verify we enumerate collections + assert.contains('db.auto_complete_coll', db_stuff); + + // Validate Collection autocompletion + const coll_stuff = testAutoComplete('db.auto_complete_coll.'); + + // Verify we enumerate built-in methods + assert.contains('db.auto_complete_coll.prototype', coll_stuff); + assert.contains('db.auto_complete_coll.hasOwnProperty', coll_stuff); + assert.contains('db.auto_complete_coll.toString(', coll_stuff); + + // Verify we have some methods we added + assert.contains('db.auto_complete_coll.aggregate(', coll_stuff); + assert.contains('db.auto_complete_coll.runCommand(', coll_stuff); +})();
\ No newline at end of file diff --git a/jstests/core/batch_write_command_delete.js b/jstests/core/batch_write_command_delete.js index 2aefcea6a7f..99b5f8e3a61 100644 --- a/jstests/core/batch_write_command_delete.js +++ b/jstests/core/batch_write_command_delete.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for delete // diff --git a/jstests/core/batch_write_command_insert.js b/jstests/core/batch_write_command_insert.js index 274f35513e7..3fa6cf98756 100644 --- a/jstests/core/batch_write_command_insert.js +++ b/jstests/core/batch_write_command_insert.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocol for inserts // diff --git a/jstests/core/batch_write_command_update.js b/jstests/core/batch_write_command_update.js index 2d9d2d699b2..987525e2515 100644 --- a/jstests/core/batch_write_command_update.js +++ b/jstests/core/batch_write_command_update.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + // // Ensures that mongod respects the batch write protocols for updates // diff --git a/jstests/core/bypass_doc_validation.js b/jstests/core/bypass_doc_validation.js index d9bca81ab6d..c2cdfff0bc9 100644 --- a/jstests/core/bypass_doc_validation.js +++ b/jstests/core/bypass_doc_validation.js @@ -1,5 +1,7 @@ // Test the bypassDocumentValidation flag with some database commands. The test uses relevant shell // helpers when they're available for the respective server commands. +// +// @tags: [requires_collmod_command] (function() { 'use strict'; diff --git a/jstests/core/capped6.js b/jstests/core/capped6.js index e94e7ea44e8..ca216fbe96a 100644 --- a/jstests/core/capped6.js +++ b/jstests/core/capped6.js @@ -1,4 +1,11 @@ // Test NamespaceDetails::cappedTruncateAfter via "captrunc" command +// +// @tags: [ +// # This test attempts to perform read operations on a capped collection after truncating +// # documents using the captrunc command. The writes from the captrunc command aren't guaranteed +// # to become visible until a later w="majority" write occurs. +// assumes_write_concern_unchanged, +// ] (function() { var coll = db.capped6; diff --git a/jstests/core/collation_plan_cache.js b/jstests/core/collation_plan_cache.js index 0eec77388e4..790bbbadaa6 100644 --- a/jstests/core/collation_plan_cache.js +++ b/jstests/core/collation_plan_cache.js @@ -1,4 +1,11 @@ // Integration testing for the plan cache and index filter commands with collation. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] (function() { 'use strict'; @@ -237,4 +244,4 @@ assert.eq(0, coll.runCommand('planCacheListFilters').filters.length, 'unexpected number of plan cache filters'); -})();
\ No newline at end of file +})(); diff --git a/jstests/core/collection_info_cache_race.js b/jstests/core/collection_info_cache_race.js index d57fc3340db..8fcde050e99 100644 --- a/jstests/core/collection_info_cache_race.js +++ b/jstests/core/collection_info_cache_race.js @@ -5,9 +5,9 @@ var coll = db.collection_info_cache_race; coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); // Fails when SERVER-16502 was not fixed, due to invariant -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); coll.drop(); assert.commandWorked(db.createCollection(coll.getName(), {autoIndexId: false})); assert.eq(null, coll.findOne()); -assert.writeOK(coll.save({_id: false}, {writeConcern: {w: 1}})); +assert.writeOK(coll.save({_id: false})); diff --git a/jstests/core/collmod.js b/jstests/core/collmod.js index 16f9694560c..e366041bd99 100644 --- a/jstests/core/collmod.js +++ b/jstests/core/collmod.js @@ -1,5 +1,7 @@ // Basic js tests for the collMod command. // Test setting the usePowerOf2Sizes flag, and modifying TTL indexes. +// +// @tags: [requires_collmod_command] function debug(x) { // printjson( x ); diff --git a/jstests/core/collmod_bad_spec.js b/jstests/core/collmod_bad_spec.js index ccce81fd4b1..c3d5e7a148e 100644 --- a/jstests/core/collmod_bad_spec.js +++ b/jstests/core/collmod_bad_spec.js @@ -2,6 +2,8 @@ // // Tests that a collMod with a bad specification does not cause any changes, and does not crash the // server. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js index ef5f8762a42..a9c72e108a1 100644 --- a/jstests/core/commands_that_do_not_write_do_not_accept_wc.js +++ b/jstests/core/commands_that_do_not_write_do_not_accept_wc.js @@ -2,6 +2,8 @@ * This file tests commands that do not support write concern. It passes both valid and invalid * writeConcern fields to commands and expects the commands to fail with a writeConcernNotSupported * error. + * + * @tags: [assumes_write_concern_unchanged] */ (function() { diff --git a/jstests/core/constructors.js b/jstests/core/constructors.js index 9e2cd26bbe8..93842780b42 100644 --- a/jstests/core/constructors.js +++ b/jstests/core/constructors.js @@ -1,4 +1,6 @@ // Tests to see what validity checks are done for 10gen specific object construction +// +// @tags: [requires_eval_command] // Takes a list of constructors and returns a new list with an extra entry for each constructor with // "new" prepended diff --git a/jstests/core/count10.js b/jstests/core/count10.js index 2a1853c399a..453775c97f5 100644 --- a/jstests/core/count10.js +++ b/jstests/core/count10.js @@ -1,4 +1,11 @@ // Test that interrupting a count returns an error code. +// +// @tags: [ +// # This test attempts to perform a count command and find it using the currentOp command. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] t = db.count10; t.drop(); diff --git a/jstests/core/count_plan_summary.js b/jstests/core/count_plan_summary.js index 48891d21e8e..365f289c457 100644 --- a/jstests/core/count_plan_summary.js +++ b/jstests/core/count_plan_summary.js @@ -1,5 +1,11 @@ -// Test that the plan summary string appears in db.currentOp() for -// count operations. SERVER-14064. +// Test that the plan summary string appears in db.currentOp() for count operations. SERVER-14064. +// +// @tags: [ +// # This test attempts to perform a find command and find it using the currentOp command. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_count_plan_summary; t.drop(); diff --git a/jstests/core/crud_api.js b/jstests/core/crud_api.js index c9dbfb40c85..d572d4b90a5 100644 --- a/jstests/core/crud_api.js +++ b/jstests/core/crud_api.js @@ -1,3 +1,5 @@ +// @tags: [assumes_write_concern_unchanged] + (function() { "use strict"; diff --git a/jstests/core/diagdata.js b/jstests/core/diagdata.js index 490e4a3eb2b..6938f8c5102 100644 --- a/jstests/core/diagdata.js +++ b/jstests/core/diagdata.js @@ -1,4 +1,5 @@ // Test that verifies getDiagnosticData returns FTDC data +load('jstests/libs/ftdc.js'); (function() { "use strict"; @@ -6,28 +7,5 @@ // Verify we require admin database assert.commandFailed(db.diagdata.runCommand("getDiagnosticData")); - // We need to retry a few times if run this test immediately after mongod is started as FTDC may - // not have run yet. - var foundGoodDocument = false; - - for (var i = 0; i < 60; ++i) { - var result = db.adminCommand("getDiagnosticData"); - assert.commandWorked(result); - - var data = result.data; - - if (!data.hasOwnProperty("start")) { - // Wait a little longer for FTDC to start - sleep(500); - } else { - // Check for a few common properties to ensure we got data - assert(data.hasOwnProperty("serverStatus"), - "does not have 'serverStatus' in '" + tojson(data) + "'"); - assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); - foundGoodDocument = true; - } - } - assert(foundGoodDocument, - "getDiagnosticData failed to return a non-empty command, is FTDC running?"); - + verifyGetDiagnosticData(db.getSiblingDB('admin')); })(); diff --git a/jstests/core/doc_validation.js b/jstests/core/doc_validation.js index a30763869e7..95dbae1b219 100644 --- a/jstests/core/doc_validation.js +++ b/jstests/core/doc_validation.js @@ -1,4 +1,6 @@ // Test basic inserts and updates with document validation. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/doc_validation_invalid_validators.js b/jstests/core/doc_validation_invalid_validators.js index b78b31c0977..70262f5de43 100644 --- a/jstests/core/doc_validation_invalid_validators.js +++ b/jstests/core/doc_validation_invalid_validators.js @@ -1,5 +1,7 @@ // Verify invalid validator statements won't work and that we // can't create validated collections on restricted databases. +// +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/doc_validation_options.js b/jstests/core/doc_validation_options.js index 8a96685e48f..d1434af6d45 100644 --- a/jstests/core/doc_validation_options.js +++ b/jstests/core/doc_validation_options.js @@ -1,3 +1,4 @@ +// @tags: [requires_collmod_command] (function() { "use strict"; diff --git a/jstests/core/dropdb_race.js b/jstests/core/dropdb_race.js index bd5e7e5ddba..d8b49d08174 100644 --- a/jstests/core/dropdb_race.js +++ b/jstests/core/dropdb_race.js @@ -1,4 +1,6 @@ // test dropping a db with simultaneous commits +// +// @tags: [assumes_write_concern_unchanged] m = db.getMongo(); baseName = "jstests_dur_droprace"; diff --git a/jstests/core/elemMatchProjection.js b/jstests/core/elemMatchProjection.js index 4e80e8a296e..5060d8a6346 100644 --- a/jstests/core/elemMatchProjection.js +++ b/jstests/core/elemMatchProjection.js @@ -4,35 +4,59 @@ t.drop(); date1 = new Date(); +// Generate monotonically increasing _id values. ObjectIds generated by the shell are not guaranteed +// to be monotically increasing, and we will depend on the _id sort order later in the test. +var currentId = 0; +function nextId() { + return ++currentId; +} + // Insert various styles of arrays for (i = 0; i < 100; i++) { - t.insert({group: 1, x: [1, 2, 3, 4, 5]}); - t.insert({group: 2, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}]}); + t.insert({_id: nextId(), group: 1, x: [1, 2, 3, 4, 5]}); + t.insert({_id: nextId(), group: 2, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}]}); t.insert({ + _id: nextId(), group: 3, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}], y: [{aa: 1, bb: 2}, {aa: 2, cc: 3}, {aa: 1, dd: 5}] }); - t.insert({group: 3, x: [{a: 1, b: 3}, {a: -6, c: 3}]}); - t.insert({group: 4, x: [{a: 1, b: 4}, {a: -6, c: 3}]}); - t.insert({group: 5, x: [new Date(), 5, 10, 'string', new ObjectId(), 123.456]}); + t.insert({_id: nextId(), group: 3, x: [{a: 1, b: 3}, {a: -6, c: 3}]}); + t.insert({_id: nextId(), group: 4, x: [{a: 1, b: 4}, {a: -6, c: 3}]}); + t.insert({_id: nextId(), group: 5, x: [new Date(), 5, 10, 'string', new ObjectId(), 123.456]}); t.insert({ + _id: nextId(), group: 6, x: [{a: 'string', b: date1}, {a: new ObjectId(), b: 1.2345}, {a: 'string2', b: date1}] }); - t.insert({group: 7, x: [{y: [1, 2, 3, 4]}]}); - t.insert({group: 8, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); - t.insert({group: 9, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}, {z: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); - t.insert({group: 10, x: [{a: 1, b: 2}, {a: 3, b: 4}], y: [{c: 1, d: 2}, {c: 3, d: 4}]}); - t.insert({group: 10, x: [{a: 1, b: 2}, {a: 3, b: 4}], y: [{c: 1, d: 2}, {c: 3, d: 4}]}); + t.insert({_id: nextId(), group: 7, x: [{y: [1, 2, 3, 4]}]}); + t.insert({_id: nextId(), group: 8, x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}]}); + t.insert({ + _id: nextId(), + group: 9, + x: [{y: [{a: 1, b: 2}, {a: 3, b: 4}]}, {z: [{a: 1, b: 2}, {a: 3, b: 4}]}] + }); + t.insert({ + _id: nextId(), + group: 10, + x: [{a: 1, b: 2}, {a: 3, b: 4}], + y: [{c: 1, d: 2}, {c: 3, d: 4}] + }); + t.insert({ + _id: nextId(), + group: 10, + x: [{a: 1, b: 2}, {a: 3, b: 4}], + y: [{c: 1, d: 2}, {c: 3, d: 4}] + }); t.insert({ + _id: nextId(), group: 11, x: [{a: 1, b: 2}, {a: 2, c: 3}, {a: 1, d: 5}], covered: [{aa: 1, bb: 2}, {aa: 2, cc: 3}, {aa: 1, dd: 5}] }); - t.insert({group: 12, x: {y: [{a: 1, b: 1}, {a: 1, b: 2}]}}); - t.insert({group: 13, x: [{a: 1, b: 1}, {a: 1, b: 2}]}); - t.insert({group: 13, x: [{a: 1, b: 2}, {a: 1, b: 1}]}); + t.insert({_id: nextId(), group: 12, x: {y: [{a: 1, b: 1}, {a: 1, b: 2}]}}); + t.insert({_id: nextId(), group: 13, x: [{a: 1, b: 1}, {a: 1, b: 2}]}); + t.insert({_id: nextId(), group: 13, x: [{a: 1, b: 2}, {a: 1, b: 1}]}); } t.ensureIndex({ group: 1, diff --git a/jstests/core/error2.js b/jstests/core/error2.js index 6f0b95bc17e..fb6a8e6e3b2 100644 --- a/jstests/core/error2.js +++ b/jstests/core/error2.js @@ -1,4 +1,5 @@ // Test that client gets stack trace on failed invoke +// @tags: [requires_eval_command] f = db.jstests_error2; diff --git a/jstests/core/eval0.js b/jstests/core/eval0.js index 5802f2597cb..c21c6be66c6 100644 --- a/jstests/core/eval0.js +++ b/jstests/core/eval0.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); assert.eq(17, diff --git a/jstests/core/eval1.js b/jstests/core/eval1.js index 8b139cae02a..b5bffac892e 100644 --- a/jstests/core/eval1.js +++ b/jstests/core/eval1.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval1; t.drop(); diff --git a/jstests/core/eval3.js b/jstests/core/eval3.js index c4f8be21056..b95837a6817 100644 --- a/jstests/core/eval3.js +++ b/jstests/core/eval3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval3; t.drop(); diff --git a/jstests/core/eval4.js b/jstests/core/eval4.js index 0d120b393de..9b0c2a49d82 100644 --- a/jstests/core/eval4.js +++ b/jstests/core/eval4.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval4; t.drop(); diff --git a/jstests/core/eval5.js b/jstests/core/eval5.js index 46bd679dd77..815365ac8b9 100644 --- a/jstests/core/eval5.js +++ b/jstests/core/eval5.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval5; t.drop(); diff --git a/jstests/core/eval6.js b/jstests/core/eval6.js index 31258f6917b..96b43b3516c 100644 --- a/jstests/core/eval6.js +++ b/jstests/core/eval6.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval6; t.drop(); diff --git a/jstests/core/eval7.js b/jstests/core/eval7.js index 80197fcdde6..3bace093db1 100644 --- a/jstests/core/eval7.js +++ b/jstests/core/eval7.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); diff --git a/jstests/core/eval9.js b/jstests/core/eval9.js index 1480ff6519c..82c230a8a95 100644 --- a/jstests/core/eval9.js +++ b/jstests/core/eval9.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); db.evalprep.drop(); diff --git a/jstests/core/eval_mr.js b/jstests/core/eval_mr.js index 4a3dc8dad6c..33ca96eea01 100644 --- a/jstests/core/eval_mr.js +++ b/jstests/core/eval_mr.js @@ -1,4 +1,6 @@ // Test that the eval command can't be used to invoke the mapReduce command. SERVER-17889. +// +// @tags: [requires_eval_command] (function() { "use strict"; db.eval_mr.drop(); diff --git a/jstests/core/eval_nolock.js b/jstests/core/eval_nolock.js index 9511784becb..0fde2666f5d 100644 --- a/jstests/core/eval_nolock.js +++ b/jstests/core/eval_nolock.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.eval_nolock; t.drop(); diff --git a/jstests/core/evala.js b/jstests/core/evala.js index 7ccf33ac754..09241eeedee 100644 --- a/jstests/core/evala.js +++ b/jstests/core/evala.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.evala; t.drop(); diff --git a/jstests/core/evalb.js b/jstests/core/evalb.js index 3391c4cc4f2..2de16f4ea83 100644 --- a/jstests/core/evalb.js +++ b/jstests/core/evalb.js @@ -1,5 +1,7 @@ // Check the return value of a db.eval function running a database query, and ensure the function's // contents are logged in the profile log. +// +// @tags: [requires_eval_command] // Use a reserved database name to avoid a conflict in the parallel test suite. var stddb = db; diff --git a/jstests/core/evald.js b/jstests/core/evald.js index 8049d2ba8ae..43e74fc4600 100644 --- a/jstests/core/evald.js +++ b/jstests/core/evald.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evald; t.drop(); diff --git a/jstests/core/evale.js b/jstests/core/evale.js index 1ddc8519fc6..20384e0d741 100644 --- a/jstests/core/evale.js +++ b/jstests/core/evale.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_evale; t.drop(); diff --git a/jstests/core/evalg.js b/jstests/core/evalg.js index 18503659217..56cb1dedd5f 100644 --- a/jstests/core/evalg.js +++ b/jstests/core/evalg.js @@ -1,4 +1,6 @@ // SERVER-17499: Test behavior of getMore on aggregation cursor under eval command. +// +// @tags: [requires_eval_command] db.evalg.drop(); for (var i = 0; i < 102; ++i) { db.evalg.insert({}); diff --git a/jstests/core/evalh.js b/jstests/core/evalh.js index 11e672f6bf4..b9c0d486d59 100644 --- a/jstests/core/evalh.js +++ b/jstests/core/evalh.js @@ -1,5 +1,7 @@ /** * Test that db.eval does not support auth. + * + * @tags: [requires_eval_command] */ (function() { 'use strict'; diff --git a/jstests/core/evalj.js b/jstests/core/evalj.js index f2326fff365..d6ef46430de 100644 --- a/jstests/core/evalj.js +++ b/jstests/core/evalj.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + (function() { "use strict"; diff --git a/jstests/core/existsa.js b/jstests/core/existsa.js index e9430b489a3..d98fd3f2d68 100644 --- a/jstests/core/existsa.js +++ b/jstests/core/existsa.js @@ -1,101 +1,111 @@ -// Sparse indexes are disallowed for $exists:false queries. SERVER-3918 - -t = db.jstests_existsa; -t.drop(); - -t.save({}); -t.save({a: 1}); -t.save({a: {x: 1}, b: 1}); - -/** Configure testing of an index { <indexKeyField>:1 }. */ -function setIndex(_indexKeyField) { - indexKeyField = _indexKeyField; - indexKeySpec = {}; - indexKeySpec[indexKeyField] = 1; - t.ensureIndex(indexKeySpec, {sparse: true}); -} -setIndex('a'); - -/** @return count when hinting the index to use. */ -function hintedCount(query) { - return t.find(query).hint(indexKeySpec).itcount(); -} - -/** The query field does not exist and the sparse index is not used without a hint. */ -function assertMissing(query, expectedMissing, expectedIndexedMissing) { - expectedMissing = expectedMissing || 1; - expectedIndexedMissing = expectedIndexedMissing || 0; - assert.eq(expectedMissing, t.count(query)); - // We also shouldn't get a different count depending on whether - // an index is used or not. - assert.eq(expectedIndexedMissing, hintedCount(query)); -} - -/** The query field exists and the sparse index is used without a hint. */ -function assertExists(query, expectedExists) { - expectedExists = expectedExists || 2; - assert.eq(expectedExists, t.count(query)); - // An $exists:true predicate generates no index filters. Add another predicate on the index key - // to trigger use of the index. - andClause = {}; - andClause[indexKeyField] = {$ne: null}; - Object.extend(query, {$and: [andClause]}); - assert.eq(expectedExists, t.count(query)); - assert.eq(expectedExists, hintedCount(query)); -} - -/** The query field exists and the sparse index is not used without a hint. */ -function assertExistsUnindexed(query, expectedExists) { - expectedExists = expectedExists || 2; - assert.eq(expectedExists, t.count(query)); - // Even with another predicate on the index key, the sparse index is disallowed. - andClause = {}; - andClause[indexKeyField] = {$ne: null}; - Object.extend(query, {$and: [andClause]}); - assert.eq(expectedExists, t.count(query)); - assert.eq(expectedExists, hintedCount(query)); -} - -// $exists:false queries match the proper number of documents and disallow the sparse index. -assertMissing({a: {$exists: false}}); -assertMissing({a: {$not: {$exists: true}}}); -assertMissing({$and: [{a: {$exists: false}}]}); -assertMissing({$or: [{a: {$exists: false}}]}); -assertMissing({$nor: [{a: {$exists: true}}]}); -assertMissing({'a.x': {$exists: false}}, 2, 1); - -// Currently a sparse index is disallowed even if the $exists:false query is on a different field. -assertMissing({b: {$exists: false}}, 2, 1); -assertMissing({b: {$exists: false}, a: {$ne: 6}}, 2, 1); -assertMissing({b: {$not: {$exists: true}}}, 2, 1); - -// Top level $exists:true queries match the proper number of documents -// and use the sparse index on { a : 1 }. -assertExists({a: {$exists: true}}); - -// Nested $exists queries match the proper number of documents and disallow the sparse index. -assertExistsUnindexed({$nor: [{a: {$exists: false}}]}); -assertExistsUnindexed({$nor: [{'a.x': {$exists: false}}]}, 1); -assertExistsUnindexed({a: {$not: {$exists: false}}}); - -// Nested $exists queries disallow the sparse index in some cases where it is not strictly -// necessary to do so. (Descriptive tests.) -assertExistsUnindexed({$nor: [{b: {$exists: false}}]}, 1); // Unindexed field. -assertExists({$or: [{a: {$exists: true}}]}); // $exists:true not $exists:false. - -// Behavior is similar with $elemMatch. -t.drop(); -t.save({a: [{}]}); -t.save({a: [{b: 1}]}); -t.save({a: [{b: 1}]}); -setIndex('a.b'); - -assertMissing({a: {$elemMatch: {b: {$exists: false}}}}); -// A $elemMatch predicate is treated as nested, and the index should be used for $exists:true. -assertExists({a: {$elemMatch: {b: {$exists: true}}}}); - -// A non sparse index will not be disallowed. -t.drop(); -t.save({}); -t.ensureIndex({a: 1}); -assert.eq(1, t.find({a: {$exists: false}}).itcount()); +/** + * Tests that sparse indexes are disallowed for $exists:false queries. + */ +(function() { + "use strict"; + + const coll = db.jstests_existsa; + coll.drop(); + + assert.writeOK(coll.insert({})); + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.insert({a: {x: 1}, b: 1})); + + let indexKeySpec = {}; + let indexKeyField = ''; + + /** Configure testing of an index { <indexKeyField>:1 }. */ + function setIndex(_indexKeyField) { + indexKeyField = _indexKeyField; + indexKeySpec = {}; + indexKeySpec[indexKeyField] = 1; + coll.ensureIndex(indexKeySpec, {sparse: true}); + } + setIndex('a'); + + /** @return count when hinting the index to use. */ + function hintedCount(query) { + return coll.find(query).hint(indexKeySpec).itcount(); + } + + /** The query field does not exist and the sparse index is not used without a hint. */ + function assertMissing(query, expectedMissing = 1, expectedIndexedMissing = 0) { + assert.eq(expectedMissing, coll.count(query)); + // We also shouldn't get a different count depending on whether + // an index is used or not. + assert.eq(expectedIndexedMissing, hintedCount(query)); + } + + /** The query field exists and the sparse index is used without a hint. */ + function assertExists(query, expectedExists = 2) { + assert.eq(expectedExists, coll.count(query)); + // An $exists:true predicate generates no index filters. Add another predicate on the index + // key to trigger use of the index. + let andClause = {}; + andClause[indexKeyField] = {$ne: null}; + Object.extend(query, {$and: [andClause]}); + assert.eq(expectedExists, coll.count(query)); + assert.eq(expectedExists, hintedCount(query)); + } + + /** The query field exists and the sparse index is not used without a hint. */ + function assertExistsUnindexed(query, expectedExists = 2) { + assert.eq(expectedExists, coll.count(query)); + // Even with another predicate on the index key, the sparse index is disallowed. + let andClause = {}; + andClause[indexKeyField] = {$ne: null}; + Object.extend(query, {$and: [andClause]}); + assert.eq(expectedExists, coll.count(query)); + assert.eq(expectedExists, hintedCount(query)); + } + + // $exists:false queries match the proper number of documents and disallow the sparse index. + assertMissing({a: {$exists: false}}); + assertMissing({a: {$not: {$exists: true}}}); + assertMissing({$and: [{a: {$exists: false}}]}); + assertMissing({$or: [{a: {$exists: false}}]}); + assertMissing({$nor: [{a: {$exists: true}}]}); + assertMissing({'a.x': {$exists: false}}, 2, 1); + + // Currently a sparse index is disallowed even if the $exists:false query is on a different + // field. + assertMissing({b: {$exists: false}}, 2, 1); + assertMissing({b: {$exists: false}, a: {$ne: 6}}, 2, 1); + assertMissing({b: {$not: {$exists: true}}}, 2, 1); + + // Top level $exists:true queries match the proper number of documents + // and use the sparse index on { a : 1 }. + assertExists({a: {$exists: true}}); + + // Nested $exists queries match the proper number of documents and disallow the sparse index. + assertExistsUnindexed({$nor: [{a: {$exists: false}}]}); + assertExistsUnindexed({$nor: [{'a.x': {$exists: false}}]}, 1); + assertExistsUnindexed({a: {$not: {$exists: false}}}); + + // Nested $exists queries disallow the sparse index in some cases where it is not strictly + // necessary to do so. (Descriptive tests.) + assertExistsUnindexed({$nor: [{b: {$exists: false}}]}, 1); // Unindexed field. + assertExists({$or: [{a: {$exists: true}}]}); // $exists:true not $exists:false. + + // Behavior is similar with $elemMatch. + coll.drop(); + assert.writeOK(coll.insert({a: [{}]})); + assert.writeOK(coll.insert({a: [{b: 1}]})); + assert.writeOK(coll.insert({a: [{b: [1]}]})); + setIndex('a.b'); + + assertMissing({a: {$elemMatch: {b: {$exists: false}}}}); + + // A $elemMatch predicate is treated as nested, and the index should be used for $exists:true. + assertExists({a: {$elemMatch: {b: {$exists: true}}}}); + + // A $not within $elemMatch should not attempt to use a sparse index for $exists:false. + assertExistsUnindexed({'a.b': {$elemMatch: {$not: {$exists: false}}}}, 1); + assertExistsUnindexed({'a.b': {$elemMatch: {$gt: 0, $not: {$exists: false}}}}, 1); + + // A non sparse index will not be disallowed. + coll.drop(); + assert.writeOK(coll.insert({})); + coll.ensureIndex({a: 1}); + assert.eq(1, coll.find({a: {$exists: false}}).itcount()); +})(); diff --git a/jstests/core/fsync.js b/jstests/core/fsync.js index 13ff20e4177..2ad625d88ba 100644 --- a/jstests/core/fsync.js +++ b/jstests/core/fsync.js @@ -6,6 +6,8 @@ * - Confirm that writes can progress after fsyncUnlock * - Confirm that the command can be run repeatedly without breaking things * - Confirm that the pseudo commands and eval can perform fsyncLock/Unlock + * + * @tags: [requires_eval_command] */ (function() { "use strict"; diff --git a/jstests/core/fts_dotted_prefix_fields.js b/jstests/core/fts_dotted_prefix_fields.js new file mode 100644 index 00000000000..f811c4a7203 --- /dev/null +++ b/jstests/core/fts_dotted_prefix_fields.js @@ -0,0 +1,15 @@ +// Test that text search works correct when the text index has dotted paths as the non-text +// prefixes. +(function() { + "use strict"; + + let coll = db.fts_dotted_prefix_fields; + coll.drop(); + assert.commandWorked(coll.createIndex({"a.x": 1, "a.y": 1, "b.x": 1, "b.y": 1, words: "text"})); + assert.writeOK(coll.insert({a: {x: 1, y: 2}, b: {x: 3, y: 4}, words: "lorem ipsum dolor sit"})); + assert.writeOK(coll.insert({a: {x: 1, y: 2}, b: {x: 5, y: 4}, words: "lorem ipsum dolor sit"})); + + assert.eq(1, + coll.find({$text: {$search: "lorem ipsum"}, "a.x": 1, "a.y": 2, "b.x": 3, "b.y": 4}) + .itcount()); +}()); diff --git a/jstests/core/fts_trailing_fields.js b/jstests/core/fts_trailing_fields.js new file mode 100644 index 00000000000..2c7f79b423d --- /dev/null +++ b/jstests/core/fts_trailing_fields.js @@ -0,0 +1,22 @@ +// Tests for predicates which can use the trailing field of a text index. +(function() { + "use strict"; + + const coll = db.fts_trailing_fields; + + coll.drop(); + assert.commandWorked(coll.createIndex({a: 1, b: "text", c: 1})); + assert.writeOK(coll.insert({a: 2, b: "lorem ipsum"})); + + assert.eq(0, coll.find({a: 2, $text: {$search: "lorem"}, c: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: 2, $text: {$search: "lorem"}, c: null}).itcount()); + assert.eq(1, coll.find({a: 2, $text: {$search: "lorem"}, c: {$exists: false}}).itcount()); + + // An equality predicate on the leading field isn't useful, but it shouldn't cause any problems. + // Same with an $elemMatch predicate on one of the trailing fields. + coll.drop(); + assert.commandWorked(coll.createIndex({a: 1, b: "text", "c.d": 1})); + assert.writeOK(coll.insert({a: 2, b: "lorem ipsum", c: {d: 3}})); + assert.eq(0, coll.find({a: [1, 2], $text: {$search: "lorem"}}).itcount()); + assert.eq(0, coll.find({a: 2, $text: {$search: "lorem"}, c: {$elemMatch: {d: 3}}}).itcount()); +}()); diff --git a/jstests/core/function_string_representations.js b/jstests/core/function_string_representations.js new file mode 100644 index 00000000000..af66e9160a9 --- /dev/null +++ b/jstests/core/function_string_representations.js @@ -0,0 +1,38 @@ +/** Demonstrate that mapReduce can accept functions represented by strings. + * Some drivers do not have a type which represents a Javascript function. These languages represent + * the arguments to mapReduce as strings. + */ + +(function() { + "use strict"; + + var col = db.function_string_representations; + col.drop(); + assert.writeOK(col.insert({ + _id: "abc123", + ord_date: new Date("Oct 04, 2012"), + status: 'A', + price: 25, + items: [{sku: "mmm", qty: 5, price: 2.5}, {sku: "nnn", qty: 5, price: 2.5}] + })); + + var mapFunction = "function() {emit(this._id, this.price);}"; + var reduceFunction = "function(keyCustId, valuesPrices) {return Array.sum(valuesPrices);}"; + assert.commandWorked(col.mapReduce(mapFunction, reduceFunction, {out: "map_reduce_example"})); + + // Provided strings may end with semicolons and/or whitespace + mapFunction += " ; "; + reduceFunction += " ; "; + assert.commandWorked(col.mapReduce(mapFunction, reduceFunction, {out: "map_reduce_example"})); + + // $where exhibits the same behavior + var whereFunction = "function() {return this.price === 25;}"; + assert.eq(1, col.find({$where: whereFunction}).itcount()); + + whereFunction += ";"; + assert.eq(1, col.find({$where: whereFunction}).itcount()); + + // db.eval does not need to be tested, as it accepts code fragments, not functions. + // system.js does not need to be tested, as its contents types' are preserved, and + // strings are not promoted into functions. +})(); diff --git a/jstests/core/geo_2d_trailing_fields.js b/jstests/core/geo_2d_trailing_fields.js new file mode 100644 index 00000000000..8f9c881ae4c --- /dev/null +++ b/jstests/core/geo_2d_trailing_fields.js @@ -0,0 +1,45 @@ +// Tests for predicates which can use the trailing field of a 2d index. +(function() { + "use strict"; + + const coll = db.geo_2d_trailing_fields; + + const isMaster = assert.commandWorked(db.adminCommand({isMaster: 1})); + const isMongos = (isMaster.msg === "isdbgrid"); + + coll.drop(); + assert.commandWorked(coll.createIndex({a: "2d", b: 1})); + assert.writeOK(coll.insert({a: [0, 0]})); + + // Verify that $near queries handle existence predicates over the trailing fields correctly. + if (!isMongos) { + assert.eq(0, coll.find({a: {$near: [0, 0]}, b: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: null}).itcount()); + assert.eq(1, coll.find({a: {$near: [0, 0]}, b: {$exists: false}}).itcount()); + } + + // Verify that non-near 2d queries handle existence predicates over the trailing fields + // correctly. + assert.eq(0, + coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: true}}).itcount()); + assert.eq(1, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: null}).itcount()); + assert.eq(1, + coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, b: {$exists: false}}).itcount()); + + coll.drop(); + assert.commandWorked(coll.createIndex({a: "2d", "b.c": 1})); + assert.writeOK(coll.insert({a: [0, 0], b: [{c: 2}, {c: 3}]})); + + // Verify that $near queries correctly handle predicates which cannot be covered due to array + // semantics. + if (!isMongos) { + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": [2, 3]}).itcount()); + assert.eq(0, coll.find({a: {$near: [0, 0]}, "b.c": {$type: "array"}}).itcount()); + } + + // Verify that non-near 2d queries correctly handle predicates which cannot be covered due to + // array semantics. + assert.eq(0, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, "b.c": [2, 3]}).itcount()); + assert.eq( + 0, coll.find({a: {$geoWithin: {$center: [[0, 0], 1]}}, "b.c": {$type: "array"}}).itcount()); +}()); diff --git a/jstests/core/geo_s2cursorlimitskip.js b/jstests/core/geo_s2cursorlimitskip.js index 427fbf8fe29..dc645dc68af 100644 --- a/jstests/core/geo_s2cursorlimitskip.js +++ b/jstests/core/geo_s2cursorlimitskip.js @@ -1,4 +1,11 @@ // Test various cursor behaviors +// +// @tags: [ +// # This test attempts to enable profiling on a server and then get profiling data by reading +// # from the "system.profile" collection. The former operation must be routed to the primary in +// # a replica set, whereas the latter may be routed to a secondary. +// assumes_read_preference_unchanged, +// ] var testDB = db.getSiblingDB("geo_s2cursorlimitskip"); var t = testDB.geo_s2getmmm; diff --git a/jstests/core/geo_update_btree.js b/jstests/core/geo_update_btree.js index a85d4274415..b4fa24df57e 100644 --- a/jstests/core/geo_update_btree.js +++ b/jstests/core/geo_update_btree.js @@ -1,4 +1,6 @@ // Tests whether the geospatial search is stable under btree updates +// +// @tags: [assumes_write_concern_unchanged] var coll = db.getCollection("jstests_geo_update_btree"); coll.drop(); diff --git a/jstests/core/getlog2.js b/jstests/core/getlog2.js index 597a85e20ee..e5287ea8c1b 100644 --- a/jstests/core/getlog2.js +++ b/jstests/core/getlog2.js @@ -1,4 +1,11 @@ // tests getlog as well as slow querying logging +// +// @tags: [ +// # This test attempts to perform a find command and see that it ran using the getLog command. +// # The former operation may be routed to a secondary in the replica set, whereas the latter must +// # be routed to the primary. +// assumes_read_preference_unchanged, +// ] glcol = db.getLogTest2; glcol.drop(); diff --git a/jstests/core/index_elemmatch2.js b/jstests/core/index_elemmatch2.js new file mode 100644 index 00000000000..ecd24035284 --- /dev/null +++ b/jstests/core/index_elemmatch2.js @@ -0,0 +1,63 @@ +/** + * Test that queries containing $elemMatch correctly use an index if each child expression is + * compatible with the index. + */ +(function() { + "use strict"; + + load("jstests/libs/analyze_plan.js"); + + const coll = db.elemMatch_index; + coll.drop(); + + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.insert({a: [{}]})); + assert.writeOK(coll.insert({a: [1, null]})); + assert.writeOK(coll.insert({a: [{type: "Point", coordinates: [0, 0]}]})); + + assert.commandWorked(coll.createIndex({a: 1}, {sparse: true})); + + function assertIndexResults(coll, query, useIndex, nReturned) { + const explainPlan = coll.find(query).explain("executionStats"); + assert.eq(isIxscan(explainPlan.queryPlanner.winningPlan), useIndex); + assert.eq(explainPlan.executionStats.nReturned, nReturned); + } + + assertIndexResults(coll, {a: {$elemMatch: {$exists: false}}}, false, 0); + + // An $elemMatch predicate is treated as nested, and the index should be used for $exists:true. + assertIndexResults(coll, {a: {$elemMatch: {$exists: true}}}, true, 3); + + // $not within $elemMatch should not attempt to use a sparse index for $exists:false. + assertIndexResults(coll, {a: {$elemMatch: {$not: {$exists: false}}}}, false, 3); + assertIndexResults(coll, {a: {$elemMatch: {$gt: 0, $not: {$exists: false}}}}, false, 1); + + // $geo within $elemMatch should not attempt to use a non-geo index. + assertIndexResults( + coll, + { + a: { + $elemMatch: { + $geoWithin: { + $geometry: + {type: "Polygon", coordinates: [[[0, 0], [0, 1], [1, 0], [0, 0]]]} + } + } + } + }, + false, + 1); + + // $in with a null value within $elemMatch should use a sparse index. + assertIndexResults(coll, {a: {$elemMatch: {$in: [null]}}}, true, 1); + + // $eq with a null value within $elemMatch should use a sparse index. + assertIndexResults(coll, {a: {$elemMatch: {$eq: null}}}, true, 1); + + // A negated regex within $elemMatch should not use an index, sparse or not. + assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3); + + coll.dropIndexes(); + assert.commandWorked(coll.createIndex({a: 1})); + assertIndexResults(coll, {a: {$elemMatch: {$not: {$in: [/^a/]}}}}, false, 3); +})(); diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js index 8684be3b2b9..58f78d0514e 100644 --- a/jstests/core/index_filter_commands.js +++ b/jstests/core/index_filter_commands.js @@ -6,20 +6,24 @@ * Displays index filters for all query shapes in a collection. * * - planCacheClearFilters - * Clears index filter for a single query shape or, - * if the query shape is omitted, all filters for the collection. + * Clears index filter for a single query shape or, if the query shape is omitted, all filters for + * the collection. * * - planCacheSetFilter * Sets index filter for a query shape. Overrides existing filter. * - * Not a lot of data access in this test suite. Hint commands - * manage a non-persistent mapping in the server of - * query shape to list of index specs. + * Not a lot of data access in this test suite. Hint commands manage a non-persistent mapping in the + * server of query shape to list of index specs. * - * Only time we might need to execute a query is to check the plan - * cache state. We would do this with the planCacheListPlans command - * on the same query shape with the index filters. + * Only time we might need to execute a query is to check the plan cache state. We would do this + * with the planCacheListPlans command on the same query shape with the index filters. * + * @tags: [ + * # This test attempts to perform queries with plan cache filters set up. The former operation + * # may be routed to a secondary in the replica set, whereas the latter must be routed to the + * # primary. + * assumes_read_preference_unchanged, + * ] */ load("jstests/libs/analyze_plan.js"); diff --git a/jstests/core/index_stats.js b/jstests/core/index_stats.js index 60b37fd571e..ee4d13d4d0a 100644 --- a/jstests/core/index_stats.js +++ b/jstests/core/index_stats.js @@ -1,3 +1,10 @@ +// @tags: [ +// # This test attempts to perform write operations and get index usage statistics using the +// # $indexStats stage. The former operation must be routed to the primary in a replica set, +// # whereas the latter may be routed to a secondary. +// assumes_read_preference_unchanged, +// ] + (function() { "use strict"; diff --git a/jstests/core/js3.js b/jstests/core/js3.js index 4d46c25bbf7..c808e7ec75a 100644 --- a/jstests/core/js3.js +++ b/jstests/core/js3.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.jstests_js3; diff --git a/jstests/core/js7.js b/jstests/core/js7.js index 810f4692d4f..99cddd114f8 100644 --- a/jstests/core/js7.js +++ b/jstests/core/js7.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_js7; t.drop(); diff --git a/jstests/core/js9.js b/jstests/core/js9.js index 515fa883aea..e0703ac39ea 100644 --- a/jstests/core/js9.js +++ b/jstests/core/js9.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + c = db.jstests_js9; c.drop(); diff --git a/jstests/core/js_jit.js b/jstests/core/js_jit.js new file mode 100644 index 00000000000..4ccdd2917ae --- /dev/null +++ b/jstests/core/js_jit.js @@ -0,0 +1,40 @@ +/** + * Validate various native types continue to work when run in JITed code. + * + * In SERVER-30362, the JIT would not compile natives types which had custom getProperty + * implementations correctly. We force the JIT to kick in by using large loops. + */ +(function() { + 'use strict'; + + function testDBCollection() { + const c = new DBCollection(null, null, "foo", "test.foo"); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "test.foo") { + throw i; + } + } + } + + function testDB() { + const c = new DB(null, "test"); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "test") { + throw i; + } + } + } + + function testDBQuery() { + const c = DBQuery('a', 'b', 'c', 'd'); + for (let i = 0; i < 100000; i++) { + if (c.toString() != "DBQuery: d -> null") { + throw i; + } + } + } + + testDBCollection(); + testDB(); + testDBQuery(); +})();
\ No newline at end of file diff --git a/jstests/core/list_collections1.js b/jstests/core/list_collections1.js index c8c3f92fbc9..ff1aac304ff 100644 --- a/jstests/core/list_collections1.js +++ b/jstests/core/list_collections1.js @@ -72,8 +72,8 @@ // var getListCollectionsCursor = function(options, subsequentBatchSize) { - return new DBCommandCursor( - mydb.getMongo(), mydb.runCommand("listCollections", options), subsequentBatchSize); + var res = mydb.runCommand("listCollections", options); + return new DBCommandCursor(res._mongo, res, subsequentBatchSize); }; var cursorCountMatching = function(cursor, pred) { @@ -282,9 +282,9 @@ assert.commandWorked(mydb.createCollection("quux")); res = mydb.runCommand("listCollections", {cursor: {batchSize: 0}}); - cursor = new DBCommandCursor(mydb.getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); cursor.close(); - cursor = new DBCommandCursor(mydb.getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); assert.throws(function() { cursor.hasNext(); }); diff --git a/jstests/core/list_collections_filter.js b/jstests/core/list_collections_filter.js index e0d18f055d0..0516d57a417 100644 --- a/jstests/core/list_collections_filter.js +++ b/jstests/core/list_collections_filter.js @@ -19,8 +19,8 @@ filter = {}; } - var cursor = new DBCommandCursor(mydb.getMongo(), - mydb.runCommand("listCollections", {filter: filter})); + var res = mydb.runCommand("listCollections", {filter: filter}); + var cursor = new DBCommandCursor(res._mongo, res); function stripToName(result) { return result.name; } diff --git a/jstests/core/list_indexes.js b/jstests/core/list_indexes.js index de0f4473980..6ec9a7a13e1 100644 --- a/jstests/core/list_indexes.js +++ b/jstests/core/list_indexes.js @@ -27,8 +27,8 @@ // var getListIndexesCursor = function(coll, options, subsequentBatchSize) { - return new DBCommandCursor( - coll.getDB().getMongo(), coll.runCommand("listIndexes", options), subsequentBatchSize); + var res = coll.runCommand("listIndexes", options); + return new DBCommandCursor(res._mongo, res, subsequentBatchSize); }; var cursorGetIndexSpecs = function(cursor) { @@ -163,9 +163,9 @@ assert.commandWorked(coll.ensureIndex({c: 1}, {unique: true})); res = coll.runCommand("listIndexes", {cursor: {batchSize: 0}}); - cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); cursor.close(); - cursor = new DBCommandCursor(coll.getDB().getMongo(), res, 2); + cursor = new DBCommandCursor(res._mongo, res, 2); assert.throws(function() { cursor.hasNext(); }); diff --git a/jstests/core/list_indexes_invalidation.js b/jstests/core/list_indexes_invalidation.js index b8cbe5eb134..9fe94de5efc 100644 --- a/jstests/core/list_indexes_invalidation.js +++ b/jstests/core/list_indexes_invalidation.js @@ -19,7 +19,7 @@ printjson(res); // Ensure the cursor has data, rename or drop the collection, and exhaust the cursor. - let cursor = new DBCommandCursor(db.getMongo(), res); + let cursor = new DBCommandCursor(res._mongo, res); let errMsg = 'expected more data from command ' + tojson(cmd) + ', with result ' + tojson(res); assert(cursor.hasNext(), errMsg); diff --git a/jstests/core/list_namespaces_invalidation.js b/jstests/core/list_namespaces_invalidation.js index 6f8033b5fe4..85eb26510bb 100644 --- a/jstests/core/list_namespaces_invalidation.js +++ b/jstests/core/list_namespaces_invalidation.js @@ -1,4 +1,6 @@ // SERVER-27996/SERVER-28022 Missing invalidation for system.namespaces writes +// +// @tags: [requires_collmod_command] (function() { 'use strict'; let dbInvalidName = 'system_namespaces_invalidations'; diff --git a/jstests/core/max_time_ms.js b/jstests/core/max_time_ms.js index 0442ffcba68..efe3dabcc6d 100644 --- a/jstests/core/max_time_ms.js +++ b/jstests/core/max_time_ms.js @@ -1,4 +1,12 @@ // Tests query/command option $maxTimeMS. +// +// @tags: [ +// # This test attempts to perform read operations after having enabled the maxTimeAlwaysTimeOut +// # failpoint. The former operations may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// requires_collmod_command, +// ] var t = db.max_time_ms; var exceededTimeLimit = 50; // ErrorCodes::ExceededTimeLimit diff --git a/jstests/core/mr4.js b/jstests/core/mr4.js index 58ea303f7e8..683583acecd 100644 --- a/jstests/core/mr4.js +++ b/jstests/core/mr4.js @@ -9,7 +9,7 @@ t.save({x: 4, tags: ["b", "c"]}); m = function() { this.tags.forEach(function(z) { - emit(z, {count: xx}); + emit(z, {count: xx.val}); }); }; @@ -21,7 +21,7 @@ r = function(key, values) { return {count: total}; }; -res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: 1}}); +res = t.mapReduce(m, r, {out: "mr4_out", scope: {xx: {val: 1}}}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); @@ -31,7 +31,7 @@ assert.eq(3, z.c.count, "A4"); res.drop(); -res = t.mapReduce(m, r, {scope: {xx: 2}, out: "mr4_out"}); +res = t.mapReduce(m, r, {scope: {xx: {val: 2}}, out: "mr4_out"}); z = res.convertToSingleObject(); assert.eq(3, Object.keySet(z).length, "A1"); diff --git a/jstests/core/mr_killop.js b/jstests/core/mr_killop.js index 78e98f0bcaa..e85986e2fc0 100644 --- a/jstests/core/mr_killop.js +++ b/jstests/core/mr_killop.js @@ -4,7 +4,7 @@ t = db.jstests_mr_killop; t.drop(); t2 = db.jstests_mr_killop_out; t2.drop(); - +db.adminCommand({"configureFailPoint": 'mr_killop_test_fp', "mode": 'alwaysOn'}); function debug(x) { // printjson( x ); } @@ -171,3 +171,4 @@ var loop = function() { }; runMRTests(loop, false); runFinalizeTests(loop, false); +db.adminCommand({"configureFailPoint": 'mr_killop_test_fp', "mode": 'off'}); diff --git a/jstests/core/mr_optim.js b/jstests/core/mr_optim.js index 7437753ca67..1c525ae3de3 100644 --- a/jstests/core/mr_optim.js +++ b/jstests/core/mr_optim.js @@ -3,8 +3,17 @@ t = db.mr_optim; t.drop(); +// We drop the output collection to ensure the test can be run multiple times successfully. We +// explicitly avoid using the DBCollection#drop() shell helper to avoid implicitly sharding the +// collection during the sharded_collections_jscore_passthrough.yml test suite when reading the +// results from the output collection in the reformat() function. +var res = db.runCommand({drop: "mr_optim_out"}); +if (res.ok !== 1) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceNotFound); +} + for (var i = 0; i < 1000; ++i) { - t.save({a: Math.random(1000), b: Math.random(10000)}); + assert.writeOK(t.save({a: Math.random(1000), b: Math.random(10000)})); } function m() { @@ -21,7 +30,7 @@ function reformat(r) { if (r.results) cursor = r.results; else - cursor = r.find(); + cursor = r.find().sort({_id: 1}); cursor.forEach(function(z) { x[z._id] = z.value; }); @@ -43,4 +52,4 @@ res.drop(); assert.eq(x, x2, "object from inline and collection are not equal"); -t.drop();
\ No newline at end of file +t.drop(); diff --git a/jstests/core/no_db_created.js b/jstests/core/no_db_created.js index 3491914d470..b67b193494c 100644 --- a/jstests/core/no_db_created.js +++ b/jstests/core/no_db_created.js @@ -1,4 +1,6 @@ // checks that operations do not create a database +// +// @tags: [requires_collmod_command] (function() { "use strict"; @@ -32,4 +34,4 @@ noDB(mydb); assert.writeOK(coll.insert({})); mydb.dropDatabase(); -}());
\ No newline at end of file +}()); diff --git a/jstests/core/notablescan.js b/jstests/core/notablescan.js index 80306c08cf2..bb4c170a603 100644 --- a/jstests/core/notablescan.js +++ b/jstests/core/notablescan.js @@ -1,4 +1,11 @@ // check notablescan mode +// +// @tags: [ +// # This test attempts to perform read operations after having enabled the notablescan server +// # parameter. The former operations may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// ] t = db.test_notablescan; t.drop(); diff --git a/jstests/core/operation_latency_histogram.js b/jstests/core/operation_latency_histogram.js index 1e3f1a59b95..947a8be6520 100644 --- a/jstests/core/operation_latency_histogram.js +++ b/jstests/core/operation_latency_histogram.js @@ -1,4 +1,10 @@ // Checks that histogram counters for collections are updated as we expect. +// +// This test attempts to perform write operations and get latency statistics using the $collStats +// stage. The former operation must be routed to the primary in a replica set, whereas the latter +// may be routed to a secondary. +// +// @tags: [assumes_read_preference_unchanged] (function() { "use strict"; diff --git a/jstests/core/plan_cache_clear.js b/jstests/core/plan_cache_clear.js index 8f9cf0ea302..778239616b5 100644 --- a/jstests/core/plan_cache_clear.js +++ b/jstests/core/plan_cache_clear.js @@ -1,5 +1,12 @@ // Test clearing of the plan cache, either manually through the planCacheClear command, // or due to system events such as an index build. +// +// @tags: [ +// # This test attempts to perform queries and introspect/manipulate the server's plan cache +// # entries. The former operation may be routed to a secondary in the replica set, whereas the +// # latter must be routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_clear; t.drop(); diff --git a/jstests/core/plan_cache_list_plans.js b/jstests/core/plan_cache_list_plans.js index 7ca599483ff..3359980ab07 100644 --- a/jstests/core/plan_cache_list_plans.js +++ b/jstests/core/plan_cache_list_plans.js @@ -1,4 +1,11 @@ // Test the planCacheListPlans command. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_list_plans; t.drop(); diff --git a/jstests/core/plan_cache_list_shapes.js b/jstests/core/plan_cache_list_shapes.js index 1c9ecdf9e1b..61c3111cd8a 100644 --- a/jstests/core/plan_cache_list_shapes.js +++ b/jstests/core/plan_cache_list_shapes.js @@ -1,5 +1,12 @@ // Test the planCacheListQueryShapes command, which returns a list of query shapes // for the queries currently cached in the collection. +// +// @tags: [ +// # This test attempts to perform queries with plan cache filters set up. The former operation +// # may be routed to a secondary in the replica set, whereas the latter must be routed to the +// # primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_list_shapes; t.drop(); diff --git a/jstests/core/plan_cache_shell_helpers.js b/jstests/core/plan_cache_shell_helpers.js index dc990b19dcc..6c4ff185014 100644 --- a/jstests/core/plan_cache_shell_helpers.js +++ b/jstests/core/plan_cache_shell_helpers.js @@ -1,4 +1,11 @@ // Test the shell helpers which wrap the plan cache commands. +// +// @tags: [ +// # This test attempts to perform queries and introspect the server's plan cache entries. The +// # former operation may be routed to a secondary in the replica set, whereas the latter must be +// # routed to the primary. +// assumes_read_preference_unchanged, +// ] var t = db.jstests_plan_cache_shell_helpers; t.drop(); diff --git a/jstests/core/profile2.js b/jstests/core/profile2.js index bb1605abd1e..9da6410d2ac 100644 --- a/jstests/core/profile2.js +++ b/jstests/core/profile2.js @@ -24,7 +24,7 @@ assert(result.hasOwnProperty('millis')); assert(result.hasOwnProperty('query')); assert.eq('string', typeof(result.query)); // String value is truncated. -assert(result.query.match(/filter: { a: "a+\.\.\." } }$/)); +assert(result.query.match(/filter: { a: "a+\.\.\." }/)); assert.commandWorked(coll.getDB().runCommand({profile: 0})); coll.getDB().system.profile.drop(); diff --git a/jstests/core/profile_getmore.js b/jstests/core/profile_getmore.js index a9272567b1a..3f6f492597e 100644 --- a/jstests/core/profile_getmore.js +++ b/jstests/core/profile_getmore.js @@ -24,12 +24,13 @@ var cursor = coll.find({a: {$gt: 0}}).sort({a: 1}).batchSize(2); cursor.next(); // Perform initial query and consume first of 2 docs returned. - var cursorId = getLatestProfilerEntry(testDB).cursorid; // Save cursorid from find. + var cursorId = + getLatestProfilerEntry(testDB, {op: "query"}).cursorid; // Save cursorid from find. cursor.next(); // Consume second of 2 docs from initial query. cursor.next(); // getMore performed, leaving open cursor. - var profileObj = getLatestProfilerEntry(testDB); + var profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.ns, coll.getFullName(), tojson(profileObj)); assert.eq(profileObj.op, "getmore", tojson(profileObj)); @@ -67,7 +68,7 @@ cursor.next(); // Consume second of 2 docs from initial query. cursor.next(); // getMore performed, leaving open cursor. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.hasSortStage, true, tojson(profileObj)); @@ -83,7 +84,7 @@ cursor.next(); // Perform initial query and consume first of 3 docs returned. cursor.itcount(); // Exhaust the cursor. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert(profileObj.hasOwnProperty("cursorid"), tojson(profileObj)); // cursorid should always be present on getMore. @@ -101,12 +102,12 @@ assert.commandWorked(coll.createIndex({a: 1})); var cursor = coll.aggregate([{$match: {a: {$gte: 0}}}], {cursor: {batchSize: 0}}); - var cursorId = getLatestProfilerEntry(testDB).cursorid; + var cursorId = getLatestProfilerEntry(testDB, {"command.aggregate": coll.getName()}).cursorid; assert.neq(0, cursorId); cursor.next(); // Consume the result set. - profileObj = getLatestProfilerEntry(testDB); + profileObj = getLatestProfilerEntry(testDB, {op: "getmore"}); assert.eq(profileObj.ns, coll.getFullName(), tojson(profileObj)); assert.eq(profileObj.op, "getmore", tojson(profileObj)); diff --git a/jstests/core/profile_insert.js b/jstests/core/profile_insert.js index 3994c896bda..836ec2ecff2 100644 --- a/jstests/core/profile_insert.js +++ b/jstests/core/profile_insert.js @@ -1,4 +1,6 @@ // Confirms that profiled insert execution contains all expected metrics with proper values. +// +// @tags: [assumes_write_concern_unchanged] (function() { "use strict"; diff --git a/jstests/core/recursion.js b/jstests/core/recursion.js index 926250be20d..1db491c8ae9 100644 --- a/jstests/core/recursion.js +++ b/jstests/core/recursion.js @@ -1,5 +1,7 @@ -// Basic tests for a form of stack recursion that's been shown to cause C++ -// side stack overflows in the past. See SERVER-19614. +// Basic tests for a form of stack recursion that's been shown to cause C++ side stack overflows in +// the past. See SERVER-19614. +// +// @tags: [requires_eval_command] (function() { "use strict"; diff --git a/jstests/core/regex_not_id.js b/jstests/core/regex_not_id.js index 1f15250f240..35b2c858867 100644 --- a/jstests/core/regex_not_id.js +++ b/jstests/core/regex_not_id.js @@ -3,10 +3,10 @@ var testColl = db.regex_not_id; testColl.drop(); -assert.writeOK(testColl.insert({_id: "ABCDEF1"}, {writeConcern: {w: 1}})); +assert.writeOK(testColl.insert({_id: "ABCDEF1"})); // Should be an error. -assert.writeError(testColl.insert({_id: /^A/}, {writeConcern: {w: 1}})); +assert.writeError(testColl.insert({_id: /^A/})); // _id doesn't have to be first; still disallowed -assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/}, {writeConcern: {w: 1}}));
\ No newline at end of file +assert.writeError(testColl.insert({xxx: "ABCDEF", _id: /ABCDEF/})); diff --git a/jstests/core/remove8.js b/jstests/core/remove8.js index 563e4708cf9..3c9fd6a11a1 100644 --- a/jstests/core/remove8.js +++ b/jstests/core/remove8.js @@ -1,3 +1,4 @@ +// @tags: [requires_eval_command] t = db.remove8; t.drop(); diff --git a/jstests/core/rename4.js b/jstests/core/rename4.js index 185193deaa9..756918db5f6 100644 --- a/jstests/core/rename4.js +++ b/jstests/core/rename4.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + t = db.jstests_rename4; t.drop(); diff --git a/jstests/core/rename_stayTemp.js b/jstests/core/rename_stayTemp.js index d8451af2d2d..8dee8f89fea 100644 --- a/jstests/core/rename_stayTemp.js +++ b/jstests/core/rename_stayTemp.js @@ -11,7 +11,7 @@ function ns(coll) { function istemp(name) { var result = db.runCommand("listCollections", {filter: {name: name}}); assert(result.ok); - var collections = new DBCommandCursor(db.getMongo(), result).toArray(); + var collections = new DBCommandCursor(result._mongo, result).toArray(); assert.eq(1, collections.length); return collections[0].options.temp ? true : false; } diff --git a/jstests/core/shell_connection_strings.js b/jstests/core/shell_connection_strings.js new file mode 100644 index 00000000000..22861e6ce25 --- /dev/null +++ b/jstests/core/shell_connection_strings.js @@ -0,0 +1,33 @@ +// Test mongo shell connect strings. +(function() { + 'use strict'; + + const mongod = new MongoURI(db.getMongo().host).servers[0]; + const host = mongod.host; + const port = mongod.port; + + function testConnect(ok, ...args) { + const exitCode = runMongoProgram('mongo', '--eval', ';', ...args); + if (ok) { + assert.eq(exitCode, 0, "failed to connect with `" + args.join(' ') + "`"); + } else { + assert.neq( + exitCode, 0, "unexpectedly succeeded connecting with `" + args.join(' ') + "`"); + } + } + + testConnect(true, `${host}:${port}`); + testConnect(true, `${host}:${port}/test`); + testConnect(true, `${host}:${port}/admin`); + testConnect(true, host, '--port', port); + testConnect(true, '--host', host, '--port', port, 'test'); + testConnect(true, '--host', host, '--port', port, 'admin'); + testConnect(true, `mongodb://${host}:${port}/test`); + testConnect(true, `mongodb://${host}:${port}/test?connectTimeoutMS=10000`); + + // if a full URI is provided, you cannot also specify host or port + testConnect(false, `${host}/test`, '--port', port); + testConnect(false, `mongodb://${host}:${port}/test`, '--port', port); + testConnect(false, `mongodb://${host}:${port}/test`, '--host', host); + testConnect(false, `mongodb://${host}:${port}/test`, '--host', host, '--port', port); +})(); diff --git a/jstests/core/shell_writeconcern.js b/jstests/core/shell_writeconcern.js index f3f190061cf..e3e7a23a9aa 100644 --- a/jstests/core/shell_writeconcern.js +++ b/jstests/core/shell_writeconcern.js @@ -1,7 +1,10 @@ "use strict"; + // check that shell writeconcern work correctly // 1.) tests that it can be set on each level and is inherited // 2.) tests that each operation (update/insert/remove/save) take and ensure a write concern +// +// @tags: [assumes_write_concern_unchanged] var collA = db.shell_wc_a; var collB = db.shell_wc_b; diff --git a/jstests/core/stages_delete.js b/jstests/core/stages_delete.js index f8e7380c75a..98a270cedf3 100644 --- a/jstests/core/stages_delete.js +++ b/jstests/core/stages_delete.js @@ -1,3 +1,9 @@ +// @tags: [ +// # This test attempts to remove documents using the stageDebug command, which doesn't support +// # specifying a writeConcern. +// assumes_write_concern_unchanged, +// ] + // Test basic delete stage functionality. var coll = db.stages_delete; var collScanStage = {cscan: {args: {direction: 1}, filter: {deleteMe: true}}}; diff --git a/jstests/core/startup_log.js b/jstests/core/startup_log.js index 3b0cbe3464d..c73013d1744 100644 --- a/jstests/core/startup_log.js +++ b/jstests/core/startup_log.js @@ -1,101 +1,108 @@ -load('jstests/aggregation/extras/utils.js');
-
-(function() {
- 'use strict';
-
- // Check that smallArray is entirely contained by largeArray
- // returns false if a member of smallArray is not in largeArray
- function arrayIsSubset(smallArray, largeArray) {
- for (var i = 0; i < smallArray.length; i++) {
- if (!Array.contains(largeArray, smallArray[i])) {
- print("Could not find " + smallArray[i] + " in largeArray");
- return false;
- }
- }
-
- return true;
- }
-
- // Test startup_log
- var stats = db.getSisterDB("local").startup_log.stats();
- assert(stats.capped);
-
- var latestStartUpLog =
- db.getSisterDB("local").startup_log.find().sort({$natural: -1}).limit(1).next();
- var serverStatus = db._adminCommand("serverStatus");
- var cmdLine = db._adminCommand("getCmdLineOpts").parsed;
-
- // Test that the startup log has the expected keys
- var verbose = false;
- var expectedKeys =
- ["_id", "hostname", "startTime", "startTimeLocal", "cmdLine", "pid", "buildinfo"];
- var keys = Object.keySet(latestStartUpLog);
- assert(arrayEq(expectedKeys, keys, verbose), 'startup_log keys failed');
-
- // Tests _id implicitly - should be comprised of host-timestamp
- // Setup expected startTime and startTimeLocal from the supplied timestamp
- var _id = latestStartUpLog._id.split('-'); // _id should consist of host-timestamp
- var _idUptime = _id.pop();
- var _idHost = _id.join('-');
- var uptimeSinceEpochRounded = Math.floor(_idUptime / 1000) * 1000;
- var startTime = new Date(uptimeSinceEpochRounded); // Expected startTime
-
- assert.eq(_idHost, latestStartUpLog.hostname, "Hostname doesn't match one from _id");
- assert.eq(serverStatus.host.split(':')[0],
- latestStartUpLog.hostname,
- "Hostname doesn't match one in server status");
- assert.closeWithinMS(startTime,
- latestStartUpLog.startTime,
- "StartTime doesn't match one from _id",
- 2000); // Expect less than 2 sec delta
- assert.eq(cmdLine, latestStartUpLog.cmdLine, "cmdLine doesn't match that from getCmdLineOpts");
- assert.eq(serverStatus.pid, latestStartUpLog.pid, "pid doesn't match that from serverStatus");
-
- // Test buildinfo
- var buildinfo = db.runCommand("buildinfo");
- delete buildinfo.ok; // Delete extra meta info not in startup_log
- var isMaster = db._adminCommand("ismaster");
-
- // Test buildinfo has the expected keys
- var expectedKeys = [
- "version",
- "gitVersion",
- "allocator",
- "versionArray",
- "javascriptEngine",
- "openssl",
- "buildEnvironment",
- "debug",
- "maxBsonObjectSize",
- "bits",
- "modules"
- ];
-
- var keys = Object.keySet(latestStartUpLog.buildinfo);
- // Disabled to check
- assert(arrayIsSubset(expectedKeys, keys),
- "buildinfo keys failed! \n expected:\t" + expectedKeys + "\n actual:\t" + keys);
- assert.eq(buildinfo,
- latestStartUpLog.buildinfo,
- "buildinfo doesn't match that from buildinfo command");
-
- // Test version and version Array
- var version = latestStartUpLog.buildinfo.version.split('-')[0];
- var versionArray = latestStartUpLog.buildinfo.versionArray;
- var versionArrayCleaned = versionArray.slice(0, 3);
- if (versionArray[3] == -100) {
- versionArrayCleaned[2] -= 1;
- }
-
- assert.eq(serverStatus.version,
- latestStartUpLog.buildinfo.version,
- "Mongo version doesn't match that from ServerStatus");
- assert.eq(
- version, versionArrayCleaned.join('.'), "version doesn't match that from the versionArray");
- var jsEngine = latestStartUpLog.buildinfo.javascriptEngine;
- assert((jsEngine == "none") || jsEngine.startsWith("mozjs"));
- assert.eq(isMaster.maxBsonObjectSize,
- latestStartUpLog.buildinfo.maxBsonObjectSize,
- "maxBsonObjectSize doesn't match one from ismaster");
-
-})();
+/** + * This test attempts to read from the "local.startup_log" collection and assert that it has an + * entry matching the server's response from the "getCmdLineOpts" command. The former operation may + * be routed to a secondary in the replica set, whereas the latter must be routed to the primary. + * + * @tags: [assumes_read_preference_unchanged] + */ +load('jstests/aggregation/extras/utils.js'); + +(function() { + 'use strict'; + + // Check that smallArray is entirely contained by largeArray + // returns false if a member of smallArray is not in largeArray + function arrayIsSubset(smallArray, largeArray) { + for (var i = 0; i < smallArray.length; i++) { + if (!Array.contains(largeArray, smallArray[i])) { + print("Could not find " + smallArray[i] + " in largeArray"); + return false; + } + } + + return true; + } + + // Test startup_log + var stats = db.getSisterDB("local").startup_log.stats(); + assert(stats.capped); + + var latestStartUpLog = + db.getSisterDB("local").startup_log.find().sort({$natural: -1}).limit(1).next(); + var serverStatus = db._adminCommand("serverStatus"); + var cmdLine = db._adminCommand("getCmdLineOpts").parsed; + + // Test that the startup log has the expected keys + var verbose = false; + var expectedKeys = + ["_id", "hostname", "startTime", "startTimeLocal", "cmdLine", "pid", "buildinfo"]; + var keys = Object.keySet(latestStartUpLog); + assert(arrayEq(expectedKeys, keys, verbose), 'startup_log keys failed'); + + // Tests _id implicitly - should be comprised of host-timestamp + // Setup expected startTime and startTimeLocal from the supplied timestamp + var _id = latestStartUpLog._id.split('-'); // _id should consist of host-timestamp + var _idUptime = _id.pop(); + var _idHost = _id.join('-'); + var uptimeSinceEpochRounded = Math.floor(_idUptime / 1000) * 1000; + var startTime = new Date(uptimeSinceEpochRounded); // Expected startTime + + assert.eq(_idHost, latestStartUpLog.hostname, "Hostname doesn't match one from _id"); + assert.eq(serverStatus.host.split(':')[0], + latestStartUpLog.hostname, + "Hostname doesn't match one in server status"); + assert.closeWithinMS(startTime, + latestStartUpLog.startTime, + "StartTime doesn't match one from _id", + 2000); // Expect less than 2 sec delta + assert.eq(cmdLine, latestStartUpLog.cmdLine, "cmdLine doesn't match that from getCmdLineOpts"); + assert.eq(serverStatus.pid, latestStartUpLog.pid, "pid doesn't match that from serverStatus"); + + // Test buildinfo + var buildinfo = db.runCommand("buildinfo"); + delete buildinfo.ok; // Delete extra meta info not in startup_log + var isMaster = db._adminCommand("ismaster"); + + // Test buildinfo has the expected keys + var expectedKeys = [ + "version", + "gitVersion", + "allocator", + "versionArray", + "javascriptEngine", + "openssl", + "buildEnvironment", + "debug", + "maxBsonObjectSize", + "bits", + "modules" + ]; + + var keys = Object.keySet(latestStartUpLog.buildinfo); + // Disabled to check + assert(arrayIsSubset(expectedKeys, keys), + "buildinfo keys failed! \n expected:\t" + expectedKeys + "\n actual:\t" + keys); + assert.eq(buildinfo, + latestStartUpLog.buildinfo, + "buildinfo doesn't match that from buildinfo command"); + + // Test version and version Array + var version = latestStartUpLog.buildinfo.version.split('-')[0]; + var versionArray = latestStartUpLog.buildinfo.versionArray; + var versionArrayCleaned = versionArray.slice(0, 3); + if (versionArray[3] == -100) { + versionArrayCleaned[2] -= 1; + } + + assert.eq(serverStatus.version, + latestStartUpLog.buildinfo.version, + "Mongo version doesn't match that from ServerStatus"); + assert.eq( + version, versionArrayCleaned.join('.'), "version doesn't match that from the versionArray"); + var jsEngine = latestStartUpLog.buildinfo.javascriptEngine; + assert((jsEngine == "none") || jsEngine.startsWith("mozjs")); + assert.eq(isMaster.maxBsonObjectSize, + latestStartUpLog.buildinfo.maxBsonObjectSize, + "maxBsonObjectSize doesn't match one from ismaster"); + +})(); diff --git a/jstests/core/storefunc.js b/jstests/core/storefunc.js index 8598e9cc62b..15abc56421e 100644 --- a/jstests/core/storefunc.js +++ b/jstests/core/storefunc.js @@ -1,3 +1,5 @@ +// @tags: [requires_eval_command] + // Use a private sister database to avoid conflicts with other tests that use system.js var testdb = db.getSisterDB("storefunc"); var res; diff --git a/jstests/core/top.js b/jstests/core/top.js index 819b41b0981..3d98f5a7b2d 100644 --- a/jstests/core/top.js +++ b/jstests/core/top.js @@ -1,5 +1,11 @@ /** * 1. check top numbers are correct + * + * This test attempts to perform read operations and get statistics using the top command. The + * former operation may be routed to a secondary in the replica set, whereas the latter must be + * routed to the primary. + * + * @tags: [assumes_read_preference_unchanged] */ (function() { load("jstests/libs/stats.js"); diff --git a/jstests/core/update_affects_indexes.js b/jstests/core/update_affects_indexes.js new file mode 100644 index 00000000000..91db3ebe565 --- /dev/null +++ b/jstests/core/update_affects_indexes.js @@ -0,0 +1,100 @@ +// This is a regression test for SERVER-32048. It checks that index keys are correctly updated when +// an update modifier implicitly creates a new array element. +(function() { + "use strict"; + + let coll = db.update_affects_indexes; + coll.drop(); + let indexKeyPattern = {"a.b": 1}; + assert.commandWorked(coll.createIndex(indexKeyPattern)); + + // Tests that the document 'docId' has all the index keys in 'expectedKeys' and none of the + // index keys in 'unexpectedKeys'. + function assertExpectedIndexKeys(docId, expectedKeys, unexpectedKeys) { + for (let key of expectedKeys) { + let res = coll.find(docId).hint(indexKeyPattern).min(key).returnKey().toArray(); + assert.eq(1, res.length, tojson(res)); + assert.eq(key, res[0]); + } + + for (let key of unexpectedKeys) { + let res = coll.find(docId).hint(indexKeyPattern).min(key).returnKey().toArray(); + if (res.length > 0) { + assert.eq(1, res.length, tojson(res)); + assert.neq(key, res[0]); + } + } + } + + // $set implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 0, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 0}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 0}, {$set: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 0}, [{"a.b": 0}, {"a.b": null}], []); + + // $set implicitly creates array element beyond end of array. + assert.writeOK(coll.insert({_id: 1, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 1}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 1}, {$set: {"a.3.c": 0}})); + assertExpectedIndexKeys({_id: 1}, [{"a.b": 0}, {"a.b": null}], []); + + // $set implicitly creates array element in empty array (no index key changes needed). + assert.writeOK(coll.insert({_id: 2, a: []})); + assertExpectedIndexKeys({_id: 2}, [{"a.b": null}], []); + assert.writeOK(coll.update({_id: 2}, {$set: {"a.0.c": 0}})); + assertExpectedIndexKeys({_id: 2}, [{"a.b": null}], []); + + // $inc implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 3, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 3}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 3}, {$inc: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 3}, [{"a.b": 0}, {"a.b": null}], []); + + // $mul implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 4, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 4}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 4}, {$mul: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 4}, [{"a.b": 0}, {"a.b": null}], []); + + // $addToSet implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 5, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 5}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 5}, {$addToSet: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 5}, [{"a.b": 0}, {"a.b": null}], []); + + // $bit implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 6, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 6}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 6}, {$bit: {"a.1.c": {and: NumberInt(1)}}})); + assertExpectedIndexKeys({_id: 6}, [{"a.b": 0}, {"a.b": null}], []); + + // $min implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 7, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 7}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 7}, {$min: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 7}, [{"a.b": 0}, {"a.b": null}], []); + + // $max implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 8, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 8}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 8}, {$max: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 8}, [{"a.b": 0}, {"a.b": null}], []); + + // $currentDate implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 9, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 9}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 9}, {$currentDate: {"a.1.c": true}})); + assertExpectedIndexKeys({_id: 9}, [{"a.b": 0}, {"a.b": null}], []); + + // $push implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 10, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 10}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 10}, {$push: {"a.1.c": 0}})); + assertExpectedIndexKeys({_id: 10}, [{"a.b": 0}, {"a.b": null}], []); + + // $pushAll implicitly creates array element at end of array. + assert.writeOK(coll.insert({_id: 11, a: [{b: 0}]})); + assertExpectedIndexKeys({_id: 11}, [{"a.b": 0}], [{"a.b": null}]); + assert.writeOK(coll.update({_id: 11}, {$pushAll: {"a.1.c": [0]}})); + assertExpectedIndexKeys({_id: 11}, [{"a.b": 0}, {"a.b": null}], []); +}()); diff --git a/jstests/core/update_multi5.js b/jstests/core/update_multi5.js index e610462a620..9e9550bc554 100644 --- a/jstests/core/update_multi5.js +++ b/jstests/core/update_multi5.js @@ -7,8 +7,8 @@ assert.writeOK(t.insert({path: 'r1', subscribers: [1, 2]})); assert.writeOK(t.insert({path: 'r2', subscribers: [3, 4]})); - var res = assert.writeOK(t.update( - {}, {$addToSet: {subscribers: 5}}, {upsert: false, multi: true, writeConcern: {w: 1}})); + var res = + assert.writeOK(t.update({}, {$addToSet: {subscribers: 5}}, {upsert: false, multi: true})); assert.eq(res.nMatched, 2, tojson(res)); diff --git a/jstests/core/views/invalid_system_views.js b/jstests/core/views/invalid_system_views.js index 3ba282d2ca1..49ac600ba8b 100644 --- a/jstests/core/views/invalid_system_views.js +++ b/jstests/core/views/invalid_system_views.js @@ -1,6 +1,8 @@ /** * Tests that invalid view definitions in system.views do not impact valid commands on existing * collections. + * + * @tags: [requires_collmod_command] */ (function() { "use strict"; diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index 0115672dcda..e1798803a1b 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -48,6 +48,8 @@ * * skipStandalone * If true, do not run this command on a standalone mongod. + * + * @tags: [requires_collmod_command] */ (function() { diff --git a/jstests/core/views/views_basic.js b/jstests/core/views/views_basic.js index d6032b105df..b4a2f177982 100644 --- a/jstests/core/views/views_basic.js +++ b/jstests/core/views/views_basic.js @@ -13,7 +13,7 @@ let res = viewsDB.runCommand(cmd); assert.commandWorked(res); - let cursor = new DBCommandCursor(db.getMongo(), res, 5); + let cursor = new DBCommandCursor(res._mongo, res, 5); let actual = cursor.toArray(); assert(arrayEq(actual, expected), "actual: " + tojson(cursor.toArray()) + ", expected:" + tojson(expected)); diff --git a/jstests/core/views/views_change.js b/jstests/core/views/views_change.js index 002284095c5..a62e7e7f04f 100644 --- a/jstests/core/views/views_change.js +++ b/jstests/core/views/views_change.js @@ -1,6 +1,7 @@ /** * Tests the behavior of views when the backing view or collection is changed. - * @tags: [requires_find_command] + * + * @tags: [requires_collmod_command, requires_find_command] */ (function() { "use strict"; diff --git a/jstests/core/views/views_collation.js b/jstests/core/views/views_collation.js index 9e4ed7feb30..38557358ce9 100644 --- a/jstests/core/views/views_collation.js +++ b/jstests/core/views/views_collation.js @@ -1,9 +1,13 @@ /** * Tests the behavior of operations when interacting with a view's default collation. + * + * @tags: [requires_collmod_command] */ (function() { "use strict"; + load("jstests/libs/analyze_plan.js"); + let viewsDB = db.getSiblingDB("views_collation"); assert.commandWorked(viewsDB.dropDatabase()); assert.commandWorked(viewsDB.runCommand({create: "simpleCollection"})); @@ -58,6 +62,15 @@ assert.commandWorked(viewsDB.runCommand({count: "filView"})); assert.commandWorked(viewsDB.runCommand({distinct: "filView", key: "x"})); + // Explain of operations that do not specify a collation succeed. + assert.commandWorked(viewsDB.runCommand({aggregate: "filView", pipeline: [], explain: true})); + assert.commandWorked( + viewsDB.runCommand({explain: {find: "filView"}, verbosity: "allPlansExecution"})); + assert.commandWorked( + viewsDB.runCommand({explain: {count: "filView"}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand( + {explain: {distinct: "filView", key: "x"}, verbosity: "allPlansExecution"})); + // Operations with a matching collation succeed. assert.commandWorked( viewsDB.runCommand({aggregate: "filView", pipeline: [], collation: {locale: "fil"}})); @@ -66,6 +79,18 @@ assert.commandWorked( viewsDB.runCommand({distinct: "filView", key: "x", collation: {locale: "fil"}})); + // Explain of operations with a matching collation succeed. + assert.commandWorked(viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "fil"}})); + assert.commandWorked(viewsDB.runCommand( + {explain: {find: "filView", collation: {locale: "fil"}}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand( + {explain: {count: "filView", collation: {locale: "fil"}}, verbosity: "allPlansExecution"})); + assert.commandWorked(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "fil"}}, + verbosity: "allPlansExecution" + })); + // Attempting to override the non-simple default collation of a view fails. assert.commandFailedWithCode( viewsDB.runCommand({aggregate: "filView", pipeline: [], collation: {locale: "en"}}), @@ -90,6 +115,46 @@ viewsDB.runCommand({distinct: "filView", key: "x", collation: {locale: "simple"}}), ErrorCodes.OptionNotSupportedOnView); + // Attempting to override the default collation of a view with explain fails. + assert.commandFailedWithCode( + viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "en"}}), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode( + viewsDB.runCommand( + {aggregate: "filView", pipeline: [], explain: true, collation: {locale: "simple"}}), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {find: "filView", collation: {locale: "fr"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {find: "filView", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {count: "filView", collation: {locale: "zh"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {count: "filView", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "es"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + assert.commandFailedWithCode(viewsDB.runCommand({ + explain: {distinct: "filView", key: "x", collation: {locale: "simple"}}, + verbosity: "allPlansExecution" + }), + ErrorCodes.OptionNotSupportedOnView); + const lookupSimpleView = { $lookup: {from: "simpleView", localField: "x", foreignField: "x", as: "result"} }; @@ -253,4 +318,56 @@ viewsDB.runCommand( {collMod: "esView", viewOn: "simpleCollection", pipeline: [graphLookupFilView]}), ErrorCodes.OptionNotSupportedOnView); + + // Make sure that when an operation does not specify the collation, it correctly uses the + // default collation associated with the view. For this, we set up a new backing collection with + // a case-insensitive view. + assert.commandWorked(viewsDB.runCommand({create: "case_sensitive_coll"})); + assert.commandWorked(viewsDB.runCommand({ + create: "case_insensitive_view", + viewOn: "case_sensitive_coll", + collation: {locale: "en", strength: 1} + })); + + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "case"})); + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "Case"})); + assert.writeOK(viewsDB.case_sensitive_coll.insert({f: "CASE"})); + + let explain, cursorStage; + + // Test that aggregate against a view with a default collation correctly uses the collation. + assert.eq(1, viewsDB.case_sensitive_coll.aggregate([{$match: {f: "case"}}]).itcount()); + assert.eq(3, viewsDB.case_insensitive_view.aggregate([{$match: {f: "case"}}]).itcount()); + explain = viewsDB.case_insensitive_view.explain().aggregate([{$match: {f: "case"}}]); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that count against a view with a default collation correctly uses the collation. + assert.eq(1, viewsDB.case_sensitive_coll.count({f: "case"})); + assert.eq(3, viewsDB.case_insensitive_view.count({f: "case"})); + explain = viewsDB.case_insensitive_view.explain().count({f: "case"}); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that distinct against a view with a default collation correctly uses the collation. + assert.eq(3, viewsDB.case_sensitive_coll.distinct("f").length); + assert.eq(1, viewsDB.case_insensitive_view.distinct("f").length); + explain = viewsDB.case_insensitive_view.explain().distinct("f"); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); + + // Test that find against a view with a default collation correctly uses the collation. + let findRes = viewsDB.runCommand({find: "case_sensitive_coll", filter: {f: "case"}}); + assert.commandWorked(findRes); + assert.eq(1, findRes.cursor.firstBatch.length); + findRes = viewsDB.runCommand({find: "case_insensitive_view", filter: {f: "case"}}); + assert.commandWorked(findRes); + assert.eq(3, findRes.cursor.firstBatch.length); + explain = viewsDB.runCommand({explain: {find: "case_insensitive_view", filter: {f: "case"}}}); + cursorStage = getAggPlanStage(explain, "$cursor"); + assert.neq(null, cursorStage, tojson(explain)); + assert.eq(1, cursorStage.$cursor.queryPlanner.collation.strength, tojson(cursorStage)); }()); diff --git a/jstests/core/views/views_find.js b/jstests/core/views/views_find.js index c196e980ce5..53c9d4e8b98 100644 --- a/jstests/core/views/views_find.js +++ b/jstests/core/views/views_find.js @@ -15,7 +15,7 @@ let assertFindResultEq = function(cmd, expected, ordered) { let res = viewsDB.runCommand(cmd); assert.commandWorked(res); - let arr = new DBCommandCursor(db.getMongo(), res, 5).toArray(); + let arr = new DBCommandCursor(res._mongo, res, 5).toArray(); let errmsg = tojson({expected: expected, got: arr}); if (typeof(ordered) === "undefined" || !ordered) diff --git a/jstests/core/views/views_rename.js b/jstests/core/views/views_rename.js new file mode 100644 index 00000000000..ad656671ea1 --- /dev/null +++ b/jstests/core/views/views_rename.js @@ -0,0 +1,17 @@ +(function() { + // SERVER-30406 Test that renaming system.views correctly invalidates the view catalog + 'use strict'; + db.view.drop(); + db.coll.drop(); + assert.commandWorked(db.createView("view", "coll", [])); + assert.writeOK(db.coll.insert({_id: 1})); + assert.eq(db.view.find().count(), 1, "couldn't find document in view"); + assert.commandWorked(db.system.views.renameCollection("views", /*dropTarget*/ true)); + assert.eq(db.view.find().count(), + 0, + "find on view should have returned no results after renaming away system.views"); + assert.commandWorked(db.views.renameCollection("system.views")); + assert.eq(db.view.find().count(), + 1, + "find on view should have worked again after renaming system.views back in place"); +})(); diff --git a/jstests/core/views/views_stats.js b/jstests/core/views/views_stats.js index 75feb857c9a..22261e9fa81 100644 --- a/jstests/core/views/views_stats.js +++ b/jstests/core/views/views_stats.js @@ -1,4 +1,10 @@ // Test that top and latency histogram statistics are recorded for views. +// +// This test attempts to perform write operations and get latency statistics using the $collStats +// stage. The former operation must be routed to the primary in a replica set, whereas the latter +// may be routed to a secondary. +// +// @tags: [assumes_read_preference_unchanged] (function() { "use strict"; diff --git a/jstests/core/views/views_validation.js b/jstests/core/views/views_validation.js index 84c7f1d3510..b32750a082f 100644 --- a/jstests/core/views/views_validation.js +++ b/jstests/core/views/views_validation.js @@ -1,3 +1,4 @@ +// @tags: [requires_collmod_command] (function() { "use strict"; let viewsDb = db.getSiblingDB("views_validation"); diff --git a/jstests/core/write_result.js b/jstests/core/write_result.js index 86486089c68..453be4ca1c9 100644 --- a/jstests/core/write_result.js +++ b/jstests/core/write_result.js @@ -1,6 +1,8 @@ // // Tests the behavior of single writes using write commands // +// @tags: [assumes_write_concern_unchanged] +// var coll = db.write_result; coll.drop(); diff --git a/jstests/hooks/validate_collections.js b/jstests/hooks/validate_collections.js index aeb38a98bf5..e64bb39d305 100644 --- a/jstests/hooks/validate_collections.js +++ b/jstests/hooks/validate_collections.js @@ -71,6 +71,21 @@ function validateCollections(db, obj) { filter = {$or: [filter, {type: {$exists: false}}]}; } + // Optionally skip collections. + if (Array.isArray(jsTest.options().skipValidationNamespaces) && + jsTest.options().skipValidationNamespaces.length > 0) { + let skippedCollections = []; + for (let ns of jsTest.options().skipValidationNamespaces) { + // Strip off the database name from 'ns' to extract the collName. + const collName = ns.replace(new RegExp('^' + db.getName() + '\.'), ''); + // Skip the collection 'collName' if the db name was removed from 'ns'. + if (collName !== ns) { + skippedCollections.push({name: {$ne: collName}}); + } + } + filter = {$and: [filter, ...skippedCollections]}; + } + let collInfo = db.getCollectionInfos(filter); for (var collDocument of collInfo) { var coll = db.getCollection(collDocument["name"]); diff --git a/jstests/libs/analyze_plan.js b/jstests/libs/analyze_plan.js index 62511ae1fac..5e52ad6aba3 100644 --- a/jstests/libs/analyze_plan.js +++ b/jstests/libs/analyze_plan.js @@ -115,3 +115,74 @@ function getChunkSkips(root) { return 0; } + +/** + * Given the root stage of agg explain's JSON representation of a query plan ('root'), returns all + * subdocuments whose stage is 'stage'. This can either be an agg stage name like "$cursor" or + * "$sort", or a query stage name like "IXSCAN" or "SORT". + * + * Returns an empty array if the plan does not have the requested stage. Asserts that agg explain + * structure matches expected format. + */ +function getAggPlanStages(root, stage) { + let results = []; + + function getDocumentSources(docSourceArray) { + let results = []; + for (let i = 0; i < docSourceArray.length; i++) { + let properties = Object.getOwnPropertyNames(docSourceArray[i]); + assert.eq(1, properties.length); + if (properties[0] === stage) { + results.push(docSourceArray[i]); + } + } + return results; + } + + if (root.hasOwnProperty("stages")) { + assert(root.stages.constructor === Array); + + results = results.concat(getDocumentSources(root.stages)); + + assert(root.stages[0].hasOwnProperty("$cursor")); + assert(root.stages[0].$cursor.hasOwnProperty("queryPlanner")); + assert(root.stages[0].$cursor.queryPlanner.hasOwnProperty("winningPlan")); + results = + results.concat(getPlanStages(root.stages[0].$cursor.queryPlanner.winningPlan, stage)); + } + + if (root.hasOwnProperty("shards")) { + for (let elem in root.shards) { + assert(root.shards[elem].stages.constructor === Array); + + results = results.concat(getDocumentSources(root.shards[elem].stages)); + + assert(root.shards[elem].stages[0].hasOwnProperty("$cursor")); + assert(root.shards[elem].stages[0].$cursor.hasOwnProperty("queryPlanner")); + assert(root.shards[elem].stages[0].$cursor.queryPlanner.hasOwnProperty("winningPlan")); + results = results.concat( + getPlanStages(root.shards[elem].stages[0].$cursor.queryPlanner.winningPlan, stage)); + } + } + + return results; +} + +/** + * Given the root stage of agg explain's JSON representation of a query plan ('root'), returns the + * subdocument with its stage as 'stage'. Returns null if the plan does not have such a stage. + * Asserts that no more than one stage is a match. + */ +function getAggPlanStage(root, stage) { + let planStageList = getAggPlanStages(root, stage); + + if (planStageList.length === 0) { + return null; + } else { + assert.eq(1, + planStageList.length, + "getAggPlanStage expects to find 0 or 1 matching stages. planStageList: " + + tojson(planStageList)); + return planStageList[0]; + } +} diff --git a/jstests/libs/ftdc.js b/jstests/libs/ftdc.js new file mode 100644 index 00000000000..7d327d39852 --- /dev/null +++ b/jstests/libs/ftdc.js @@ -0,0 +1,102 @@ +/** + * Utility test functions for FTDC + */ +'use strict'; + +/** + * Verify that getDiagnosticData is working correctly. + */ +function verifyGetDiagnosticData(adminDb) { + // We need to retry a few times if run this test immediately after mongod is started as FTDC may + // not have run yet. + var foundGoodDocument = false; + + for (var i = 0; i < 60 && foundGoodDocument == false; ++i) { + var result = adminDb.runCommand("getDiagnosticData"); + assert.commandWorked(result); + + var data = result.data; + + if (!data.hasOwnProperty("start")) { + // Wait a little longer for FTDC to start + jsTestLog("Running getDiagnosticData: " + tojson(result)); + + sleep(500); + } else { + // Check for a few common properties to ensure we got data + assert(data.hasOwnProperty("serverStatus"), + "does not have 'serverStatus' in '" + tojson(data) + "'"); + assert(data.hasOwnProperty("end"), "does not have 'end' in '" + tojson(data) + "'"); + foundGoodDocument = true; + + jsTestLog("Got good getDiagnosticData: " + tojson(result)); + } + } + + assert(foundGoodDocument, + "getDiagnosticData failed to return a non-empty command, is FTDC running?"); +} + +/** + * Validate all the common FTDC parameters are set correctly and can be manipulated. + */ +function verifyCommonFTDCParameters(adminDb, isEnabled) { + // Are we running against MongoS? + var isMongos = ("isdbgrid" == adminDb.runCommand("ismaster").msg); + + // Check the defaults are correct + // + function getparam(field) { + var q = {getParameter: 1}; + q[field] = 1; + + var ret = adminDb.runCommand(q); + return ret[field]; + } + + // Verify the defaults are as we documented them + assert.eq(getparam("diagnosticDataCollectionEnabled"), isEnabled); + assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); + assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); + assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); + assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); + assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); + + function setparam(obj) { + var ret = adminDb.runCommand(Object.extend({setParameter: 1}, obj)); + return ret; + } + + if (!isMongos) { + // The MongoS specific behavior for diagnosticDataCollectionEnabled is tested in + // ftdc_setdirectory.js. + assert.commandWorked(setparam({"diagnosticDataCollectionEnabled": 1})); + } + assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 1})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 2})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 2})); + + // Negative tests - set values below minimums + assert.commandFailed(setparam({"diagnosticDataCollectionPeriodMillis": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerChunk": 1})); + assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 1})); + + // Negative test - set file size bigger then directory size + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + assert.commandFailed(setparam({"diagnosticDataCollectionFileSizeMB": 100})); + + // Negative test - set directory size less then file size + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 50})); + assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); + + // Reset + assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); + assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); + assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); + assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); +}
\ No newline at end of file diff --git a/jstests/libs/override_methods/override_helpers.js b/jstests/libs/override_methods/override_helpers.js new file mode 100644 index 00000000000..12bd4b54739 --- /dev/null +++ b/jstests/libs/override_methods/override_helpers.js @@ -0,0 +1,117 @@ +/** + * The OverrideHelpers object defines convenience methods for overriding commands and functions in + * the mongo shell. + */ +var OverrideHelpers = (function() { + "use strict"; + + function isAggregationWithOutStage(commandName, commandObj) { + if (commandName !== "aggregate" || typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (!Array.isArray(commandObj.pipeline) || commandObj.pipeline.length === 0) { + return false; + } + + const lastStage = commandObj.pipeline[commandObj.pipeline.length - 1]; + if (typeof lastStage !== "object" || lastStage === null) { + return false; + } + + return Object.keys(lastStage)[0] === "$out"; + } + + function isMapReduceWithInlineOutput(commandName, commandObj) { + if ((commandName !== "mapReduce" && commandName !== "mapreduce") || + typeof commandObj !== "object" || commandObj === null) { + return false; + } + + if (typeof commandObj.out !== "object") { + return false; + } + + return commandObj.out.hasOwnProperty("inline"); + } + + function prependOverrideInParallelShell(overrideFile) { + const startParallelShellOriginal = startParallelShell; + + startParallelShell = function(jsCode, port, noConnect) { + let newCode; + if (typeof jsCode === "function") { + // Load the override file and immediately invoke the supplied function. + newCode = `load("${overrideFile}"); (${jsCode})();`; + } else { + newCode = `load("${overrideFile}"); ${jsCode};`; + } + + return startParallelShellOriginal(newCode, port, noConnect); + }; + } + + function overrideRunCommand(overrideFunc) { + const DBQueryOriginal = DBQuery; + const mongoRunCommandOriginal = Mongo.prototype.runCommand; + const mongoRunCommandWithMetadataOriginal = Mongo.prototype.runCommandWithMetadata; + + DBQuery = function( + mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { + // If the query isn't being run against the "$cmd" or "$cmd.sys" namespaces, then it + // represents an OP_QUERY find on that collection. We skip calling overrideFunc() in + // this case because the operation doesn't represent a command. + if (!(collection instanceof DBCollection && + (collection.getName() === "$cmd" || collection.getName().startsWith("$cmd.")))) { + return DBQueryOriginal.apply(this, arguments); + } + + // Due to the function signatures of Mongo.prototype.runCommand() and + // Mongo.prototype.runCommandWithMetadata(), the overrideFunc() function expects that + // the Mongo connection object is passed as the first argument and also represents the + // 'this' parameter. As a workaround, we bind the appropriate 'this' value to the + // DBQueryOriginal constructor ahead of time. + const commandName = Object.keys(query)[0]; + return overrideFunc( + mongo, + db.getName(), + commandName, + query, + DBQueryOriginal.bind(this), + (query) => + [mongo, db, collection, ns, query, fields, limit, skip, batchSize, options]); + }; + + // Copy any properties (e.g. DBQuery.Option) that are set on DBQueryOriginal. + Object.keys(DBQueryOriginal).forEach(function(key) { + DBQuery[key] = DBQueryOriginal[key]; + }); + + Mongo.prototype.runCommand = function(dbName, commandObj, options) { + const commandName = Object.keys(commandObj)[0]; + return overrideFunc(this, + dbName, + commandName, + commandObj, + mongoRunCommandOriginal, + (commandObj) => [dbName, commandObj, options]); + }; + + Mongo.prototype.runCommandWithMetadata = function(dbName, metadata, commandArgs) { + const commandName = Object.keys(commandArgs)[0]; + return overrideFunc(this, + dbName, + commandName, + commandArgs, + mongoRunCommandWithMetadataOriginal, + (commandArgs) => [dbName, metadata, commandArgs]); + }; + } + + return { + isAggregationWithOutStage: isAggregationWithOutStage, + isMapReduceWithInlineOutput: isMapReduceWithInlineOutput, + prependOverrideInParallelShell: prependOverrideInParallelShell, + overrideRunCommand: overrideRunCommand, + }; +})(); diff --git a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js b/jstests/libs/override_methods/set_majority_read_and_write_concerns.js deleted file mode 100644 index d3bb4449ed4..00000000000 --- a/jstests/libs/override_methods/set_majority_read_and_write_concerns.js +++ /dev/null @@ -1,178 +0,0 @@ -/** - * Use prototype overrides to set a read concern of "majority" and a write concern of "majority" - * while running core tests. - */ -(function() { - "use strict"; - var defaultWriteConcern = { - w: "majority", - // Use a "signature" value that won't typically match a value assigned in normal use. - wtimeout: 60321 - }; - var defaultReadConcern = {level: "majority"}; - - var originalDBQuery = DBQuery; - - DBQuery = function(mongo, db, collection, ns, query, fields, limit, skip, batchSize, options) { - if (ns.endsWith("$cmd")) { - if (query.hasOwnProperty("writeConcern") && - bsonWoCompare(query.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: DBQuery overriding existing writeConcern of: " + - tojson(query.writeConcern)); - query.writeConcern = defaultWriteConcern; - } - } - - return originalDBQuery.apply(this, arguments); - }; - - DBQuery.Option = originalDBQuery.Option; - - var originalStartParallelShell = startParallelShell; - startParallelShell = function(jsCode, port, noConnect) { - var newCode; - var overridesFile = "jstests/libs/override_methods/set_majority_read_and_write_concerns.js"; - - if (typeof(jsCode) === "function") { - // Load the override file and immediately invoke the supplied function. - newCode = `load("${overridesFile}"); (${jsCode})();`; - } else { - newCode = `load("${overridesFile}"); ${jsCode};`; - } - - return originalStartParallelShell(newCode, port, noConnect); - }; - - DB.prototype._runCommandImpl = function(dbName, obj, options) { - var cmdName = ""; - for (var fieldName in obj) { - cmdName = fieldName; - break; - } - - // These commands directly support a writeConcern argument. - var commandsToForceWriteConcern = [ - "_mergeAuthzCollections", - "appendOplogNote", - "applyOps", - "authSchemaUpgrade", - "captrunc", - "cleanupOrphaned", - "clone", - "cloneCollection", - "cloneCollectionAsCapped", - // "collMod", SERVER-25196 - not supported - "convertToCapped", - "copydb", - "create", - "createIndexes", - "createRole", - "createUser", - "delete", - "drop", - "dropDatabase", - "dropAllRolesFromDatabase", - "dropAllUsersFromDatabase", - "dropDatabase", - "dropIndexes", - "dropRole", - "dropUser", - "emptycapped", - "findAndModify", - "findandmodify", - "godinsert", - "grantPrivilegesToRole", - "grantRolesToRole", - "grantRolesToUser", - "insert", - "mapReduceFinish", - "mergeAuthzCollections", - "moveChunk", - "movePrimary", - "reIndex", - "remove", - "renameCollection", - "resvChunkStart", - "revokePriviligesFromRole", - "revokeRolesFromRole", - "revokeRolesFromUser", - "update", - "updateRole", - "updateUser", - ]; - - // These are reading commands that support majority readConcern. - var commandsToForceReadConcern = [ - "count", - "distinct", - "find", - "geoNear", - "geoSearch", - "group", - ]; - - var forceWriteConcern = Array.contains(commandsToForceWriteConcern, cmdName); - var forceReadConcern = Array.contains(commandsToForceReadConcern, cmdName); - - if (cmdName === "aggregate") { - // Aggregate can be either a read or a write depending on whether it has a $out stage. - // $out is required to be the last stage of the pipeline. - var stages = obj.pipeline; - const lastStage = stages && Array.isArray(stages) && (stages.length !== 0) - ? stages[stages.length - 1] - : undefined; - const hasOut = - lastStage && (typeof lastStage === 'object') && lastStage.hasOwnProperty('$out'); - if (hasOut) { - forceWriteConcern = true; - } else { - forceReadConcern = true; - } - } - - else if (cmdName === "mapReduce") { - var stages = obj.pipeline; - const lastStage = stages && Array.isArray(stages) && (stages.length !== 0) - ? stages[stages.length - 1] - : undefined; - const hasOut = - lastStage && (typeof lastStage === 'object') && lastStage.hasOwnProperty('$out'); - if (hasOut) { - forceWriteConcern = true; - } - } - - if (forceWriteConcern) { - if (obj.hasOwnProperty("writeConcern")) { - if (bsonWoCompare(obj.writeConcern, defaultWriteConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing writeConcern of: " + - tojson(obj.writeConcern)); - obj.writeConcern = defaultWriteConcern; - } - } else { - obj.writeConcern = defaultWriteConcern; - } - - } else if (forceReadConcern) { - if (obj.hasOwnProperty("readConcern")) { - if (bsonWoCompare(obj.readConcern, defaultReadConcern) !== 0) { - jsTestLog("Warning: _runCommandImpl overriding existing readConcern of: " + - tojson(obj.readConcern)); - obj.readConcern = defaultReadConcern; - } - } else { - obj.readConcern = defaultReadConcern; - } - } - - var res = this.getMongo().runCommand(dbName, obj, options); - - return res; - }; - - // Use a majority write concern if the operation does not specify one. - DBCollection.prototype.getWriteConcern = function() { - return new WriteConcern(defaultWriteConcern); - }; - -})(); diff --git a/jstests/libs/override_methods/set_read_and_write_concerns.js b/jstests/libs/override_methods/set_read_and_write_concerns.js new file mode 100644 index 00000000000..f371644c8f7 --- /dev/null +++ b/jstests/libs/override_methods/set_read_and_write_concerns.js @@ -0,0 +1,216 @@ +/** + * Use prototype overrides to set read concern and write concern while running tests. + */ +(function() { + "use strict"; + + load("jstests/libs/override_methods/override_helpers.js"); + + if (typeof TestData === "undefined" || !TestData.hasOwnProperty("defaultReadConcernLevel")) { + throw new Error( + "The readConcern level to use must be set as the 'defaultReadConcernLevel'" + + " property on the global TestData object"); + } + + const kDefaultReadConcern = {level: TestData.defaultReadConcernLevel}; + const kDefaultWriteConcern = + (TestData.hasOwnProperty("defaultWriteConcern")) ? TestData.defaultWriteConcern : { + w: "majority", + // Use a "signature" value that won't typically match a value assigned in normal use. + // This way the wtimeout set by this override is distinguishable in the server logs. + wtimeout: 5 * 60 * 1000 + 321, // 300321ms + }; + + const kCommandsSupportingReadConcern = new Set([ + "aggregate", + "count", + "distinct", + "find", + "geoNear", + "geoSearch", + "group", + "parallelCollectionScan", + ]); + + const kCommandsSupportingWriteConcern = new Set([ + "_configsvrAddShard", + "_configsvrAddShardToZone", + "_configsvrCommitChunkMerge", + "_configsvrCommitChunkMigration", + "_configsvrCommitChunkSplit", + "_configsvrCreateDatabase", + "_configsvrEnableSharding", + "_configsvrMoveChunk", + "_configsvrMovePrimary", + "_configsvrRemoveShard", + "_configsvrRemoveShardFromZone", + "_configsvrShardCollection", + "_configsvrUpdateZoneKeyRange", + "_mergeAuthzCollections", + "_recvChunkStart", + "appendOplogNote", + "applyOps", + "authSchemaUpgrade", + "aggregate", + "captrunc", + "cleanupOrphaned", + "clone", + "cloneCollection", + "cloneCollectionAsCapped", + // "collMod", SERVER-25196 - not supported + "convertToCapped", + "copydb", + "create", + "createIndexes", + "createRole", + "createUser", + "delete", + "deleteIndexes", + "drop", + "dropAllRolesFromDatabase", + "dropAllUsersFromDatabase", + "dropDatabase", + "dropIndexes", + "dropRole", + "dropUser", + "emptycapped", + "findAndModify", + "findandmodify", + "godinsert", + "grantPrivilegesToRole", + "grantRolesToRole", + "grantRolesToUser", + "insert", + "mapReduce", + "mapreduce", + "mapreduce.shardedfinish", + "moveChunk", + "renameCollection", + "revokePrivilegesFromRole", + "revokeRolesFromRole", + "revokeRolesFromUser", + "setFeatureCompatibilityVersion", + "update", + "updateRole", + "updateUser", + ]); + + function runCommandWithReadAndWriteConcerns( + conn, dbName, commandName, commandObj, func, makeFuncArgs) { + if (typeof commandObj !== "object" || commandObj === null) { + return func.apply(conn, makeFuncArgs(commandObj)); + } + + // If the command is in a wrapped form, then we look for the actual command object inside + // the query/$query object. + let commandObjUnwrapped = commandObj; + if (commandName === "query" || commandName === "$query") { + commandObjUnwrapped = commandObj[commandName]; + commandName = Object.keys(commandObjUnwrapped)[0]; + } + + if (commandName === "collMod" || commandName === "eval" || commandName === "$eval") { + throw new Error("Cowardly refusing to run test with overridden write concern when it" + + " uses a command that can only perform w=1 writes: " + + tojson(commandObj)); + } + + let shouldForceReadConcern = kCommandsSupportingReadConcern.has(commandName); + let shouldForceWriteConcern = kCommandsSupportingWriteConcern.has(commandName); + + if (commandName === "aggregate") { + if (OverrideHelpers.isAggregationWithOutStage(commandName, commandObjUnwrapped)) { + // The $out stage can only be used with readConcern={level: "local"}. + shouldForceReadConcern = false; + } else { + // A writeConcern can only be used with a $out stage. + shouldForceWriteConcern = false; + } + + if (commandObjUnwrapped.explain) { + // Attempting to specify a readConcern while explaining an aggregation would always + // return an error prior to SERVER-30582 and it otherwise only compatible with + // readConcern={level: "local"}. + shouldForceReadConcern = false; + } + } else if (OverrideHelpers.isMapReduceWithInlineOutput(commandName, commandObjUnwrapped)) { + // A writeConcern can only be used with non-inline output. + shouldForceWriteConcern = false; + } else if (commandObj[commandName] === "system.profile") { + // Writes to the "system.profile" collection aren't guaranteed to be visible in the same + // majority-committed snapshot as the command they originated from. We don't override + // the readConcern for operations on the "system.profile" collection so that tests which + // assert on its contents continue to succeed. + shouldForceReadConcern = false; + } + + const inWrappedForm = commandObj !== commandObjUnwrapped; + + if (shouldForceReadConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("readConcern")) { + let readConcern = commandObjUnwrapped.readConcern; + + if (typeof readConcern !== "object" || readConcern === null || + (readConcern.hasOwnProperty("level") && + bsonWoCompare({_: readConcern.level}, {_: kDefaultReadConcern.level}) !== 0)) { + throw new Error("Cowardly refusing to override read concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the readConcern object to avoid mutating the parameter the + // caller specified. + readConcern = Object.assign({}, readConcern, kDefaultReadConcern); + commandObjUnwrapped.readConcern = readConcern; + } else { + commandObjUnwrapped.readConcern = kDefaultReadConcern; + } + } + + if (shouldForceWriteConcern) { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + if (inWrappedForm) { + commandObjUnwrapped = Object.assign({}, commandObjUnwrapped); + commandObj[Object.keys(commandObj)[0]] = commandObjUnwrapped; + } else { + commandObjUnwrapped = commandObj; + } + + if (commandObjUnwrapped.hasOwnProperty("writeConcern")) { + let writeConcern = commandObjUnwrapped.writeConcern; + + if (typeof writeConcern !== "object" || writeConcern === null || + (writeConcern.hasOwnProperty("w") && + bsonWoCompare({_: writeConcern.w}, {_: kDefaultWriteConcern.w}) !== 0)) { + throw new Error("Cowardly refusing to override write concern of command: " + + tojson(commandObj)); + } + + // We create a copy of the writeConcern object to avoid mutating the parameter the + // caller specified. + writeConcern = Object.assign({}, writeConcern, kDefaultWriteConcern); + commandObjUnwrapped.writeConcern = writeConcern; + } else { + commandObjUnwrapped.writeConcern = kDefaultWriteConcern; + } + } + + return func.apply(conn, makeFuncArgs(commandObj)); + } + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/set_read_and_write_concerns.js"); + + OverrideHelpers.overrideRunCommand(runCommandWithReadAndWriteConcerns); +})(); diff --git a/jstests/libs/override_methods/set_read_preference_secondary.js b/jstests/libs/override_methods/set_read_preference_secondary.js new file mode 100644 index 00000000000..d1d26433c5c --- /dev/null +++ b/jstests/libs/override_methods/set_read_preference_secondary.js @@ -0,0 +1,162 @@ +/** + * Use prototype overrides to set read preference to "secondary" when running tests. + */ +(function() { + "use strict"; + + load("jstests/libs/override_methods/override_helpers.js"); + + const kReadPreferenceSecondary = {mode: "secondary"}; + const kCommandsSupportingReadPreference = new Set([ + "aggregate", + "collStats", + "count", + "dbStats", + "distinct", + "find", + "geoNear", + "geoSearch", + "group", + "mapReduce", + "mapreduce", + "parallelCollectionScan", + ]); + + // This list of cursor-generating commands is incomplete. For example, "listCollections", + // "listIndexes", "parallelCollectionScan", and "repairCursor" are all missing from this list. + // If we ever add tests that attempt to run getMore or killCursors on cursors generated from + // those commands, then we should update the contents of this list and also handle any + // differences in the server's response format. + const kCursorGeneratingCommands = new Set(["aggregate", "find"]); + + const CursorTracker = (function() { + const kNoCursor = new NumberLong(0); + + const connectionsByCursorId = {}; + + return { + getConnectionUsedForCursor: function getConnectionUsedForCursor(cursorId) { + return (cursorId instanceof NumberLong) ? connectionsByCursorId[cursorId] + : undefined; + }, + + setConnectionUsedForCursor: function setConnectionUsedForCursor(cursorId, cursorConn) { + if (cursorId instanceof NumberLong && + !bsonBinaryEqual({_: cursorId}, {_: kNoCursor})) { + connectionsByCursorId[cursorId] = cursorConn; + } + }, + }; + })(); + + function runCommandWithReadPreferenceSecondary( + conn, dbName, commandName, commandObj, func, makeFuncArgs) { + if (typeof commandObj !== "object" || commandObj === null) { + return func.apply(conn, makeFuncArgs(commandObj)); + } + + // If the command is in a wrapped form, then we look for the actual command object inside + // the query/$query object. + let commandObjUnwrapped = commandObj; + if (commandName === "query" || commandName === "$query") { + commandObjUnwrapped = commandObj[commandName]; + commandName = Object.keys(commandObjUnwrapped)[0]; + } + + if (commandObj[commandName] === "system.profile") { + throw new Error("Cowardly refusing to run test with overridden read preference" + + " when it reads from a non-replicated collection: " + + tojson(commandObj)); + } + + if (conn.isReplicaSetConnection()) { + // When a "getMore" or "killCursors" command is issued on a replica set connection, we + // attempt to automatically route the command to the server the cursor(s) were + // originally established on. This makes it possible to use the + // set_read_preference_secondary.js override without needing to update calls of + // DB#runCommand() to explicitly track the connection that was used. If the connection + // is actually a direct connection to a mongod or mongos process, or if the cursor id + // cannot be found in the CursorTracker, then we'll fall back to using DBClientRS's + // server selection and send the operation to the current primary. It is possible that + // the test is trying to exercise the behavior around when an unknown cursor id is sent + // to the server. + if (commandName === "getMore") { + const cursorId = commandObjUnwrapped[commandName]; + const cursorConn = CursorTracker.getConnectionUsedForCursor(cursorId); + if (cursorConn !== undefined) { + return func.apply(cursorConn, makeFuncArgs(commandObj)); + } + } else if (commandName === "killCursors") { + const cursorIds = commandObjUnwrapped.cursors; + if (Array.isArray(cursorIds)) { + let cursorConn; + + for (let cursorId of cursorIds) { + const otherCursorConn = CursorTracker.getConnectionUsedForCursor(cursorId); + if (cursorConn === undefined) { + cursorConn = otherCursorConn; + } else if (otherCursorConn !== undefined) { + // We set 'cursorConn' back to undefined and break out of the loop so + // that we don't attempt to automatically route the "killCursors" + // command when there are cursors from different servers. + cursorConn = undefined; + break; + } + } + + if (cursorConn !== undefined) { + return func.apply(cursorConn, makeFuncArgs(commandObj)); + } + } + } + } + + let shouldForceReadPreference = kCommandsSupportingReadPreference.has(commandName); + if (OverrideHelpers.isAggregationWithOutStage(commandName, commandObjUnwrapped)) { + // An aggregation with a $out stage must be sent to the primary. + shouldForceReadPreference = false; + } else if ((commandName === "mapReduce" || commandName === "mapreduce") && + !OverrideHelpers.isMapReduceWithInlineOutput(commandName, commandObjUnwrapped)) { + // A map-reduce operation with non-inline output must be sent to the primary. + shouldForceReadPreference = false; + } + + if (shouldForceReadPreference) { + if (commandObj === commandObjUnwrapped) { + // We wrap the command object using a "query" field rather than a "$query" field to + // match the implementation of DB.prototype._attachReadPreferenceToCommand(). + commandObj = {query: commandObj}; + } else { + // We create a copy of 'commandObj' to avoid mutating the parameter the caller + // specified. + commandObj = Object.assign({}, commandObj); + } + + if (commandObj.hasOwnProperty("$readPreference") && + !bsonBinaryEqual({_: commandObj.$readPreference}, {_: kReadPreferenceSecondary})) { + throw new Error("Cowardly refusing to override read preference of command: " + + tojson(commandObj)); + } + + commandObj.$readPreference = kReadPreferenceSecondary; + } + + const serverResponse = func.apply(conn, makeFuncArgs(commandObj)); + + if (conn.isReplicaSetConnection() && kCursorGeneratingCommands.has(commandName) && + serverResponse.ok === 1 && serverResponse.hasOwnProperty("cursor")) { + // We associate the cursor id returned by the server with the connection that was used + // to establish it so that we can attempt to automatically route subsequent "getMore" + // and "killCursors" commands. + CursorTracker.setConnectionUsedForCursor(serverResponse.cursor.id, + serverResponse._mongo); + } + + return serverResponse; + } + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/set_read_preference_secondary.js"); + + OverrideHelpers.overrideRunCommand(runCommandWithReadPreferenceSecondary); +})(); diff --git a/jstests/multiVersion/initial_sync_last_stable_from_latest.js b/jstests/multiVersion/initial_sync_last_stable_from_latest.js new file mode 100644 index 00000000000..3f036c17fe9 --- /dev/null +++ b/jstests/multiVersion/initial_sync_last_stable_from_latest.js @@ -0,0 +1,15 @@ +/** + * Multiversion initial sync test. Tests that initial sync succeeds when a 'last-stable' version + * secondary syncs from a 'latest' version replica set. + */ + +'use strict'; + +load("./jstests/multiVersion/libs/initial_sync.js"); + +var testName = "multiversion_initial_sync_last_stable_from_latest"; +let replSetVersion = "latest"; +let newSecondaryVersion = "last-stable"; +let fcv = "3.2"; + +multversionInitialSyncTest(testName, replSetVersion, newSecondaryVersion, {}, fcv); diff --git a/jstests/multiVersion/initial_sync_latest_from_last_stable.js b/jstests/multiVersion/initial_sync_latest_from_last_stable.js new file mode 100644 index 00000000000..fd92b64e87a --- /dev/null +++ b/jstests/multiVersion/initial_sync_latest_from_last_stable.js @@ -0,0 +1,14 @@ +/** + * Multiversion initial sync test. Tests that initial sync succeeds when a 'latest' version + * secondary syncs from a 'last-stable' version replica set. + */ + +'use strict'; + +load("./jstests/multiVersion/libs/initial_sync.js"); + +var testName = "multiversion_initial_sync_latest_from_last_stable"; +let replSetVersion = "last-stable"; +let newSecondaryVersion = "latest"; + +multversionInitialSyncTest(testName, replSetVersion, newSecondaryVersion, {}); diff --git a/jstests/multiVersion/initialsync.js b/jstests/multiVersion/initialsync.js deleted file mode 100644 index e9a424fd05c..00000000000 --- a/jstests/multiVersion/initialsync.js +++ /dev/null @@ -1,58 +0,0 @@ -// Multiversion initial sync test. -load("./jstests/multiVersion/libs/multi_rs.js"); -load("./jstests/replsets/rslib.js"); - -var oldVersion = "last-stable"; -var newVersion = "latest"; - -var name = "multiversioninitsync"; - -var multitest = function(replSetVersion, newNodeVersion) { - var nodes = {n1: {binVersion: replSetVersion}, n2: {binVersion: replSetVersion}}; - - print("Start up a two-node " + replSetVersion + " replica set."); - var rst = new ReplSetTest({name: name, nodes: nodes}); - rst.startSet(); - var config = rst.getReplSetConfig(); - // Set protocol version to 0 for 3.2 replset. - if (replSetVersion == newVersion) { - config.protocolVersion = 0; - } - rst.initiate(config); - - // Wait for a primary node. - var primary = rst.getPrimary(); - - // Insert some data and wait for replication. - for (var i = 0; i < 25; i++) { - primary.getDB("foo").foo.insert({_id: i}); - } - rst.awaitReplication(); - - print("Bring up a new node with version " + newNodeVersion + " and add to set."); - rst.add({binVersion: newNodeVersion}); - rst.reInitiate(); - - // Wait for a primary node. - var primary = rst.getPrimary(); - var secondaries = rst.getSecondaries(); - - print("Wait for new node to be synced."); - rst.awaitReplication(); - - rst.stopSet(); -}; - -// ***************************************** -// Test A: -// "Latest" version secondary is synced from -// an old ReplSet. -// ***************************************** -multitest(oldVersion, newVersion); - -// ***************************************** -// Test B: -// Old Secondary is synced from a "latest" -// version ReplSet. -// ***************************************** -multitest(newVersion, oldVersion); diff --git a/jstests/multiVersion/libs/initial_sync.js b/jstests/multiVersion/libs/initial_sync.js new file mode 100644 index 00000000000..866f825c8b9 --- /dev/null +++ b/jstests/multiVersion/libs/initial_sync.js @@ -0,0 +1,50 @@ + +'use strict'; + +load("./jstests/multiVersion/libs/multi_rs.js"); +load("./jstests/replsets/rslib.js"); + +/** + * Test that starts up a replica set with 2 nodes of version 'replSetVersion', inserts some data, + * then adds a new node to the replica set with version 'newNodeVersion' and waits for initial sync + * to complete. If the 'fcv' argument is given, sets the feature compatibility version of the + * replica set to 'fcv' before adding the third node. + */ +var multversionInitialSyncTest = function( + name, replSetVersion, newNodeVersion, configSettings, fcv) { + + var nodes = {n1: {binVersion: replSetVersion}, n2: {binVersion: replSetVersion}}; + + jsTestLog("Starting up a two-node '" + replSetVersion + "' version replica set."); + var rst = new ReplSetTest({name: name, nodes: nodes}); + rst.startSet(); + + var conf = rst.getReplSetConfig(); + conf.settings = configSettings; + rst.initiate(conf); + + // Wait for a primary node. + var primary = rst.getPrimary(); + + // Set 'featureCompatibilityVersion' if given. + if (fcv) { + jsTestLog("Setting FCV to '" + fcv + "' on the primary."); + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: fcv})); + } + + // Insert some data and wait for replication. + for (var i = 0; i < 25; i++) { + primary.getDB("foo").foo.insert({_id: i}); + } + rst.awaitReplication(); + + jsTestLog("Bringing up a new node with version '" + newNodeVersion + "' and adding to set."); + rst.add({binVersion: newNodeVersion}); + rst.reInitiate(); + + jsTestLog("Waiting for new node to be synced."); + rst.awaitReplication(); + rst.awaitSecondaryNodes(); + + rst.stopSet(); +};
\ No newline at end of file diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js index da976c7f4dc..87d5995ef48 100644 --- a/jstests/multiVersion/libs/multi_rs.js +++ b/jstests/multiVersion/libs/multi_rs.js @@ -25,6 +25,7 @@ ReplSetTest.prototype.upgradeSet = function(options, user, pwd) { var node = nodesToUpgrade[i]; if (node == primary) { node = this.stepdown(node); + this.waitForState(node, ReplSetTest.State.SECONDARY); primary = this.getPrimary(); } diff --git a/jstests/multiVersion/set_feature_compatibility_version.js b/jstests/multiVersion/set_feature_compatibility_version.js index d5c73f26efb..73a5785aec9 100644 --- a/jstests/multiVersion/set_feature_compatibility_version.js +++ b/jstests/multiVersion/set_feature_compatibility_version.js @@ -54,6 +54,23 @@ // featureCompatibilityVersion cannot be set via setParameter. assert.commandFailed(adminDB.runCommand({setParameter: 1, featureCompatibilityVersion: "3.2"})); + // setFeatureCompatibilityVersion fails to downgrade to FCV=3.2 if the write fails. + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "alwaysOn" + })); + assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.2"})); + res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); + assert.commandWorked(res); + assert.eq(res.featureCompatibilityVersion, "3.4"); + assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version, "3.4"); + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "off" + })); + // featureCompatibilityVersion can be set to 3.2. assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.2"})); res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); @@ -70,6 +87,23 @@ "Expected index with name 'incompatible_with_version_32' to have been removed: " + tojson(allIndexes)); + // setFeatureCompatibilityVersion fails to upgrade to FCV=3.4 if the write fails. + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "alwaysOn" + })); + assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"})); + res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); + assert.commandWorked(res); + assert.eq(res.featureCompatibilityVersion, "3.2"); + assert.eq(adminDB.system.version.findOne({_id: "featureCompatibilityVersion"}).version, "3.2"); + assert.commandWorked(adminDB.runCommand({ + configureFailPoint: "failCollectionUpdates", + data: {collectionNS: "admin.system.version"}, + mode: "off" + })); + // featureCompatibilityVersion can be set to 3.4. assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.4"})); res = adminDB.runCommand({getParameter: 1, featureCompatibilityVersion: 1}); diff --git a/jstests/noPassthrough/ftdc_setdirectory.js b/jstests/noPassthrough/ftdc_setdirectory.js new file mode 100644 index 00000000000..c1173055e2a --- /dev/null +++ b/jstests/noPassthrough/ftdc_setdirectory.js @@ -0,0 +1,118 @@ +/** + * Test that verifies FTDC works in mongos. + */ +load('jstests/libs/ftdc.js'); + +(function() { + 'use strict'; + let testPath1 = MongoRunner.toRealPath('ftdc_setdir1'); + let testPath2 = MongoRunner.toRealPath('ftdc_setdir2'); + let testPath3 = MongoRunner.toRealPath('ftdc_setdir3'); + let testLog3 = testPath3 + "mongos_ftdc.log"; + + // Make the log file directory for mongos. + mkdir(testPath3); + + // Startup 3 mongos: + // 1. Normal MongoS with no log file to verify FTDC can be startup at runtime with a path. + // 2. MongoS with explict diagnosticDataCollectionDirectoryPath setParameter at startup. + // 3. MongoS with log file to verify automatic FTDC path computation works. + let st = new ShardingTest({ + shards: 1, + mongos: { + s0: {verbose: 0}, + s1: {setParameter: {diagnosticDataCollectionDirectoryPath: testPath2}}, + s2: {logpath: testLog3} + } + }); + + let admin1 = st.s0.getDB('admin'); + let admin2 = st.s1.getDB('admin'); + let admin3 = st.s2.getDB('admin'); + + function setParam(admin, obj) { + var ret = admin.runCommand(Object.extend({setParameter: 1}, obj)); + return ret; + } + + function getParam(admin, field) { + var q = {getParameter: 1}; + q[field] = 1; + + var ret = admin.runCommand(q); + assert.commandWorked(ret); + return ret[field]; + } + + // Verify FTDC can be started at runtime. + function verifyFTDCDisabledOnStartup() { + jsTestLog("Running verifyFTDCDisabledOnStartup"); + verifyCommonFTDCParameters(admin1, false); + + // 1. Try to enable and fail + assert.commandFailed(setParam(admin1, {"diagnosticDataCollectionEnabled": 1})); + + // 2. Set path and succeed + assert.commandWorked( + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 3. Set path again and fail + assert.commandFailed( + setParam(admin1, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 4. Enable successfully + assert.commandWorked(setParam(admin1, {"diagnosticDataCollectionEnabled": 1})); + + // 5. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin1); + } + + // Verify FTDC is already running if there was a path set at startup. + function verifyFTDCStartsWithPath() { + jsTestLog("Running verifyFTDCStartsWithPath"); + verifyCommonFTDCParameters(admin2, true); + + // 1. Set path fail + assert.commandFailed( + setParam(admin2, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 2. Enable successfully + assert.commandWorked(setParam(admin2, {"diagnosticDataCollectionEnabled": 1})); + + // 3. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin2); + } + + function normpath(path) { + return path.replace(/\\/g, "/"); + } + + // Verify FTDC is already running if there was a path set at startup. + function verifyFTDCStartsWithLogFile() { + jsTestLog("Running verifyFTDCStartsWithLogFile"); + verifyCommonFTDCParameters(admin3, true); + + // 1. Verify that path is computed correctly. + let computedPath = getParam(admin3, "diagnosticDataCollectionDirectoryPath"); + assert.eq(normpath(computedPath), normpath(testPath3 + "mongos_ftdc.diagnostic.data")); + + // 2. Set path fail + assert.commandFailed( + setParam(admin3, {"diagnosticDataCollectionDirectoryPath": testPath2})); + + // 3. Enable successfully + assert.commandWorked(setParam(admin3, {"diagnosticDataCollectionEnabled": 1})); + + // 4. Validate getDiagnosticData returns FTDC data now + jsTestLog("Verifying FTDC getDiagnosticData"); + verifyGetDiagnosticData(admin3); + } + + verifyFTDCDisabledOnStartup(); + verifyFTDCStartsWithPath(); + verifyFTDCStartsWithLogFile(); + + st.stop(); +})(); diff --git a/jstests/noPassthrough/geo_near_fcv32.js b/jstests/noPassthrough/geo_near_fcv32.js new file mode 100644 index 00000000000..10b331a411d --- /dev/null +++ b/jstests/noPassthrough/geo_near_fcv32.js @@ -0,0 +1,85 @@ +/** + * Confirms that $geoNear aggregation and geoNear command succeed when FCV is 3.2. + */ +(function() { + "use strict"; + + const conn = MongoRunner.runMongod({}); + assert.neq(null, conn, "mongod was unable to start up"); + + const testDB = conn.getDB("geo_near_fcv32"); + testDB.test32.drop(); + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.2"})); + + // Create 2dsphere index. + assert.commandWorked(testDB.test32.createIndex({loc: "2dsphere"})); + + // Assert that $geoNear aggregate command does not fail due to collation errors when FCV is 3.2. + assert.eq(0, + testDB.test32 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true + } + }]) + .itcount()); + + // Assert that specifying the simple collation in $geoNear aggregate fails with FCV 3.2. + assert.throws(function() { + testDB.test32.aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + collation: {locale: "simple"}, + } + }]); + }); + + // Assert that specifying the simple collation in geoNear command fails with FCV 3.2. + assert.commandFailed(testDB.runCommand({ + geoNear: "test32", + near: {type: "Point", coordinates: [1.23, 1.23]}, + spherical: true, + collation: {locale: "simple"} + })); + + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.4"})); + + // Create collection with case sensitive collation. + assert.commandWorked( + testDB.createCollection("test34", {collation: {locale: "en_US", strength: 2}})); + assert.commandWorked(testDB.test34.createIndex({loc: "2dsphere"})); + assert.writeOK(testDB.test34.insert({loc: [1.23, 1.23], str: "A"})); + + // Assert that after upgrading FCV to 3.4 $geoNear aggregate inherits the collection's default + // collation. + assert.eq(1, + testDB.test34 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + query: {str: "a"}, + } + }]) + .itcount()); + + // Assert that the $geoNear aggregate accepts a specific collation and overrides the default + // collation in FCV 3.4. + assert.eq(1, + testDB.test34 + .aggregate([{ + $geoNear: { + near: {type: "Point", coordinates: [1.23, 1.23]}, + distanceField: "distance", + spherical: true, + query: {str: "Ã "}, + } + }], + {collation: {locale: "en_US", strength: 1}}) + .itcount()); +})(); diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js index b4e02415081..e66f0f8af97 100644 --- a/jstests/noPassthrough/libs/backup_restore.js +++ b/jstests/noPassthrough/libs/backup_restore.js @@ -352,15 +352,31 @@ var BackupRestoreTest = function(options) { stopMongoProgramByPid(fsmPid); // Wait up to 5 minutes until the new hidden node is in state SECONDARY. + jsTestLog('CRUD and FSM clients stopped. Waiting for hidden node ' + hiddenHost + + ' to become SECONDARY'); rst.waitForState(hiddenNode, ReplSetTest.State.SECONDARY); // Wait for secondaries to finish catching up before shutting down. + jsTestLog( + 'Hidden node ' + hiddenHost + + ' is now SECONDARY. Waiting for CRUD and FSM operations to be applied on all nodes.'); + rst.awaitReplication(); + + jsTestLog('CRUD and FSM operations successfully applied on all nodes. ' + + 'Waiting for all nodes to agree on the primary.'); rst.awaitNodesAgreeOnPrimary(); + + jsTestLog('All nodes agree on the primary. Getting current primary.'); primary = rst.getPrimary(); - assert.writeOK(primary.getDB("test").foo.insert( - {}, {writeConcern: {w: rst.nodes.length, wtimeout: 10 * 60 * 1000}})); + + jsTestLog('Inserting single document into primary ' + primary.host + + ' with writeConcern w:' + rst.nodes.length); + var writeResult = assert.writeOK(primary.getDB("test").foo.insert( + {}, {writeConcern: {w: rst.nodes.length, wtimeout: ReplSetTest.kDefaultTimeoutMS}})); // Stop set. + jsTestLog('Insert operation successful: ' + tojson(writeResult) + + '. Stopping replica set.'); rst.stopSet(); // Cleanup the files from the test diff --git a/jstests/noPassthrough/non_atomic_apply_ops_logging.js b/jstests/noPassthrough/non_atomic_apply_ops_logging.js new file mode 100644 index 00000000000..f93e9d38189 --- /dev/null +++ b/jstests/noPassthrough/non_atomic_apply_ops_logging.js @@ -0,0 +1,62 @@ +// SERVER-28594 Ensure non-atomic ops are individually logged in applyOps +// and atomic ops are collectively logged in applyOps. +(function() { + "use strict"; + + let rst = new ReplSetTest({nodes: 1}); + rst.startSet(); + rst.initiate(); + + let primary = rst.getPrimary(); + let testDB = primary.getDB("test"); + let oplogColl = primary.getDB("local").oplog.rs; + let testCollName = "testColl"; + let rerenamedCollName = "rerenamedColl"; + + testDB.runCommand({drop: testCollName}); + testDB.runCommand({drop: rerenamedCollName}); + assert.commandWorked(testDB.runCommand({create: testCollName})); + let testColl = testDB[testCollName]; + + // Ensure atomic apply ops logging only produces one oplog entry + // per call to apply ops and does not log individual operations + // separately. + assert.commandWorked(testDB.runCommand({ + applyOps: [ + {op: "i", ns: testColl.getFullName(), o: {_id: 1, a: "foo"}}, + {op: "i", ns: testColl.getFullName(), o: {_id: 2, a: "bar"}} + ] + })); + assert.eq(oplogColl.find({"o.applyOps": {"$exists": true}}).count(), 1); + assert.eq(oplogColl.find({"op": "i"}).count(), 0); + // Ensure non-atomic apply ops logging produces an oplog entry for + // each operation in the apply ops call and no record of applyOps + // appears for these operations. + assert.commandWorked(testDB.runCommand({ + applyOps: [ + { + op: "c", + ns: "test.$cmd", + o: { + renameCollection: "test.testColl", + to: "test.renamedColl", + stayTemp: false, + dropTarget: false + } + }, + { + op: "c", + ns: "test.$cmd", + o: { + renameCollection: "test.renamedColl", + to: "test." + rerenamedCollName, + stayTemp: false, + dropTarget: false + } + } + ] + })); + assert.eq(oplogColl.find({"o.renameCollection": {"$exists": true}}).count(), 2); + assert.eq(oplogColl.find({"o.applyOps": {"$exists": true}}).count(), 1); + rst.stopSet(); +})(); diff --git a/jstests/noPassthrough/partial_unique_indexes.js b/jstests/noPassthrough/partial_unique_indexes.js new file mode 100644 index 00000000000..7c376872ade --- /dev/null +++ b/jstests/noPassthrough/partial_unique_indexes.js @@ -0,0 +1,47 @@ +/** + * SERVER-32001: Test that indexing paths for non-unique, partial, unique, partial&unique + * crud operations correctly handle WriteConflictExceptions. + */ +(function() { + "strict"; + + let conn = MongoRunner.runMongod(); + let testDB = conn.getDB("test"); + + let t = testDB.jstests_parallel_allops; + t.drop(); + + t.createIndex({x: 1, _id: 1}, {partialFilterExpression: {_id: {$lt: 500}}, unique: true}); + t.createIndex({y: -1, _id: 1}, {unique: true}); + t.createIndex({x: -1}, {partialFilterExpression: {_id: {$gte: 500}}, unique: false}); + t.createIndex({y: 1}, {unique: false}); + + let _id = {"#RAND_INT": [0, 1000]}; + let ops = [ + {op: "remove", ns: t.getFullName(), query: {_id}}, + {op: "update", ns: t.getFullName(), query: {_id}, update: {$inc: {x: 1}}, upsert: true}, + {op: "update", ns: t.getFullName(), query: {_id}, update: {$inc: {y: 1}}, upsert: true}, + ]; + + let seconds = 5; + let parallel = 5; + let host = testDB.getMongo().host; + + let benchArgs = {ops, seconds, parallel, host}; + + assert.commandWorked(testDB.adminCommand({ + configureFailPoint: 'WTWriteConflictExceptionForReads', + mode: {activationProbability: 0.01} + })); + assert.commandWorked(testDB.adminCommand( + {configureFailPoint: 'WTWriteConflictException', mode: {activationProbability: 0.01}})); + res = benchRun(benchArgs); + printjson({res}); + + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'WTWriteConflictException', mode: "off"})); + assert.commandWorked( + testDB.adminCommand({configureFailPoint: 'WTWriteConflictExceptionForReads', mode: "off"})); + res = t.validate(); + assert(res.valid, tojson(res)); +})(); diff --git a/jstests/noPassthrough/skip_sharding_configuration_checks.js b/jstests/noPassthrough/skip_sharding_configuration_checks.js new file mode 100644 index 00000000000..ff3e95440eb --- /dev/null +++ b/jstests/noPassthrough/skip_sharding_configuration_checks.js @@ -0,0 +1,53 @@ +/** + * Starts standalone RS with skipShardingConfigurationChecks. + * @tags: [requires_persistence] + */ +(function() { + 'use strict'; + + function expectState(rst, state) { + assert.soon(function() { + var status = rst.status(); + if (status.myState != state) { + print("Waiting for state " + state + " in replSetGetStatus output: " + + tojson(status)); + } + return status.myState == state; + }); + } + + let configSvr = MongoRunner.runMongod( + {configsvr: "", setParameter: 'skipShardingConfigurationChecks=true'}); + assert.eq(configSvr, null); + + let shardSvr = + MongoRunner.runMongod({shardsvr: "", setParameter: 'skipShardingConfigurationChecks=true'}); + assert.eq(shardSvr, null); + + var st = new ShardingTest({name: "skipConfig", shards: {rs0: {nodes: 1}}}); + var configRS = st.configRS; + var shardRS = st.rs0; + + st.stopAllMongos(); + shardRS.stopSet(15, true); + configRS.stopSet(undefined, true); + + jsTestLog("Restarting configRS as a standalone ReplicaSet"); + + for (let i = 0; i < configRS.nodes.length; i++) { + delete configRS.nodes[i].fullOptions.configsvr; + configRS.nodes[i].fullOptions.setParameter = 'skipShardingConfigurationChecks=true'; + } + configRS.startSet({}, true); + expectState(configRS, ReplSetTest.State.PRIMARY); + configRS.stopSet(); + + jsTestLog("Restarting shardRS as a standalone ReplicaSet"); + for (let i = 0; i < shardRS.nodes.length; i++) { + delete shardRS.nodes[i].fullOptions.shardsvr; + shardRS.nodes[i].fullOptions.setParameter = 'skipShardingConfigurationChecks=true'; + } + shardRS.startSet({}, true); + expectState(shardRS, ReplSetTest.State.PRIMARY); + shardRS.stopSet(); +})(); diff --git a/jstests/noPassthroughWithMongod/apply_ops_index_collation.js b/jstests/noPassthroughWithMongod/apply_ops_index_collation.js new file mode 100644 index 00000000000..6248bc63b0f --- /dev/null +++ b/jstests/noPassthroughWithMongod/apply_ops_index_collation.js @@ -0,0 +1,80 @@ +// Cannot implicitly shard accessed collections because of collection existing when none +// expected. +// @tags: [assumes_no_implicit_collection_creation_after_drop] + +// Tests creation of indexes using applyOps for collections with a non-simple default collation. +// Indexes created through applyOps should be built exactly according to their index spec, without +// inheriting the collection default collation, since this is how the oplog entries are replicated. +// TODO SERVER-31435: Move this test into core once applyOps with createIndexes replicates +// correctly. +(function() { + "use strict"; + + load("jstests/libs/get_index_helpers.js"); + + const coll = db.apply_ops_index_collation; + coll.drop(); + assert.commandWorked(db.createCollection(coll.getName(), {collation: {locale: "fr_CA"}})); + + // An index created using an insert-style oplog entry with a non-simple collation does not + // inherit the collection default collation. + let res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: { + v: 2, + key: {c: 1}, + name: "c_1_en", + ns: coll.getFullName(), + collation: { + locale: "en_US", + caseLevel: false, + caseFirst: "off", + strength: 3, + numericOrdering: false, + alternate: "non-ignorable", + maxVariable: "punct", + normalization: false, + backwards: false, + version: "57.1" + } + } + }] + })); + let allIndexes = coll.getIndexes(); + let spec = GetIndexHelpers.findByName(allIndexes, "c_1_en"); + assert.neq(null, spec, "Index 'c_1_en' not found: " + tojson(allIndexes)); + assert.eq(2, spec.v, tojson(spec)); + assert.eq("en_US", spec.collation.locale, tojson(spec)); + + // An index created using an insert-style oplog entry with a simple collation does not inherit + // the collection default collation. + res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: {v: 2, key: {c: 1}, name: "c_1", ns: coll.getFullName()} + }] + })); + allIndexes = coll.getIndexes(); + spec = GetIndexHelpers.findByName(allIndexes, "c_1"); + assert.neq(null, spec, "Index 'c_1' not found: " + tojson(allIndexes)); + assert.eq(2, spec.v, tojson(spec)); + assert(!spec.hasOwnProperty("collation"), tojson(spec)); + + // A v=1 index created using an insert-style oplog entry does not inherit the collection default + // collation. + res = assert.commandWorked(db.adminCommand({ + applyOps: [{ + op: "i", + ns: db.system.indexes.getFullName(), + o: {v: 1, key: {d: 1}, name: "d_1", ns: coll.getFullName()} + }] + })); + allIndexes = coll.getIndexes(); + spec = GetIndexHelpers.findByName(allIndexes, "d_1"); + assert.neq(null, spec, "Index 'd_1' not found: " + tojson(allIndexes)); + assert.eq(1, spec.v, tojson(spec)); + assert(!spec.hasOwnProperty("collation"), tojson(spec)); +})(); diff --git a/jstests/noPassthroughWithMongod/ftdc_params.js b/jstests/noPassthroughWithMongod/ftdc_params.js index ced6c1b5675..08714040fcb 100644 --- a/jstests/noPassthroughWithMongod/ftdc_params.js +++ b/jstests/noPassthroughWithMongod/ftdc_params.js @@ -1,58 +1,10 @@ // FTDC test cases // +load('jstests/libs/ftdc.js'); + (function() { 'use strict'; var admin = db.getSiblingDB("admin"); - // Check the defaults are correct - // - function getparam(field) { - var q = {getParameter: 1}; - q[field] = 1; - - var ret = admin.runCommand(q); - return ret[field]; - } - - // Verify the defaults are as we documented them - assert.eq(getparam("diagnosticDataCollectionEnabled"), true); - assert.eq(getparam("diagnosticDataCollectionPeriodMillis"), 1000); - assert.eq(getparam("diagnosticDataCollectionDirectorySizeMB"), 200); - assert.eq(getparam("diagnosticDataCollectionFileSizeMB"), 10); - assert.eq(getparam("diagnosticDataCollectionSamplesPerChunk"), 300); - assert.eq(getparam("diagnosticDataCollectionSamplesPerInterimUpdate"), 10); - - function setparam(obj) { - var ret = admin.runCommand(Object.extend({setParameter: 1}, obj)); - return ret; - } - - assert.commandWorked(setparam({"diagnosticDataCollectionEnabled": 1})); - assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 100})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 1})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 2})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 2})); - - // Negative tests - set values below minimums - assert.commandFailed(setparam({"diagnosticDataCollectionPeriodMillis": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerChunk": 1})); - assert.commandFailed(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 1})); - - // Negative test - set file size bigger then directory size - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - assert.commandFailed(setparam({"diagnosticDataCollectionFileSizeMB": 100})); - - // Negative test - set directory size less then file size - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 100})); - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 50})); - assert.commandFailed(setparam({"diagnosticDataCollectionDirectorySizeMB": 10})); - - // Reset - assert.commandWorked(setparam({"diagnosticDataCollectionFileSizeMB": 10})); - assert.commandWorked(setparam({"diagnosticDataCollectionDirectorySizeMB": 200})); - assert.commandWorked(setparam({"diagnosticDataCollectionPeriodMillis": 1000})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerChunk": 300})); - assert.commandWorked(setparam({"diagnosticDataCollectionSamplesPerInterimUpdate": 10})); + verifyCommonFTDCParameters(admin, true); })(); diff --git a/jstests/noPassthroughWithMongod/host_connection_string_validation.js b/jstests/noPassthroughWithMongod/host_connection_string_validation.js index a252ef39230..f7bac51db88 100644 --- a/jstests/noPassthroughWithMongod/host_connection_string_validation.js +++ b/jstests/noPassthroughWithMongod/host_connection_string_validation.js @@ -89,8 +89,8 @@ print("Testing " + (isGood ? "good" : "bad") + " connection string " + i + "..."); print(" * testing " + connectionString); testHost(connectionString, isGood); - print(" * testing mongodb://" + connectionString); - testHost("mongodb://" + connectionString, isGood); + print(" * testing mongodb://" + encodeURIComponent(connectionString)); + testHost("mongodb://" + encodeURIComponent(connectionString), isGood); } var i; diff --git a/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js b/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js index 3a47e14447c..d186a8f59ec 100644 --- a/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js +++ b/jstests/noPassthroughWithMongod/initial_sync_oplog_rollover.js @@ -34,7 +34,7 @@ assert.writeOK(coll.insert({a: 1})); function getFirstOplogEntry(conn) { - return conn.getDB('local').oplog.rs.find().sort({ts: 1}).limit(1)[0]; + return conn.getDB('local').oplog.rs.find().sort({$natural: 1}).limit(1)[0]; } var firstOplogEntry = getFirstOplogEntry(primary); diff --git a/jstests/noPassthroughWithMongod/replset_host_connection_validation.js b/jstests/noPassthroughWithMongod/replset_host_connection_validation.js index 7e48de363e2..e65387dd705 100644 --- a/jstests/noPassthroughWithMongod/replset_host_connection_validation.js +++ b/jstests/noPassthroughWithMongod/replset_host_connection_validation.js @@ -1,5 +1,6 @@ // Test --host with a replica set. (function() { + 'use strict'; const replSetName = 'hostTestReplSetName'; @@ -26,35 +27,56 @@ // Pass the inner test's exit code back as the outer test's exit code if (exitCode != 0) { - doassert("inner test failed with exit code " + exitcode); + doassert("inner test failed with exit code " + exitCode); } return; } - const testHost = function(host) { - const exitCode = runMongoProgram('mongo', '--eval', ';', '--host', host); - if (exitCode !== 0) { - doassert("failed to connect with `--host " + host + - "`, but expected success. Exit code: " + exitCode); + function testHost(host, uri, ok) { + const exitCode = runMongoProgram('mongo', '--eval', ';', '--host', host, uri); + if (ok) { + assert.eq(exitCode, 0, "failed to connect with `--host " + host + "`"); + } else { + assert.neq(exitCode, 0, "unexpectedly succeeded to connect with `--host " + host + "`"); } - }; - - const connStrings = [ - `localhost:${port}`, - `${replSetName}/localhost:${port}`, - `mongodb://localhost:${port}/admin?replicaSet=${replSetName}`, - `mongodb://localhost:${port}`, - ]; - - function runConnectionStringTestFor(i, connectionString) { - print("Testing connection string " + i + "..."); - print(" * testing " + connectionString); - testHost(connectionString); } - for (let i = 0; i < connStrings.length; ++i) { - runConnectionStringTestFor(i, connStrings[i]); + function runConnectionStringTestFor(connectionString, uri, ok) { + print("* Testing: --host " + connectionString + " " + uri); + if (!ok) { + print(" This should fail"); + } + testHost(connectionString, uri, ok); + } + + function expSuccess(str) { + runConnectionStringTestFor(str, '', true); + if (!str.startsWith('mongodb://')) { + runConnectionStringTestFor(str, 'dbname', true); + } } + function expFailure(str) { + runConnectionStringTestFor(str, '', false); + } + + expSuccess(`localhost:${port}`); + expSuccess(`${replSetName}/localhost:${port}`); + expSuccess(`${replSetName}/localhost:${port},[::1]:${port}`); + expSuccess(`${replSetName}/localhost:${port},`); + expSuccess(`${replSetName}/localhost:${port},,`); + expSuccess(`mongodb://localhost:${port}/admin?replicaSet=${replSetName}`); + expSuccess(`mongodb://localhost:${port}`); + + expFailure(','); + expFailure(',,'); + expFailure(`${replSetName}/`); + expFailure(`${replSetName}/,`); + expFailure(`${replSetName}/,,`); + expFailure(`${replSetName}//not/a/socket`); + expFailure(`mongodb://localhost:${port}/admin?replicaSet=`); + expFailure('mongodb://localhost:'); + expFailure(`mongodb://:${port}`); + jsTest.log("SUCCESSFUL test completion"); })(); diff --git a/jstests/replsets/apply_batches_totalMillis.js b/jstests/replsets/apply_batches_totalMillis.js new file mode 100644 index 00000000000..9e093211cb6 --- /dev/null +++ b/jstests/replsets/apply_batches_totalMillis.js @@ -0,0 +1,63 @@ +/** + * serverStatus.metrics.repl.apply.batches.totalMillis is a cumulative measure of how much time a + * node spends applying batches. This test checks that it includes the time spent waiting for + * batches to finish, by comparing the time recorded after replicating a small and a large load. + */ + +(function() { + "use strict"; + + // Gets the value of metrics.repl.apply.batches.totalMillis. + function getTotalMillis(node) { + return assert.commandWorked(node.adminCommand({serverStatus: 1})) + .metrics.repl.apply.batches.totalMillis; + } + + // Do a bulk insert of documents as: {{key: 0}, {key: 1}, {key: 2}, ... , {key: num-1}} + function performBulkInsert(coll, key, num) { + let bulk = coll.initializeUnorderedBulkOp(); + for (let i = 0; i < num; i++) { + let doc = {}; + doc[key] = i; + bulk.insert(doc); + } + assert.writeOK(bulk.execute()); + rst.awaitReplication(); + } + + let name = "apply_batches_totalMillis"; + let rst = new ReplSetTest({name: name, nodes: 2}); + rst.startSet(); + rst.initiate(); + + let primary = rst.getPrimary(); + let secondary = rst.getSecondary(); + let coll = primary.getDB(name)["foo"]; + + // Perform an initial write on the system and ensure steady state. + assert.writeOK(coll.insert({init: 0})); + rst.awaitReplication(); + let baseTime = getTotalMillis(secondary); + + // Introduce a small load and wait for it to be replicated. + performBulkInsert(coll, "small", 1000); + + // Record the time spent applying the small load. + let timeAfterSmall = getTotalMillis(secondary); + let deltaSmall = timeAfterSmall - baseTime; + + // Insert a significantly larger load. + performBulkInsert(coll, "large", 20000); + + // Record the time spent applying the large load. + let timeAfterLarge = getTotalMillis(secondary); + let deltaLarge = timeAfterLarge - timeAfterSmall; + + jsTestLog(`Recorded deltas: {small: ${deltaSmall}ms, large: ${deltaLarge}ms}.`); + + // We should have recorded at least as much time on the second load as we did on the first. + // This is a crude comparison that is only taken to check that the timer is used correctly. + assert(deltaLarge >= deltaSmall, "Expected a higher net totalMillis for the larger load."); + rst.stopSet(); + +})();
\ No newline at end of file diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js b/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js new file mode 100644 index 00000000000..05cb6f9e996 --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_different_db.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test1.coll1', + ns2: 'test2.coll2', + requiresDocumentLevelConcurrency: false, + }).run(); +}()); diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js b/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js new file mode 100644 index 00000000000..004eeaaa52f --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_same_collection.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test.coll', + ns2: 'test.coll', + requiresDocumentLevelConcurrency: true, + }).run(); +}()); diff --git a/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js b/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js new file mode 100644 index 00000000000..10f874382a5 --- /dev/null +++ b/jstests/replsets/apply_ops_concurrent_non_atomic_same_db.js @@ -0,0 +1,11 @@ +(function() { + 'use strict'; + + load('jstests/replsets/libs/apply_ops_concurrent_non_atomic.js'); + + new ApplyOpsConcurrentNonAtomicTest({ + ns1: 'test.coll1', + ns2: 'test.coll2', + requiresDocumentLevelConcurrency: false, + }).run(); +}()); diff --git a/jstests/replsets/clean_shutdown_oplog_state.js b/jstests/replsets/clean_shutdown_oplog_state.js index 5ac2f72d556..91b31fc66d7 100644 --- a/jstests/replsets/clean_shutdown_oplog_state.js +++ b/jstests/replsets/clean_shutdown_oplog_state.js @@ -10,7 +10,7 @@ var rst = new ReplSetTest({ name: "name", nodes: 2, - oplogSize: 100, + oplogSize: 500, }); rst.startSet(); diff --git a/jstests/replsets/index_delete.js b/jstests/replsets/index_delete.js index f7c45e2f2e0..1c352a18581 100644 --- a/jstests/replsets/index_delete.js +++ b/jstests/replsets/index_delete.js @@ -60,7 +60,7 @@ try { } else { return false; } - }, "index not started on secondary", 30000, 50); + }, "index not started on secondary"); } finally { // Turn off failpoint and let the index build resumes. assert.commandWorked( diff --git a/jstests/replsets/initial_sync_rename_collection_unsafe.js b/jstests/replsets/initial_sync_rename_collection_unsafe.js new file mode 100644 index 00000000000..a105e1c9287 --- /dev/null +++ b/jstests/replsets/initial_sync_rename_collection_unsafe.js @@ -0,0 +1,61 @@ +/** + * Tests that renameCollection commands do not abort initial sync when users specify + * 'allowUnsafeRenamesDuringInitialSync'. + */ + +(function() { + 'use strict'; + + load("jstests/libs/check_log.js"); + + var parameters = TestData.setParameters; + if (parameters && parameters.indexOf("use3dot2InitialSync=true") != -1) { + jsTest.log("Skipping this test because use3dot2InitialSync was provided."); + return; + } + + const basename = 'initial_sync_rename_collection_unsafe'; + + const rst = new ReplSetTest({name: basename, nodes: 1}); + rst.startSet(); + rst.initiate(); + + const dbName = 'd'; + const primary = rst.getPrimary(); + const primaryDB = primary.getDB(dbName); + + assert.writeOK(primaryDB['foo'].save({})); + + jsTestLog('Bring up a new node'); + const secondary = rst.add({setParameter: {allowUnsafeRenamesDuringInitialSync: true}}); + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'alwaysOn'})); + rst.reInitiate(); + assert.eq(primary, rst.getPrimary(), 'Primary changed after reconfig'); + + // Wait for fail point message to be logged. + checkLog.contains(secondary, + 'initial sync - initialSyncHangBeforeCopyingDatabases fail point enabled'); + + jsTestLog('Rename collection on the primary'); + assert.commandWorked(primaryDB['foo'].renameCollection('renamed')); + + assert.commandWorked(secondary.adminCommand( + {configureFailPoint: 'initialSyncHangBeforeCopyingDatabases', mode: 'off'})); + + checkLog.contains(secondary, 'allowUnsafeRenamesDuringInitialSync set to true'); + + jsTestLog('Wait for both nodes to be up-to-date'); + rst.awaitSecondaryNodes(); + rst.awaitReplication(); + + jsTestLog('Check that all collections were renamed correctly on the secondary'); + const secondaryDB = secondary.getDB(dbName); + assert.eq(secondaryDB['renamed'].find().itcount(), 1, 'renamed collection does not exist'); + assert.eq(secondaryDB['foo'].find().itcount(), 0, 'collection `foo` exists after rename'); + + let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1})); + assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 0); + + rst.stopSet(); +})(); diff --git a/jstests/replsets/last_vote.js b/jstests/replsets/last_vote.js index b9e0474217c..44d349f3237 100644 --- a/jstests/replsets/last_vote.js +++ b/jstests/replsets/last_vote.js @@ -134,9 +134,6 @@ "replSetRequestVotes response had the wrong term: " + tojson(response)); assert(!response.voteGranted, "node granted vote in term before last vote doc: " + tojson(response)); - assert.eq(response.reason, - "candidate's term is lower than mine", - "replSetRequestVotes response had the wrong reason: " + tojson(response)); assertNodeHasLastVote(node0, term, rst.nodes[0]); assertCurrentTerm(node0, term); @@ -178,9 +175,6 @@ "replSetRequestVotes response had the wrong term: " + tojson(response)); assert(!response.voteGranted, "node granted vote in term of last vote doc: " + tojson(response)); - assert.eq(response.reason, - "already voted for another candidate this term", - "replSetRequestVotes response had the wrong reason: " + tojson(response)); assertNodeHasLastVote(node0, term, rst.nodes[0]); assertCurrentTerm(node0, term); diff --git a/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js b/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js new file mode 100644 index 00000000000..5e170378de2 --- /dev/null +++ b/jstests/replsets/libs/apply_ops_concurrent_non_atomic.js @@ -0,0 +1,232 @@ +/** + * This test ensures that multiple non-atomic applyOps commands can run concurrently. + * Prior to SERVER-29802, applyOps would acquire the global lock regardless of the + * atomicity of the operations (as a whole) being applied. + * + * Every instance of ApplyOpsConcurrentNonAtomicTest is configured with an "options" document + * with the following format: + * { + * ns1: <string>, + * ns1: <string>, + * requiresDocumentLevelConcurrency: <bool>, + * } + * + * ns1: + * Fully qualified namespace of first set of CRUD operations. For simplicity, only insert + * operations will be used. The set of documents generated for the inserts into ns1 will have + * _id values distinct from those generated for ns2. + * + * ns2: + * Fully qualified namespace of second set of CRUD operations. This may be the same namespace as + * ns1. As with ns1, only insert operations will be used. + * + * requiresDocumentLevelConcurrency: + * Set to true if this test case can only be run with a storage engine that supports document + * level concurrency. + */ +var ApplyOpsConcurrentNonAtomicTest = function(options) { + 'use strict'; + + load('jstests/concurrency/fsm_workload_helpers/server_types.js'); + + if (!(this instanceof ApplyOpsConcurrentNonAtomicTest)) { + return new ApplyOpsConcurrentNonAtomicTest(options); + } + + // Capture the 'this' reference + var self = this; + + self.options = options; + + /** + * Logs message using test name as prefix. + */ + function testLog(message) { + jsTestLog('ApplyOpsConcurrentNonAtomicTest: ' + message); + } + + /** + * Creates an array of insert operations for applyOps into collection 'coll'. + */ + function generateInsertOps(coll, numOps, id) { + // Explicit 'use strict' to prevent mozjs from injecting its own "use strict" directive + // (with incorrect indentation) when we convert this function into a string for + // startParallelShell(). + 'use strict'; + const ops = Array(numOps).fill('ignored').map((unused, i) => { + return {op: 'i', ns: coll.getFullName(), o: {_id: (id * numOps + i), id: id}}; + }); + return ops; + } + + /** + * Runs applyOps in non-atomic mode to insert 'numOps' documents into collection 'coll'. + */ + function applyOpsInsertNonAtomic(coll, numOps, id) { + 'use strict'; + const ops = generateInsertOps(coll, numOps, id); + const mydb = coll.getDB(); + assert.commandWorked(mydb.runCommand({applyOps: ops, allowAtomic: false}), + 'failed to insert documents into ' + coll.getFullName()); + } + + /** + * Parses 'numOps' and collection namespace from 'options' and runs applyOps to inserted + * generated documents. + * + * options format: + * { + * ns: <string>, + * numOps: <int>, + * id: <int>, + * } + * + * ns: + * Fully qualified namespace of collection to insert documents into. + * + * numOps: + * Number of insert operations to generate for applyOps command. + * + * id: + * Index of collection for applyOps. Used with 'numOps' to generate _id values that will not + * collide with collections with different indexes. + */ + function insertFunction(options) { + 'use strict'; + + const coll = db.getMongo().getCollection(options.ns); + const numOps = options.numOps; + const id = options.id; + + testLog('Starting to apply ' + numOps + ' operations in collection ' + coll.getFullName()); + applyOpsInsertNonAtomic(coll, numOps, id); + testLog('Successfully applied ' + numOps + ' operations in collection ' + + coll.getFullName()); + } + + /** + * Creates a function for startParallelShell() to run that will insert documents into + * collection 'coll' using applyOps. + */ + function createInsertFunction(coll, numOps, id) { + const options = { + ns: coll.getFullName(), + numOps: numOps, + id: id, + }; + const functionName = 'insertFunction_' + coll.getFullName().replace(/\./g, '_'); + const s = // + '\n\n' + // + 'const testLog = ' + testLog + ';\n\n' + // + 'const generateInsertOps = ' + generateInsertOps + ';\n\n' + // + 'const applyOpsInsertNonAtomic = ' + applyOpsInsertNonAtomic + ';\n\n' + // + 'const ' + functionName + ' = ' + insertFunction + ';\n\n' + // + functionName + '(' + tojson(options) + ');'; // + return s; + } + + /** + * Returns number of insert operations reported by serverStatus. + * In 3.4 'opcountersRepl', not 'opcounters' was previously the correct field. Now non-atomic + * ops are now replicated as they are applied and are counted toward the global op counter. + */ + function getInsertOpCount(serverStatus) { + return serverStatus.opcounters.insert; + } + + /** + * Runs the test. + */ + this.run = function() { + const options = this.options; + + assert(options.ns1, 'collection 1 namespace not provided'); + assert(options.ns2, 'collection 2 namespace not provided'); + + const replTest = new ReplSetTest({nodes: 1}); + replTest.startSet(); + replTest.initiate(); + + const primary = replTest.getPrimary(); + const adminDb = primary.getDB('admin'); + + if (options.requiresDocumentLevelConcurrency && + !supportsDocumentLevelConcurrency(adminDb)) { + testLog('Skipping test because storage engine does not support document level ' + + 'concurrency.'); + return; + } + + const coll1 = primary.getCollection(options.ns1); + const db1 = coll1.getDB(); + const coll2 = primary.getCollection(options.ns2); + const db2 = coll2.getDB(); + + assert.commandWorked(db1.createCollection(coll1.getName())); + if (coll1.getFullName() !== coll2.getFullName()) { + assert.commandWorked(db2.createCollection(coll2.getName())); + } + + // Enable fail point to pause applyOps between operations. + assert.commandWorked(primary.adminCommand( + {configureFailPoint: 'applyOpsPauseBetweenOperations', mode: 'alwaysOn'})); + + // This logs each operation being applied. + const previousLogLevel = + assert.commandWorked(primary.setLogLevel(3, 'replication')).was.replication.verbosity; + + testLog('Applying operations in collections ' + coll1.getFullName() + ' and ' + + coll2.getFullName()); + + const numOps = 100; + const insertProcess1 = + startParallelShell(createInsertFunction(coll1, numOps, 0), replTest.getPort(0)); + const insertProcess2 = + startParallelShell(createInsertFunction(coll2, numOps, 1), replTest.getPort(0)); + + // The fail point will prevent applyOps from advancing past the first operation in each + // batch of operations. If applyOps is applying both sets of operations concurrently without + // holding the global lock, the insert opcounter will eventually be incremented to 2. + try { + let insertOpCount = 0; + const expectedFinalOpCount = 2; + assert.soon( + function() { + const serverStatus = adminDb.serverStatus(); + insertOpCount = getInsertOpCount(serverStatus); + // This assertion may fail if the fail point is not implemented correctly within + // applyOps. This allows us to fail fast instead of waiting for the + // assert.soon() function to time out. + assert.lte(insertOpCount, + expectedFinalOpCount, + 'Expected at most ' + expectedFinalOpCount + + ' documents inserted with fail point enabled. ' + + 'Most recent insert operation count = ' + insertOpCount); + return insertOpCount === expectedFinalOpCount; + }, + 'Insert operation count did not reach ' + expectedFinalOpCount + + ' as expected with fail point enabled. Most recent insert operation count = ' + + insertOpCount); + } finally { + assert.commandWorked(primary.adminCommand( + {configureFailPoint: 'applyOpsPauseBetweenOperations', mode: 'off'})); + } + + insertProcess1(); + insertProcess2(); + + testLog('Successfully applied operations in collections ' + coll1.getFullName() + ' and ' + + coll2.getFullName()); + + // Reset log level. + primary.setLogLevel(previousLogLevel, 'replication'); + + const serverStatus = adminDb.serverStatus(); + assert.eq(200, + getInsertOpCount(serverStatus), + 'incorrect number of insert operations in server status after applyOps: ' + + tojson(serverStatus)); + + replTest.stopSet(); + }; +}; diff --git a/jstests/replsets/libs/apply_ops_insert_write_conflict.js b/jstests/replsets/libs/apply_ops_insert_write_conflict.js index 88b299c08b6..d0a47f5b6d8 100644 --- a/jstests/replsets/libs/apply_ops_insert_write_conflict.js +++ b/jstests/replsets/libs/apply_ops_insert_write_conflict.js @@ -36,14 +36,6 @@ var ApplyOpsInsertWriteConflictTest = function(options) { return {op: 'i', ns: t.getFullName(), o: {_id: i}}; }); - if (!options.atomic) { - // Adding a command to the list of operations to prevent the applyOps command from - // applying - // all the operations atomically. - ops.push({ns: "test.$cmd", op: "c", o: {applyOps: []}}); - numOps++; - } - // Probabilities for WCE are chosen based on empirical testing. // The probability for WCE during an atomic applyOps should be much smaller than that for // the non-atomic case because we have to attempt to re-apply the entire batch of 'numOps' @@ -62,7 +54,7 @@ var ApplyOpsInsertWriteConflictTest = function(options) { var previousLogLevel = assert.commandWorked(primaryDB.setLogLevel(3, 'replication')).was.replication.verbosity; - var applyOpsResult = primaryDB.adminCommand({applyOps: ops}); + var applyOpsResult = primaryDB.adminCommand({applyOps: ops, allowAtomic: options.atomic}); // Reset log level. primaryDB.setLogLevel(previousLogLevel, 'replication'); diff --git a/jstests/replsets/no_flapping_during_network_partition.js b/jstests/replsets/no_flapping_during_network_partition.js index 5a289b2afdd..daab9be2bca 100644 --- a/jstests/replsets/no_flapping_during_network_partition.js +++ b/jstests/replsets/no_flapping_during_network_partition.js @@ -39,7 +39,7 @@ primary.disconnect(secondary); jsTestLog("Wait long enough for the secondary to call for an election."); - checkLog.contains(secondary, "can see a healthy primary of equal or greater priority"); + checkLog.contains(secondary, "can see a healthy primary"); jsTestLog("Verify the primary and secondary do not change during the partition."); assert.eq(primary, replTest.getPrimary()); diff --git a/jstests/replsets/noop_writes_wait_for_write_concern.js b/jstests/replsets/noop_writes_wait_for_write_concern.js new file mode 100644 index 00000000000..61ffc518df4 --- /dev/null +++ b/jstests/replsets/noop_writes_wait_for_write_concern.js @@ -0,0 +1,235 @@ +/** + * This file tests that if a user initiates a write that becomes a noop due to being a duplicate + * operation, that we still wait for write concern. This is because we must wait for write concern + * on the write that made this a noop so that we can be sure it doesn't get rolled back if we + * acknowledge it. + */ + +(function() { + "use strict"; + load('jstests/libs/write_concern_util.js'); + + var name = 'noop_writes_wait_for_write_concern'; + var replTest = new ReplSetTest({ + name: name, + nodes: [{}, {rsConfig: {priority: 0}}, {rsConfig: {priority: 0}}], + }); + replTest.startSet(); + replTest.initiate(); + // Stops node 1 so that all w:3 write concerns time out. We have 3 data bearing nodes so that + // 'dropDatabase' can satisfy its implicit writeConcern: majority but still time out from the + // explicit w:3 write concern. + replTest.stop(1); + + var primary = replTest.getPrimary(); + assert.eq(primary, replTest.nodes[0]); + var dbName = 'testDB'; + var db = primary.getDB(dbName); + var collName = 'testColl'; + var coll = db[collName]; + + function dropTestCollection() { + coll.drop(); + assert.eq(0, coll.find().itcount(), "test collection not empty"); + } + + // Each entry in this array contains a command whose noop write concern behavior needs to be + // tested. Entries have the following structure: + // { + // req: <object>, // Command request object that will result in a noop + // // write after the setup function is called. + // + // setupFunc: <function()>, // Function to run to ensure that the request is a + // // noop. + // + // confirmFunc: <function(res)>, // Function to run after the command is run to ensure + // // that it executed properly. Accepts the result of + // // the noop request to validate it. + // } + var commands = []; + + commands.push({ + req: {applyOps: [{op: "i", ns: coll.getFullName(), o: {_id: 1}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.applied, 1); + assert.eq(res.results[0], true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({_id: 1}), 1); + } + }); + + // 'update' where the document to update does not exist. + commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {b: 2}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.update({a: 1}, {b: 2})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } + }); + + // 'update' where the update has already been done. + commands.push({ + req: {update: collName, updates: [{q: {a: 1}, u: {$set: {b: 2}}}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.update({a: 1}, {$set: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 1); + assert.eq(res.nModified, 0); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } + }); + + commands.push({ + req: {delete: collName, deletes: [{q: {a: 1}, limit: 1}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.writeOK(coll.remove({a: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(coll.count({a: 1}), 0); + } + }); + + commands.push({ + req: {createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({createIndexes: collName, indexes: [{key: {a: 1}, name: "a_1"}]})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.numIndexesBefore, res.numIndexesAfter); + } + }); + + // 'findAndModify' where the document to update does not exist. + commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {b: 2}}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {b: 2}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.lastErrorObject.updatedExisting, false); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({b: 2}), 1); + } + }); + + // 'findAndModify' where the update has already been done. + commands.push({ + req: {findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked( + db.runCommand({findAndModify: collName, query: {a: 1}, update: {$set: {b: 2}}})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.lastErrorObject.updatedExisting, true); + assert.eq(coll.find().itcount(), 1); + assert.eq(coll.count({a: 1, b: 2}), 1); + } + }); + + commands.push({ + req: {dropDatabase: 1}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked(db.runCommand({dropDatabase: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + } + }); + + commands.push({ + req: {drop: collName}, + setupFunc: function() { + assert.writeOK(coll.insert({a: 1})); + assert.commandWorked(db.runCommand({drop: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceNotFound); + } + }); + + commands.push({ + req: {create: collName}, + setupFunc: function() { + assert.commandWorked(db.runCommand({create: collName})); + }, + confirmFunc: function(res) { + assert.commandFailedWithCode(res, ErrorCodes.NamespaceExists); + } + }); + + commands.push({ + req: {insert: collName, documents: [{_id: 1}]}, + setupFunc: function() { + assert.writeOK(coll.insert({_id: 1})); + }, + confirmFunc: function(res) { + assert.commandWorked(res); + assert.eq(res.n, 0); + assert.eq(res.writeErrors[0].code, ErrorCodes.DuplicateKey); + assert.eq(coll.count({_id: 1}), 1); + } + }); + + function testCommandWithWriteConcern(cmd) { + // Provide a small wtimeout that we expect to time out. + cmd.req.writeConcern = {w: 3, wtimeout: 1000}; + jsTest.log("Testing " + tojson(cmd.req)); + + dropTestCollection(); + + cmd.setupFunc(); + + // We run the command on a different connection. If the the command were run on the + // same connection, then the client last op for the noop write would be set by the setup + // operation. By using a fresh connection the client last op begins as null. + // This test explicitly tests that write concern for noop writes works when the + // client last op has not already been set by a duplicate operation. + var shell2 = new Mongo(primary.host); + + // We check the error code of 'res' in the 'confirmFunc'. + var res = shell2.getDB(dbName).runCommand(cmd.req); + + try { + // Tests that the command receives a write concern error. If we don't wait for write + // concern on noop writes then we won't get a write concern error. + assertWriteConcernError(res); + cmd.confirmFunc(res); + } catch (e) { + // Make sure that we print out the response. + printjson(res); + throw e; + } + } + + commands.forEach(function(cmd) { + testCommandWithWriteConcern(cmd); + }); + +})();
\ No newline at end of file diff --git a/jstests/replsets/read_committed_with_catalog_changes.js b/jstests/replsets/read_committed_with_catalog_changes.js index 02d26759e26..7928b16d3ae 100644 --- a/jstests/replsets/read_committed_with_catalog_changes.js +++ b/jstests/replsets/read_committed_with_catalog_changes.js @@ -214,7 +214,7 @@ load("jstests/replsets/rslib.js"); // For startSetIfSupportsReadMajority. "Expected read of " + coll.getFullName() + " to block"); } - function assertReadsSucceed(coll, timeoutMs = 10000) { + function assertReadsSucceed(coll, timeoutMs = 20000) { var res = coll.runCommand('find', {"readConcern": {"level": "majority"}, "maxTimeMS": timeoutMs}); assert.commandWorked(res, 'reading from ' + coll.getFullName()); diff --git a/jstests/replsets/startParallelShell.js b/jstests/replsets/startParallelShell.js new file mode 100644 index 00000000000..beca88d19a9 --- /dev/null +++ b/jstests/replsets/startParallelShell.js @@ -0,0 +1,33 @@ +// Test startParallelShell() in a replica set. + +var db; + +(function() { + 'use strict'; + + const setName = 'rs0'; + const replSet = new ReplSetTest({name: setName, nodes: 3}); + const nodes = replSet.nodeList(); + replSet.startSet(); + replSet.initiate(); + + const url = replSet.getURL(); + print("* Connecting to " + url); + const mongo = new Mongo(url); + db = mongo.getDB('admin'); + assert.eq(url, mongo.host, "replSet.getURL() should match active connection string"); + + print("* Starting parallel shell on --host " + db.getMongo().host); + startParallelShell('db.coll0.insert({test: "connString only"});'); + assert.soon(function() { + return db.coll0.find({test: "connString only"}).count() === 1; + }); + + const uri = new MongoURI(url); + const port0 = uri.servers[0].port; + print("* Starting parallel shell w/ --port " + port0); + startParallelShell('db.coll0.insert({test: "explicit port"});', port0); + assert.soon(function() { + return db.coll0.find({test: "explicit port"}).count() === 1; + }); +})(); diff --git a/jstests/replsets/too_stale_secondary.js b/jstests/replsets/too_stale_secondary.js index 369662e5f16..1a0c28b3454 100644 --- a/jstests/replsets/too_stale_secondary.js +++ b/jstests/replsets/too_stale_secondary.js @@ -34,7 +34,7 @@ "use strict"; function getFirstOplogEntry(conn) { - return conn.getDB('local').oplog.rs.find().sort({ts: 1}).limit(1)[0]; + return conn.getDB('local').oplog.rs.find().sort({$natural: 1}).limit(1)[0]; } /** diff --git a/jstests/sharding/auth.js b/jstests/sharding/auth.js index 61b25f10dde..037532acaec 100644 --- a/jstests/sharding/auth.js +++ b/jstests/sharding/auth.js @@ -38,7 +38,7 @@ name: "auth", mongos: 1, shards: 0, - other: {keyFile: "jstests/libs/key1", chunkSize: 1, enableAutoSplit: true}, + other: {keyFile: "jstests/libs/key1", chunkSize: 1, enableAutoSplit: false}, }); if (s.getDB('admin').runCommand('buildInfo').bits < 64) { @@ -167,6 +167,7 @@ s.getDB("test").foo.remove({}); var num = 10000; + assert.commandWorked(s.s.adminCommand({split: "test.foo", middle: {x: num / 2}})); var bulk = s.getDB("test").foo.initializeUnorderedBulkOp(); for (i = 0; i < num; i++) { bulk.insert( diff --git a/jstests/sharding/auto_rebalance_parallel.js b/jstests/sharding/auto_rebalance_parallel.js index 955319b8c5d..c7078a6898a 100644 --- a/jstests/sharding/auto_rebalance_parallel.js +++ b/jstests/sharding/auto_rebalance_parallel.js @@ -1,43 +1,71 @@ /** * Tests that the cluster is balanced in parallel in one balancer round (standalone). */ + (function() { 'use strict'; var st = new ShardingTest({shards: 4}); + var config = st.s0.getDB('config'); assert.commandWorked(st.s0.adminCommand({enableSharding: 'TestDB'})); st.ensurePrimaryShard('TestDB', st.shard0.shardName); - assert.commandWorked(st.s0.adminCommand({shardCollection: 'TestDB.TestColl', key: {Key: 1}})); - var coll = st.s0.getDB('TestDB').TestColl; + function prepareCollectionForBalance(collName) { + assert.commandWorked(st.s0.adminCommand({shardCollection: collName, key: {Key: 1}})); + + var coll = st.s0.getCollection(collName); + + // Create 4 chunks initially and ensure they get balanced within 1 balancer round + assert.writeOK(coll.insert({Key: 1, Value: 'Test value 1'})); + assert.writeOK(coll.insert({Key: 10, Value: 'Test value 10'})); + assert.writeOK(coll.insert({Key: 20, Value: 'Test value 20'})); + assert.writeOK(coll.insert({Key: 30, Value: 'Test value 30'})); + + assert.commandWorked(st.splitAt(collName, {Key: 10})); + assert.commandWorked(st.splitAt(collName, {Key: 20})); + assert.commandWorked(st.splitAt(collName, {Key: 30})); + + // Move two of the chunks to shard0001 so we have option to do parallel balancing + assert.commandWorked(st.moveChunk(collName, {Key: 20}, st.shard1.shardName)); + assert.commandWorked(st.moveChunk(collName, {Key: 30}, st.shard1.shardName)); - // Create 4 chunks initially and ensure they get balanced within 1 balancer round - assert.writeOK(coll.insert({Key: 1, Value: 'Test value 1'})); - assert.writeOK(coll.insert({Key: 10, Value: 'Test value 10'})); - assert.writeOK(coll.insert({Key: 20, Value: 'Test value 20'})); - assert.writeOK(coll.insert({Key: 30, Value: 'Test value 30'})); + assert.eq(2, config.chunks.find({ns: collName, shard: st.shard0.shardName}).itcount()); + assert.eq(2, config.chunks.find({ns: collName, shard: st.shard1.shardName}).itcount()); + } - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 10})); - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 20})); - assert.commandWorked(st.splitAt('TestDB.TestColl', {Key: 30})); + function checkCollectionBalanced(collName) { + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard0.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard1.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard2.shardName}).itcount()); + assert.eq(1, config.chunks.find({ns: collName, shard: st.shard3.shardName}).itcount()); + } - // Move two of the chunks to shard0001 so we have option to do parallel balancing - assert.commandWorked(st.moveChunk('TestDB.TestColl', {Key: 20}, st.shard1.shardName)); - assert.commandWorked(st.moveChunk('TestDB.TestColl', {Key: 30}, st.shard1.shardName)); + function countMoves(collName) { + return config.changelog.find({what: 'moveChunk.start', ns: collName}).itcount(); + } - assert.eq(2, st.s0.getDB('config').chunks.find({shard: st.shard0.shardName}).itcount()); - assert.eq(2, st.s0.getDB('config').chunks.find({shard: st.shard1.shardName}).itcount()); + prepareCollectionForBalance('TestDB.TestColl1'); + prepareCollectionForBalance('TestDB.TestColl2'); + + // Count the moveChunk start attempts accurately and ensure that only the correct number of + // migrations are scheduled + const testColl1InitialMoves = countMoves('TestDB.TestColl1'); + const testColl2InitialMoves = countMoves('TestDB.TestColl2'); - // Do enable the balancer and wait for a single balancer round st.startBalancer(); st.awaitBalancerRound(); + st.awaitBalancerRound(); st.stopBalancer(); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard0.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard1.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard2.shardName}).itcount()); - assert.eq(1, st.s0.getDB('config').chunks.find({shard: st.shard3.shardName}).itcount()); + checkCollectionBalanced('TestDB.TestColl1'); + checkCollectionBalanced('TestDB.TestColl2'); + + assert.eq(2, countMoves('TestDB.TestColl1') - testColl1InitialMoves); + assert.eq(2, countMoves('TestDB.TestColl2') - testColl2InitialMoves); + + // Ensure there are no migration errors reported + assert.eq(0, config.changelog.find({what: 'moveChunk.error'}).itcount()); st.stop(); })(); diff --git a/jstests/sharding/autosplit.js b/jstests/sharding/autosplit.js index bb34021487f..0eba386b6a3 100644 --- a/jstests/sharding/autosplit.js +++ b/jstests/sharding/autosplit.js @@ -4,7 +4,12 @@ (function() { 'use strict'; - var s = new ShardingTest({name: "auto1", shards: 2, mongos: 1, other: {enableAutoSplit: true}}); + var s = new ShardingTest({ + name: "auto1", + shards: 2, + mongos: 1, + other: {enableAutoSplit: true, chunkSize: 10}, + }); assert.commandWorked(s.s0.adminCommand({enablesharding: "test"})); s.ensurePrimaryShard('test', 'shard0001'); diff --git a/jstests/sharding/mapReduce_inSharded_outSharded.js b/jstests/sharding/mapReduce_inSharded_outSharded.js index d1aba2599f0..d73eb517e98 100644 --- a/jstests/sharding/mapReduce_inSharded_outSharded.js +++ b/jstests/sharding/mapReduce_inSharded_outSharded.js @@ -1,60 +1,70 @@ -var verifyOutput = function(out) { - printjson(out); - assert.eq(out.counts.input, 51200, "input count is wrong"); - assert.eq(out.counts.emit, 51200, "emit count is wrong"); - assert.gt(out.counts.reduce, 99, "reduce count is wrong"); - assert.eq(out.counts.output, 512, "output count is wrong"); -}; - -var st = new ShardingTest( - {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); - -st.adminCommand({enablesharding: "mrShard"}); -st.ensurePrimaryShard('mrShard', 'shard0001'); -st.adminCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}}); - -var db = st.getDB("mrShard"); - -var bulk = db.srcSharded.initializeUnorderedBulkOp(); -for (j = 0; j < 100; j++) { - for (i = 0; i < 512; i++) { - bulk.insert({j: j, i: i}); +(function() { + "use strict"; + + var verifyOutput = function(out) { + printjson(out); + assert.eq(out.counts.input, 51200, "input count is wrong"); + assert.eq(out.counts.emit, 51200, "emit count is wrong"); + assert.gt(out.counts.reduce, 99, "reduce count is wrong"); + assert.eq(out.counts.output, 512, "output count is wrong"); + }; + + var st = new ShardingTest( + {shards: 2, verbose: 1, mongos: 1, other: {chunkSize: 1, enableBalancer: true}}); + + var admin = st.s0.getDB('admin'); + + assert.commandWorked(admin.runCommand({enablesharding: "mrShard"})); + st.ensurePrimaryShard('mrShard', 'shard0001'); + assert.commandWorked( + admin.runCommand({shardcollection: "mrShard.srcSharded", key: {"_id": 1}})); + + var db = st.s0.getDB("mrShard"); + + var bulk = db.srcSharded.initializeUnorderedBulkOp(); + for (var j = 0; j < 100; j++) { + for (var i = 0; i < 512; i++) { + bulk.insert({j: j, i: i}); + } + } + assert.writeOK(bulk.execute()); + + function map() { + emit(this.i, 1); + } + function reduce(key, values) { + return Array.sum(values); } -} -assert.writeOK(bulk.execute()); - -function map() { - emit(this.i, 1); -} -function reduce(key, values) { - return Array.sum(values); -} - -// sharded src sharded dst -var suffix = "InShardedOutSharded"; - -var out = - db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); -verifyOutput(out); - -out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); -verifyOutput(out); -assert(out.results != 'undefined', "no results for inline"); - -out = db.srcSharded.mapReduce( - map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); -verifyOutput(out); - -out = db.runCommand({ - mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce - map: map, - reduce: reduce, - out: "mrBasic" + "srcSharded", -}); -verifyOutput(out); + + // sharded src sharded dst + var suffix = "InShardedOutSharded"; + + var out = + db.srcSharded.mapReduce(map, reduce, {out: {replace: "mrReplace" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {merge: "mrMerge" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {reduce: "mrReduce" + suffix, sharded: true}}); + verifyOutput(out); + + out = db.srcSharded.mapReduce(map, reduce, {out: {inline: 1}}); + verifyOutput(out); + assert(out.results != 'undefined', "no results for inline"); + + out = db.srcSharded.mapReduce( + map, reduce, {out: {replace: "mrReplace" + suffix, db: "mrShardOtherDB", sharded: true}}); + verifyOutput(out); + + out = db.runCommand({ + mapReduce: "srcSharded", // use new name mapReduce rather than mapreduce + map: map, + reduce: reduce, + out: "mrBasic" + "srcSharded", + }); + verifyOutput(out); + + st.stop(); + +})(); diff --git a/jstests/sharding/migrateBig_balancer.js b/jstests/sharding/migrateBig_balancer.js index 9eb50c2168e..03e98fa5493 100644 --- a/jstests/sharding/migrateBig_balancer.js +++ b/jstests/sharding/migrateBig_balancer.js @@ -31,11 +31,8 @@ assert.eq(40, coll.count(), "prep1"); - printjson(coll.stats()); - - admin.printShardingStatus(); - - admin.runCommand({shardcollection: "" + coll, key: {_id: 1}}); + assert.commandWorked(admin.runCommand({shardcollection: "" + coll, key: {_id: 1}})); + st.printShardingStatus(); assert.lt( 5, mongos.getDB("config").chunks.find({ns: "test.stuff"}).count(), "not enough chunks"); @@ -56,5 +53,4 @@ }, "never migrated", 10 * 60 * 1000, 1000); st.stop(); - })(); diff --git a/jstests/sharding/movechunk_commit_changelog_stats.js b/jstests/sharding/movechunk_commit_changelog_stats.js new file mode 100644 index 00000000000..d257bb6ed94 --- /dev/null +++ b/jstests/sharding/movechunk_commit_changelog_stats.js @@ -0,0 +1,41 @@ +// +// Tests that the changelog entry for moveChunk.commit contains stats on the migration. +// + +(function() { + 'use strict'; + + var st = new ShardingTest({mongos: 1, shards: 2}); + var kDbName = 'db'; + + var mongos = st.s0; + var shard0 = st.shard0.shardName; + var shard1 = st.shard1.shardName; + + assert.commandWorked(mongos.adminCommand({enableSharding: kDbName})); + st.ensurePrimaryShard(kDbName, shard0); + + function assertCountsInChangelog() { + let changeLog = st.s.getDB('config').changelog.find({what: 'moveChunk.commit'}).toArray(); + assert.gt(changeLog.length, 0); + for (let i = 0; i < changeLog.length; i++) { + assert(changeLog[i].details.hasOwnProperty('counts') || + changeLog[i].details.hasOwnProperty('clonedBytes')); + } + } + + var ns = kDbName + '.fooHashed'; + assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {_id: 'hashed'}})); + + var aChunk = mongos.getDB('config').chunks.findOne({_id: RegExp(ns), shard: shard0}); + assert(aChunk); + + // Assert counts field exists in the changelog entry for moveChunk.commit + assert.commandWorked( + mongos.adminCommand({moveChunk: ns, bounds: [aChunk.min, aChunk.max], to: shard1})); + assertCountsInChangelog(); + + mongos.getDB(kDbName).fooHashed.drop(); + + st.stop(); +})();
\ No newline at end of file diff --git a/jstests/sharding/printShardingStatus.js b/jstests/sharding/printShardingStatus.js index 85330311fd4..cdc36999e51 100644 --- a/jstests/sharding/printShardingStatus.js +++ b/jstests/sharding/printShardingStatus.js @@ -7,6 +7,8 @@ var st = new ShardingTest({shards: 1, mongos: 2, config: 1, other: {smallfiles: true}}); + var standalone = MongoRunner.runMongod(); + var mongos = st.s0; var admin = mongos.getDB("admin"); @@ -85,9 +87,10 @@ testBasicVerboseOnly(outputVerbose); // Take a copy of the config db, in order to test the harder-to-setup cases below. + // Copy into a standalone to also test running printShardingStatus() against a config dump. // TODO: Replace this manual copy with copydb once SERVER-13080 is fixed. var config = mongos.getDB("config"); - var configCopy = mongos.getDB("configCopy"); + var configCopy = standalone.getDB("configCopy"); config.getCollectionInfos().forEach(function(c) { // Create collection with options. assert.commandWorked(configCopy.createCollection(c.name, c.options)); @@ -140,11 +143,11 @@ configCopy.mongos.remove({}); var output = grabStatusOutput(configCopy, false); - assertPresentInOutput(output, "most recently active mongoses:\n\tnone", "no mongoses"); + assertPresentInOutput(output, "most recently active mongoses:\n none", "no mongoses"); var output = grabStatusOutput(configCopy, true); assertPresentInOutput( - output, "most recently active mongoses:\n\tnone", "no mongoses (verbose)"); + output, "most recently active mongoses:\n none", "no mongoses (verbose)"); assert(mongos.getDB(dbName).dropDatabase()); @@ -237,5 +240,7 @@ assert(mongos.getDB("test").dropDatabase()); + MongoRunner.stopMongod(standalone); + st.stop(); })(); diff --git a/jstests/sharding/shard_existing_coll_chunk_count.js b/jstests/sharding/shard_existing_coll_chunk_count.js new file mode 100644 index 00000000000..60145fef712 --- /dev/null +++ b/jstests/sharding/shard_existing_coll_chunk_count.js @@ -0,0 +1,165 @@ +/** + * This test confirms that after sharding a collection with some pre-existing data, + * the resulting chunks aren't auto-split too aggressively. + */ +(function() { + 'use strict'; + + var s = new ShardingTest({ + name: "shard_existing_coll_chunk_count", + shards: 1, + mongos: 1, + other: {enableAutoSplit: true}, + }); + + assert.commandWorked(s.s.adminCommand({enablesharding: "test"})); + + var collNum = 0; + var overhead = Object.bsonsize({_id: ObjectId(), i: 1, pad: ""}); + + var getNumberChunks = function(ns) { + return s.configRS.getPrimary().getDB("config").getCollection("chunks").count({ns}); + }; + + var runCase = function(opts) { + // Expected options. + assert.gte(opts.docSize, 0); + assert.gte(opts.stages.length, 2); + + // Compute padding. + if (opts.docSize < overhead) { + var pad = ""; + } else { + var pad = (new Array(opts.docSize - overhead + 1)).join(' '); + } + + collNum++; + var db = s.getDB("test"); + var collName = "coll" + collNum; + var coll = db.getCollection(collName); + var i = 0; + var limit = 0; + var stageNum = 0; + var stage = opts.stages[stageNum]; + + // Insert initial docs. + var bulk = coll.initializeUnorderedBulkOp(); + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + bulk.insert({i, pad}); + } + assert.writeOK(bulk.execute()); + + // Create shard key index. + assert.commandWorked(coll.createIndex({i: 1})); + + // Shard collection. + assert.commandWorked(s.s.adminCommand({shardcollection: coll.getFullName(), key: {i: 1}})); + + // Confirm initial number of chunks. + var numChunks = getNumberChunks(coll.getFullName()); + assert.eq(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' initial chunks, but found ' + numChunks + '\nopts: ' + tojson(opts) + + '\nchunks:\n' + s.getChunksString(coll.getFullName())); + + // Do the rest of the stages. + for (stageNum = 1; stageNum < opts.stages.length; stageNum++) { + stage = opts.stages[stageNum]; + + // Insert the later docs (one at a time, to maximise the autosplit effects). + limit += stage.numDocsToInsert; + for (; i < limit; i++) { + coll.insert({i, pad}); + } + + // Confirm number of chunks for this stage. + var numChunks = getNumberChunks(coll.getFullName()); + assert.eq(numChunks, + stage.expectedNumChunks, + 'in ' + coll.getFullName() + ' expected ' + stage.expectedNumChunks + + ' chunks for stage ' + stageNum + ', but found ' + numChunks + + '\nopts: ' + tojson(opts) + '\nchunks:\n' + + s.getChunksString(coll.getFullName())); + } + }; + + // Original problematic case. + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 20000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Original problematic case (worse). + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 90000, expectedNumChunks: 1}, + {numDocsToInsert: 7, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Pathological case #1. + runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8191, expectedNumChunks: 1}, + {numDocsToInsert: 2, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Pathological case #2. + runCase({ + docSize: 522, + stages: [ + {numDocsToInsert: 8192, expectedNumChunks: 1}, + {numDocsToInsert: 8192, expectedNumChunks: 1}, + ], + }); + + // Lower chunksize to 1MB, and restart the mongos for it to take. + assert.writeOK( + s.getDB("config").getCollection("settings").update({_id: "chunksize"}, {$set: {value: 1}}, { + upsert: true + })); + s.restartMongos(0); + + // Original problematic case, scaled down to smaller chunksize. + runCase({ + docSize: 0, + stages: [ + {numDocsToInsert: 10000, expectedNumChunks: 1}, + {numDocsToInsert: 10, expectedNumChunks: 1}, + {numDocsToInsert: 20, expectedNumChunks: 1}, + {numDocsToInsert: 40, expectedNumChunks: 1}, + {numDocsToInsert: 1000, expectedNumChunks: 1}, + ], + }); + + // Docs just smaller than half chunk size. + runCase({ + docSize: 510 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 6}, + {numDocsToInsert: 10, expectedNumChunks: 12}, + ], + }); + + // Docs just larger than half chunk size. + runCase({ + docSize: 514 * 1024, + stages: [ + {numDocsToInsert: 10, expectedNumChunks: 10}, + {numDocsToInsert: 10, expectedNumChunks: 20}, + ], + }); + + s.stop(); +})(); diff --git a/jstests/sharding/shard_identity_rollback.js b/jstests/sharding/shard_identity_rollback.js index c7b6fedaacc..37cc8b10726 100644 --- a/jstests/sharding/shard_identity_rollback.js +++ b/jstests/sharding/shard_identity_rollback.js @@ -89,8 +89,9 @@ } }, function() { - var oldPriOplog = priConn.getDB('local').oplog.rs.find().sort({ts: -1}).toArray(); - var newPriOplog = newPriConn.getDB('local').oplog.rs.find().sort({ts: -1}).toArray(); + var oldPriOplog = priConn.getDB('local').oplog.rs.find().sort({$natural: -1}).toArray(); + var newPriOplog = + newPriConn.getDB('local').oplog.rs.find().sort({$natural: -1}).toArray(); return "timed out waiting for original primary to shut down after rollback. " + "Old primary oplog: " + tojson(oldPriOplog) + "; new primary oplog: " + tojson(newPriOplog); diff --git a/jstests/sharding/write_cmd_auto_split.js b/jstests/sharding/write_cmd_auto_split.js index 1cf9b5ab39a..95151b1e7e9 100644 --- a/jstests/sharding/write_cmd_auto_split.js +++ b/jstests/sharding/write_cmd_auto_split.js @@ -40,7 +40,7 @@ assert.eq(1, configDB.chunks.find().itcount()); - for (var x = 0; x < 1100; x++) { + for (var x = 0; x < 3100; x++) { assert.writeOK(testDB.runCommand({ update: 'update', updates: [{q: {x: x}, u: {x: x, v: doc1k}, upsert: true}], @@ -80,7 +80,7 @@ // Note: Estimated 'chunk size' tracked by mongos is initialized with a random value so // we are going to be conservative. - for (var x = 0; x < 1100; x += 400) { + for (var x = 0; x < 3100; x += 400) { var docs = []; for (var y = 0; y < 400; y++) { @@ -101,7 +101,7 @@ assert.eq(1, configDB.chunks.find().itcount()); - for (var x = 0; x < 1100; x += 400) { + for (var x = 0; x < 3100; x += 400) { var docs = []; for (var y = 0; y < 400; y++) { |