diff options
60 files changed, 981 insertions, 1354 deletions
diff --git a/SConstruct b/SConstruct index 6657d605904..bc1fc314f7b 100644 --- a/SConstruct +++ b/SConstruct @@ -450,7 +450,17 @@ asio = has_option( "asio" ) usePCH = has_option( "usePCH" ) -env = Environment( BUILD_DIR=variantDir, +# On 3.0 and beyond, we honor several scons Variables on the command +# line. Those don't exist on 2.6, but it is easy to get in the habit +# of using them. By default SCons silently ignores them, which can be +# very confusing if you say 'scons CC=clang' on the v2.6 branch, for +# instance. We create an empty Variables object here and feed it to +# the Environment constructor so that we can ask for any unknown +# variables (and all should be), below. +env_vars = Variables() + +env = Environment( variables=env_vars, + BUILD_DIR=variantDir, DIST_ARCHIVE_SUFFIX='.tgz', EXTRAPATH=get_option("extrapath"), MODULE_BANNERS=[], @@ -472,6 +482,12 @@ env = Environment( BUILD_DIR=variantDir, CONFIGURELOG = '#' + scons_data_dir + '/config.log' ) +# Report any unknown variables as an error. +unknown_vars = env_vars.UnknownVariables() +if unknown_vars: + print "Unknown variables specified: {0}".format(", ".join(unknown_vars.keys())) + Exit(1) + if has_option("cache"): EnsureSConsVersion( 2, 3, 0 ) if has_option("release"): diff --git a/doxygenConfig b/doxygenConfig index 4eb2fc93d94..799528a7caa 100644 --- a/doxygenConfig +++ b/doxygenConfig @@ -3,7 +3,7 @@ #--------------------------------------------------------------------------- DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = MongoDB -PROJECT_NUMBER = 2.6.11 +PROJECT_NUMBER = 2.6.12 OUTPUT_DIRECTORY = docs/doxygen CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 0c5a35855f2..72da3d09b6d 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -923,6 +923,7 @@ tasks: ${python|python} buildscripts/smoke.py --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json ${test_flags|} tool - name: push + patchable: false depends_on: - name: "*" stepback: false @@ -939,7 +940,7 @@ tasks: silent: true script: | set -o errexit - echo "${signing_auth_token}" > signing_auth_token + echo "${signing_auth_token_26}" > signing_auth_token - command: shell.exec params: working_dir: src @@ -1643,7 +1644,7 @@ buildvariants: display_name: Windows 64-bit modules: ~ run_on: - - windows-64-test + - windows-64-vs2010-test expansions: push_path: win32 push_bucket: downloads.mongodb.org @@ -1657,7 +1658,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: aggregation - name: auth - name: core @@ -1688,7 +1689,7 @@ buildvariants: display_name: Windows 64-bit DEBUG modules: ~ run_on: - - windows-64-test + - windows-64-vs2010-test expansions: push_path: win32 push_bucket: downloads.mongodb.org @@ -1701,7 +1702,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: aggregation - name: auth - name: core @@ -1725,7 +1726,7 @@ buildvariants: display_name: Windows 64-bit 2008R2+ modules: ~ run_on: - - windows-64-test + - windows-64-vs2010-test expansions: push_path: win32 push_bucket: downloads.mongodb.org @@ -1739,7 +1740,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: aggregation - name: auth - name: core @@ -1768,7 +1769,7 @@ buildvariants: display_name: Windows 64-bit 2008R2+ DEBUG modules: ~ run_on: - - windows-64-test + - windows-64-vs2010-test expansions: push_path: win32 push_bucket: downloads.mongodb.org @@ -1781,7 +1782,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: aggregation - name: auth - name: core @@ -1806,7 +1807,7 @@ buildvariants: modules: - enterprise run_on: - - windows-64-test + - windows-64-vs2010-test expansions: push_path: win32 push_bucket: downloads.10gen.com @@ -1820,7 +1821,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: audit - name: jsCore_auth - name: replicasets_auth @@ -1851,7 +1852,7 @@ buildvariants: tasks: - name: compile distros: - - windows-64-compile + - windows-64-vs2010-compile - name: aggregation - name: auth - name: core diff --git a/jstests/auth/authz_modifications_access_control.js b/jstests/auth/authz_modifications_access_control.js index cc9b94a9cbc..69e9b2b2c6e 100644 --- a/jstests/auth/authz_modifications_access_control.js +++ b/jstests/auth/authz_modifications_access_control.js @@ -65,6 +65,19 @@ function runTest(conn) { assert.commandFailedWithCode(res, authzErrorCode); })(); + (function () { + jsTestLog("Testing role creation, of user-defined roles with same name as built-in roles"); + + var cmdObj = {createRole: "readWrite", roles: [], privileges: []}; + var res = adminUserAdmin.runCommand(cmdObj); + assert.commandFailed(res, tojson(cmdObj)); + + var roleObj = adminUserAdmin.system.roles.findOne({role: "readWrite", db: "admin"}); + // double check that no role object named "readWrite" has been created + assert(!roleObj, "user-defined \"readWrite\" role was created: " + tojson(roleObj)); + + })(); + (function testViewUser() { jsTestLog("Testing viewing user information"); diff --git a/jstests/core/queryoptimizer3.js b/jstests/core/queryoptimizer3.js index a90c7985839..8cace47dc66 100644 --- a/jstests/core/queryoptimizer3.js +++ b/jstests/core/queryoptimizer3.js @@ -12,22 +12,31 @@ for( i = 0; i < 100; ++i ) { for( j = 0; j < 100; ++j ) { t.save({a:j,b:j}); } - m = i % 5; - if ( m == 0 ) { - t.count({a:{$gte:0},b:{$gte:0}}); - } - else if ( m == 1 ) { - t.find({a:{$gte:0},b:{$gte:0}}).itcount(); - } - else if ( m == 2 ) { - t.remove({a:{$gte:0},b:{$gte:0}}); - } - else if ( m == 3 ) { - t.update({a:{$gte:0},b:{$gte:0}},{}); + + try { + m = i % 5; + if ( m == 0 ) { + t.count({a:{$gte:0},b:{$gte:0}}); + } + else if ( m == 1 ) { + t.find({a:{$gte:0},b:{$gte:0}}).itcount(); + } + else if ( m == 2 ) { + t.remove({a:{$gte:0},b:{$gte:0}}); + } + else if ( m == 3 ) { + t.update({a:{$gte:0},b:{$gte:0}},{}); + } + else if ( m == 4 ) { + t.distinct('x',{a:{$gte:0},b:{$gte:0}}); + } } - else if ( m == 4 ) { - t.distinct('x',{a:{$gte:0},b:{$gte:0}}); + catch (e) { + print("Op killed during yield: " + e.message); } } p(); + +// Ensure that the server is still responding. +assert.commandWorked(db.runCommand({isMaster: 1})); diff --git a/jstests/libs/trace_missing_docs.js b/jstests/libs/trace_missing_docs.js index 3faf50b4606..0a939f4d8d9 100644 --- a/jstests/libs/trace_missing_docs.js +++ b/jstests/libs/trace_missing_docs.js @@ -74,9 +74,7 @@ function traceMissingDoc( coll, doc, mongos ) { } var compareOps = function( opA, opB ) { - if ( opA.ts < opB.ts ) return -1; - if ( opB.ts < opA.ts ) return 1; - else return 0; + return bsonWoCompare( opA.ts, opB.ts ); } allOps.sort( compareOps ); diff --git a/jstests/multiVersion/upgrade_cluster_v3_to_v4.js b/jstests/multiVersion/upgrade_cluster_v3_to_v4.js deleted file mode 100644 index 139aa83f1c2..00000000000 --- a/jstests/multiVersion/upgrade_cluster_v3_to_v4.js +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Tests upgrading a cluster which has 2.0-style sharded collections as well as 2.2-style sharded - * collections to be compatible with 2.4. - */ - -load( './jstests/multiVersion/libs/multi_rs.js' ) -load( './jstests/multiVersion/libs/multi_cluster.js' ) - -// BIG OUTER LOOP, RS CLUSTER OR NOT! -for( var test = 0; test < 4; test++ ){ - -var isRSCluster = (test % 2 == 1); -var isSyncCluster = (test / 2 >= 1); - -jsTest.log( "Starting" + ( isRSCluster ? " (replica set)" : "" ) + " cluster" + - ( isSyncCluster ? " (sync)" : "" ) + "..." ); - -jsTest.log( "Starting 2.0 cluster..." ); - -var options = { - - mongosOptions : { binVersion : "2.0" }, - configOptions : { binVersion : "2.0" }, - shardOptions : { binVersion : "2.0" }, - - rsOptions : { binVersion : "2.0" /*, oplogSize : 100, smallfiles : null */ }, - - separateConfig : true, - sync : isSyncCluster, - rs : isRSCluster -} - -var st = new ShardingTest({ shards : 2, mongos : 2, other : options }); - -// Just stop balancer, to simulate race conds -st.setBalancer(false); - -var shards = st.s0.getDB("config").shards.find().toArray(); -var configConnStr = st._configDB; - -// -// Make sure 2.4 mongoses won't start in 2.0 cluster -// - -jsTest.log("Starting v2.4 mongos in 2.0 cluster...") - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr }) -assert.eq(null, mongos); - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongos); - -jsTest.log("2.4 mongoses did not start or upgrade in 2.0 cluster (which is correct).") - -// -// Add sharded collection in 2.0 cluster -// - -/** - * Creates a sharded collection and splits on both shards, to ensure multi-version clusters - * have metadata created by mongoses and mongods of different versions. - */ -var createShardedCollection = function(admin, coll) { - - printjson(admin.runCommand({ enableSharding : coll.getDB() + "" })); - printjson(admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id })); - printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } })); - - printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } })); - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id })); - - printjson(admin.runCommand({ split : coll + "", middle : { _id : -300 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : -200 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : -100 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 100 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 200 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 300 } })); - - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : -200 }, to : shards[1]._id })); - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 200 }, to : shards[0]._id })); -} - -jsTest.log("Creating new collection in 2.0 cluster..."); - -var mongos20 = st.s0; - -createShardedCollection(mongos20.getDB("admin"), mongos20.getCollection("foo20.bar20")); - -st.printShardingStatus(); - -// -// Upgrade 2.0 cluster to 2.0/2.2 -// - -jsTest.log("Upgrading 2.0 cluster to 2.0/2.2 cluster..."); - -st.upgradeCluster(MongoRunner.versionIterator(["2.0","2.2"])); -// Restart of mongos here is unfortunately necessary, connection pooling otherwise causes problems -st.restartMongoses(); - -// -// Make sure 2.4 mongoses won't start in 2.0/2.2 cluster -// - -jsTest.log("Starting v2.4 mongos in 2.0/2.2 cluster....") - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr }) -assert.eq(null, mongos); - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongos); - -jsTest.log("2.4 mongoses did not start or upgrade in 2.0/2.2 cluster (which is correct).") - -// -// Add sharded collections in 2.0/2.2 cluster -// - -jsTest.log("Creating new collection in 2.0/2.2 cluster..."); - -var mongos22 = st.getMongosAtVersion("2.2") -var mongos20 = st.getMongosAtVersion("2.0") - -createShardedCollection(mongos20.getDB("admin"), mongos20.getCollection("fooMixed20.barMixed20")); -createShardedCollection(mongos22.getDB("admin"), mongos22.getCollection("fooMixed22.barMixed22")); - -st.printShardingStatus(); - -// -// Upgrade 2.0/2.2 cluster to all mongoses at 2.2 -// - -jsTest.log("Upgrading all mongoses to 2.2..."); - -st.upgradeCluster("2.2", { upgradeShards : false, upgradeMongos : true, upgradeConfigs : true }); -st.restartMongoses(); - -// -// Make sure 2.4 mongoses won't start in 2.0/2.2 shard cluster -// - -jsTest.log("Starting v2.4 mongos in 2.0/2.2 (shard) cluster....") - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr }) -assert.eq(null, mongos); - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongos); - -jsTest.log("2.4 mongoses did not start or upgrade in 2.0/2.2 (shard) cluster (which is correct).") - -// -// Upgrade 2.0/2.2 cluster to only 2.2 -// - -jsTest.log("Upgrading 2.0/2.2 cluster to 2.2 cluster..."); - -st.upgradeCluster("2.2"); -st.restartMongoses(); - -// -// Make sure 2.4 mongoses will successfully upgrade in 2.4 cluster -// - -jsTest.log("Starting v2.4 mongos in 2.2 cluster....") - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr }) -assert.eq(null, mongos); - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongos); -MongoRunner.stopMongos(mongos); - -jsTest.log("2.4 mongoses started in 2.2 cluster.") - -// -// Add sharded collection in 2.2 cluster -// - -jsTest.log("Creating new collection in 2.2 cluster..."); - -var mongos22 = st.getMongosAtVersion("2.2") - -createShardedCollection(mongos22.getDB("admin"), mongos22.getCollection("foo22.bar22")); - -st.printShardingStatus(); - -// -// Verify that all collections have correct epochs in new cluster -// - -var config = mongos22.getDB("config") - -var collections = config.collections.find().toArray(); -var chunks = config.chunks.find().toArray(); - -for (var i = 0; i < collections.length; i++) { - - var collection = collections[i]; - var epoch = collection.lastmodEpoch - assert(epoch); - - for (var j = 0; j < chunks.length; j++) { - var chunk = chunks[j]; - if (chunk.ns != collection._id) continue; - assert.eq(chunk.lastmodEpoch, epoch); - } -} - -// -// Verify backup collections are present -// - -var collNames = config.getCollectionNames(); - -var hasBackupColls = false; -var hasBackupChunks = false; - -for (var i = 0; i < collNames.length; i++) { - var collName = collNames[i]; - if (/^collections-backup/.test(collName)) { - print("Found backup collections " + collName + "...") - hasBackupColls = true; - } - if (/^chunks-backup/.test(collName)) { - print("Found backup chunks " + collName + "...") - hasBackupChunks = true; - } -} - -assert(hasBackupColls); -assert(hasBackupChunks); - -// -// Verify cluster version is correct -// - -var version = config.getMongo().getCollection("config.version").findOne(); -printjson(version) - -assert.eq(version.version, 3); -assert.eq(version.minCompatibleVersion, 3); -assert.eq(version.currentVersion, 4); -assert(version.clusterId); -assert.eq(version.excluding, undefined); - -jsTest.log("DONE!") - -st.stop(); - -} // END OUTER LOOP FOR RS CLUSTER - - - - - - - - diff --git a/jstests/multiVersion/upgrade_cluster_v3_to_v4_db.js b/jstests/multiVersion/upgrade_cluster_v3_to_v4_db.js deleted file mode 100644 index 93413f1bfb1..00000000000 --- a/jstests/multiVersion/upgrade_cluster_v3_to_v4_db.js +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Tests upgrading a config db which has different types of sharded collection data from v3 to v4 - */ - -load( './jstests/multiVersion/libs/multi_rs.js' ) -load( './jstests/multiVersion/libs/multi_cluster.js' ) - -jsTest.log( "Starting 2.2 cluster..." ); - -var options = { - - mongosOptions : { binVersion : "2.2" }, - configOptions : { binVersion : "2.2" }, - shardOptions : { binVersion : "2.2" }, - - separateConfig : true, - sync : false -} - -var st = new ShardingTest({ shards : 1, mongos : 1, other : options }); - -// Just set balancer to false, don't wait for it -st.setBalancer(false); - -var mongos = st.s0 -var config = mongos.getDB("config") -var admin = mongos.getDB("admin") -var shards = config.shards.find().toArray(); -var configConnStr = st._configDB; -var originalVersion = config.getMongo().getCollection("config.version").findOne(); - -st.printShardingStatus(); - -var resetBackupDBs = function() { - - var configConn = new Mongo(configConnStr); - var databases = configConn.getDBs().databases; - - // - // Drop all new backup databases - // - - for (var i = 0; i < databases.length; i++) { - var dbName = databases[i].name + ""; - if (!/^config$|^admin$|^local$/.test(dbName)) { - print("Dropping " + dbName + "...") - configConn.getDB(dbName).dropDatabase(); - } - } -} - -var resetVersion = function() { - config.getMongo().getCollection("config.version").update({ _id : 1 }, originalVersion, true); - assert.eq(null, config.getLastError()); -} - -var checkUpgraded = function() { - - // - // Verify backup collections are present - // - - var collNames = config.getCollectionNames(); - - var hasBackupColls = false; - var hasBackupChunks = false; - - for (var i = 0; i < collNames.length; i++) { - var collName = collNames[i]; - if (/^collections-backup/.test(collName)) { - print("Found backup collections " + collName + "...") - hasBackupColls = true; - } - if (/^chunks-backup/.test(collName)) { - print("Found backup chunks " + collName + "...") - hasBackupChunks = true; - } - } - - assert(hasBackupColls); - assert(hasBackupChunks); - - // - // Verify cluster version is correct - // - - var version = config.getMongo().getCollection("config.version").findOne(); - printjson(version) - - assert.eq(version.version, 3); - assert.eq(version.minCompatibleVersion, 3); - assert.eq(version.currentVersion, 4); - assert(version.clusterId); - assert.eq(version.excluding, undefined) - -} - -// -// Default config upgrade -// - -jsTest.log("Upgrading empty config server from v3 to v4..."); - -// Make sure up -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongosNew); -MongoRunner.stopMongos(mongosNew); -checkUpgraded(); -resetVersion(); -resetBackupDBs(); - - -// -// Invalid config.version upgrade -// - -jsTest.log("Clearing config.version collection...") - -config.getMongo().getCollection("config.version").remove({}) -assert.eq(null, config.getLastError()); - -// Make sure down -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongosNew); -resetVersion(); -resetBackupDBs(); - - -// -// Bad config.version upgrade -// - -jsTest.log("Adding bad config.version data...") - -config.getMongo().getCollection("config.version").update({ _id : 1 }, { $unset : { version : 1 } }); -assert.eq(null, config.getLastError()); - -// Make sure down -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongosNew); -resetVersion(); -resetBackupDBs(); - - -// -// Invalid config.collections upgrade -// - -jsTest.log("Adding bad sharded collection data...") - -var coll = mongos.getCollection("foo.bar"); - -printjson(admin.runCommand({ enableSharding : coll.getDB() + "" })); -printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } })); -printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } })); - -config.collections.update({ _id : coll + "" }, { $set : { lastmodEpoch : ObjectId() }}); -assert.eq(null, config.getLastError()); - -// Make sure down -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongosNew); -resetBackupDBs(); - - -// -// Dropped collection upgrade -// - -jsTest.log("Adding bad (dropped) sharded collection data...") - -printjson(coll.drop()); -// Disable balancing on the (dropped) coll to trigger additional collection validation. -// At least in 2.2, dropping a collection drops the noBalance flag as well, but this has been seen -// in the wild. -// TODO: Enable when 2.4.4 comes out -//sh.disableBalancing( coll ); -printjson(config.collections.find().toArray()); - -// Make sure up -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongosNew); -MongoRunner.stopMongos(mongosNew); -checkUpgraded(); -resetVersion(); -resetBackupDBs(); - - -// -// Invalid chunks collection upgrade -// - -jsTest.log("Adding bad sharded chunks data...") - -var coll = mongos.getCollection("foo2.bar2"); - -printjson(admin.runCommand({ enableSharding : coll.getDB() + "" })); -printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } })); -printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } })); - -config.chunks.update({ ns : coll + "" }, { $set : { lastmodEpoch : ObjectId() }}); -assert.eq(null, config.getLastError()); - -// Make sure down -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.eq(null, mongosNew); - -// -// Upgrade with modified old upgrade data -// - -jsTest.log("Fiddling with data from last failed upgrade...") - -var upgradeCollRegex = /^collections-upgrade/; -var configColls = config.getCollectionNames(); -for (var i = 0; i < configColls.length; i++) { - var configColl = configColls[i]; - if (upgradeCollRegex.test(configColl)) { - print("Dropping collection: " + configColl); - config.getCollection(configColl).drop(); - break; - } -} - -// Fix chunk data -config.chunks.update({}, { $unset : { versionEpoch : 1 }, $unset : { lastmodEpoch : 1 }}, false, true); -assert.eq(null, config.getLastError()); - -// Make sure up -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongosNew); -MongoRunner.stopMongos(mongosNew); -checkUpgraded(); - -jsTest.log("DONE!") - -st.stop(); - - - - - - - - diff --git a/jstests/multiVersion/upgrade_cluster_v3_to_v4_wait_for_mongos.js b/jstests/multiVersion/upgrade_cluster_v3_to_v4_wait_for_mongos.js deleted file mode 100644 index ba859edd7a7..00000000000 --- a/jstests/multiVersion/upgrade_cluster_v3_to_v4_wait_for_mongos.js +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Tests upgrading a cluster where there was a recently active mongos process. - */ - -load( './jstests/multiVersion/libs/multi_rs.js' ) -load( './jstests/multiVersion/libs/multi_cluster.js' ) - -jsTest.log( "Starting 2.2 cluster..." ); - -var options = { - - mongosOptions : { binVersion : "2.2" }, - configOptions : { binVersion : "2.2" }, - shardOptions : { binVersion : "2.2" }, - - separateConfig : true, - sync : false -}; - -var st = new ShardingTest({ shards : 1, mongos : 1, other : options }); - -// Turn balancer off, don't wait -st.setBalancer(false); - -var mongos = st.s0; - -jsTest.log( "Starting v2.0 mongos..." ); - -var mongos20 = MongoRunner.runMongos({ binVersion : "2.0", configdb : st._configDB }) - -jsTest.log( "Waiting for 2.0 ping document..." ); - -var hasPing = function() { - return mongos.getCollection("config.mongos").findOne({ _id : RegExp(":" + mongos20.port + "$") }) != null; -} - -assert.soon( hasPing ); - -jsTest.log( "Stopping 2.0 mongos..." ); - -MongoRunner.stopMongos(mongos20); - -jsTest.log( "Upgrade should be unsuccessful..." ); - -// Make sure down -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : st._configDB, upgrade : "" }) -assert.eq(null, mongosNew); - -jsTest.log("Resetting time to zero..."); - -printjson(mongos.getCollection("config.mongos").findOne({ _id : RegExp(":" + mongos20.port + "$") })); - -mongos.getCollection("config.mongos").update({ _id : RegExp(":" + mongos20.port + "$") }, - { $set : { ping : new Date(0) } }); -assert.eq(null, mongos.getDB("config").getLastError()); - -printjson(mongos.getCollection("config.mongos").findOne({ _id : RegExp(":" + mongos20.port + "$") })); - -jsTest.log("Trying to restart mongos..."); - -// Make sure up -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : st._configDB, upgrade : "" }) -assert.neq(null, mongosNew); - -jsTest.log("Mongos started!"); - -MongoRunner.stopMongos(mongosNew); - -jsTest.log("DONE!") - -st.stop(); - - - diff --git a/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_parallel_ops.js b/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_parallel_ops.js deleted file mode 100644 index 77579de5615..00000000000 --- a/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_parallel_ops.js +++ /dev/null @@ -1,281 +0,0 @@ -/** - * Tests upgrading a config db which has different types of sharded collection data from v3 to v4 - */ - -load( './jstests/multiVersion/libs/multi_rs.js' ) -load( './jstests/multiVersion/libs/multi_cluster.js' ) -load( './jstests/libs/test_background_ops.js' ) - -jsTest.log( "Starting mixed 2.2/2.0 cluster..." ); - -var options = { - - mongosOptions : { binVersion : MongoRunner.versionIterator(["2.0","2.2"]) }, - configOptions : { binVersion : MongoRunner.versionIterator(["2.0","2.2"]) }, - shardOptions : { binVersion : MongoRunner.versionIterator(["2.0","2.2"]) }, - - separateConfig : true, - sync : true -} - -// -// Create a basic mixed sharded cluster with extra mongoses to do work while we upgrade -// - -var st = new ShardingTest({ shards : 2, mongos : 3, other : options }); - -var mongos = st.s0 -var parallelMongoses = st._mongos.concat([]).splice(1); -var config = mongos.getDB("config") -var admin = mongos.getDB("admin") - -var shards = config.shards.find().toArray(); -var configConnStr = st._configDB; -var originalVersion = config.getMongo().getCollection("config.version").findOne(); - -// -// Shard a collection for each of our extra mongoses and distribute chunks to v2.0 and v2.2 shards -// - -st.stopBalancer(); - -var shardedColls = []; -for (var i = 0; i < parallelMongoses.length; i++) { - - var parallelMongos = parallelMongoses[i]; - - var coll = parallelMongos.getCollection("foo" + i + ".bar" + i); - var admin = parallelMongos.getDB("admin"); - - printjson(admin.runCommand({ enableSharding : coll.getDB() + "" })); - printjson(admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id })); - printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } })); - - printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } })); - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id })); - - printjson(admin.runCommand({ split : coll + "", middle : { _id : -0.3 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : -0.2 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : -0.1 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 0.1 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 0.2 } })); - printjson(admin.runCommand({ split : coll + "", middle : { _id : 0.3 } })); - - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : -0.2 }, to : shards[1]._id })); - printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 0.2 }, to : shards[0]._id })); - - shardedColls.push(coll); -} - -st.startBalancer(); - -st.printShardingStatus(); - -// -// Upgrade cluster to v2.2 -// - -jsTest.log("Upgrading cluster to 2.2...") - -st.upgradeCluster("2.2"); -//Restart of mongos here is unfortunately necessary, connection pooling otherwise causes problems -st.restartMongoses(); - -jsTest.log("Cluster upgraded...") - -var mongos = st.s0 -var parallelMongoses = st._mongos.concat([]).splice(1); -var config = mongos.getDB("config"); -var admin = mongos.getDB("admin"); - -function splitAndMove( mongosURL, shards, ns ){ - - var coll = null; - - // Make sure we can eventually connect to the mongos - assert.soon( function(){ - try{ - print("Waiting for connect to " + mongosURL + "..."); - coll = new Mongo(mongosURL).getCollection(ns + ""); - return true; - } - catch (e) { - printjson(e); - return false; - } - }) - - var splitCount = 0; - var moveCount = 0; - - jsTest.log("Starting splits and moves to " + ns + "...") - - while (!isFinished()) { - try { - // Split 75% of the time - var isSplit = Random.rand() < 0.75; - var key = (Random.rand() * 2.0) - 1.0; - var shard = shards[parseInt(Random.rand() * shards.length)] - - var admin = coll.getMongo().getDB("admin"); - var result = null; - - if (isSplit) { - result = admin.runCommand({ split : coll + "", - middle : { _id : key } }); - splitCount++; - result["isSplit"] = true; - } - else { - result = admin.runCommand({ moveChunk : coll + "", - find : key, - to : shards[shard]._id }) - moveCount++; - result["isMove"] = true; - } - - if (result.ok != 0) printjson(result); - } - catch (e) { - sleep(1); - printjson(e); - } - } - - jsTest.log("Finished splits and moves to " + ns + "...") - return { splitCount : splitCount, moveCount : moveCount }; -} - -// -// Start split and move operations in the 2.2 cluster -// - -jsTest.log("Starting split and move operations...") - -var staticMongod = MongoRunner.runMongod({}) -printjson( staticMongod ) - -var joinSplitAndMoves = []; -for (var i = 0; i < parallelMongoses.length; i++) { - joinSplitAndMoves.push( - startParallelOps( staticMongod, // The connection where the test info is passed and stored - splitAndMove, - [ parallelMongoses[i].host, shards, shardedColls[i] + "" ] ) - ); -} - -jsTest.log("Sleeping for metadata operations to start...") - -sleep(10 * 1000); - -printShardingStatus(config, true); - -// -// Do a config upgrade while the split and move operations are active -// - -jsTest.log("Upgrading config db from v3 to v4..."); - -// Just stop the balancer, but don't wait for it to stop, to simulate race conds -st.setBalancer(false); -printjson(config.settings.find().toArray()); - -var startTime = new Date(); - -// Make sure up -var mongosNew = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongosNew); -MongoRunner.stopMongos(mongosNew); - -var endTime = new Date(); - -jsTest.log( "Config db upgrade took " + ((endTime - startTime) / 1000) + " secs" ); - -// -// Stop the split and move operations -// - -for (var i = 0; i < parallelMongoses.length; i++) { - joinSplitAndMoves[i](); -} - -printShardingStatus(config, true) - -// -// Make sure our cluster was successfully upgraded with epochs -// - -var checkUpgraded = function() { - - // - // Verify backup collections are present - // - - var collNames = config.getCollectionNames(); - - var hasBackupColls = false; - var hasBackupChunks = false; - - for (var i = 0; i < collNames.length; i++) { - var collName = collNames[i]; - if (/^collections-backup/.test(collName)) { - print("Found backup collections " + collName + "...") - hasBackupColls = true; - } - if (/^chunks-backup/.test(collName)) { - print("Found backup chunks " + collName + "...") - hasBackupChunks = true; - } - } - - assert(hasBackupColls); - assert(hasBackupChunks); - - // - // Verify that all collections have correct epochs in new cluster - // - var collections = config.collections.find().toArray(); - - var chunks = config.chunks.find().toArray(); - for (var i = 0; i < collections.length; i++) { - - var collection = collections[i]; - if (collection.dropped) continue; - - var epoch = collection.lastmodEpoch - assert(epoch); - - for (var j = 0; j < chunks.length; j++) { - var chunk = chunks[j]; - if (chunk.ns != collection._id) continue; - assert.eq(chunk.lastmodEpoch, epoch); - } - } - - // - // Verify cluster version is correct - // - - var version = config.getMongo().getCollection("config.version").findOne(); - printjson(version) - - assert.eq(version.version, 3); - assert.eq(version.minCompatibleVersion, 3); - assert.eq(version.currentVersion, 4); - assert(version.clusterId); - assert.eq(version.excluding, undefined); -} - -checkUpgraded(); - -jsTest.log("DONE!") - -st.stop(); - - - - - - - - diff --git a/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_stale_mongod.js b/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_stale_mongod.js deleted file mode 100644 index 36119e85483..00000000000 --- a/jstests/multiVersion/upgrade_cluster_v3_to_v4_with_stale_mongod.js +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Tests upgrading a cluster to config version 4 with epochs and ensures 2.2 mongod will write the - * correct epochs on the next move/split. - */ - -load( './jstests/multiVersion/libs/multi_rs.js' ) -load( './jstests/multiVersion/libs/multi_cluster.js' ) - -jsTest.log( "Starting 2.0 cluster..." ); - -var options = { - - mongosOptions : { binVersion : "2.0" }, - configOptions : { binVersion : "2.0" }, - shardOptions : { binVersion : "2.0" }, - - separateConfig : true, - sync : false -} - -var st = new ShardingTest({ shards : 3, mongos : 2, other : options }); - -// Stop balancer, otherwise the balancer lock can get wedged for 15 mins after our upgrades -// on bad mongos shutdowns. -st.stopBalancer(); - -var shards = st.s0.getDB("config").shards.find().toArray(); -var configConnStr = st._configDB; - -// -// Add sharded collection in 2.0 cluster -// - -jsTest.log("Creating new collection in 2.0 cluster..."); - -var mongos20 = st.s0; -var coll = mongos20.getCollection("foo.bar"); -var admin = mongos20.getDB("admin"); - -printjson(admin.runCommand({ enableSharding : coll.getDB() + "" })); -printjson(admin.runCommand({ movePrimary : coll.getDB() + "", to : shards[0]._id })); -printjson(admin.runCommand({ shardCollection : coll + "", key : { _id : 1 } })); - -st.printShardingStatus(); - -// -// Upgrade 2.0 cluster to 2.0/2.2 -// - -jsTest.log("Upgrading 2.0 cluster to 2.2 cluster..."); - -st.upgradeCluster("2.2"); -// Restart of mongos here is unfortunately necessary, connection pooling otherwise causes problems -st.restartMongoses(); - -var mongos22A = st.s0; -var mongos22B = st.s1; - -jsTest.log("Performing metadata operations without epochs..."); - -var coll = mongos22A.getCollection("foo.bar"); -var admin = mongos22A.getDB("admin"); -var config = mongos22A.getDB("config"); - -//Split collection into several parts -printjson(admin.runCommand({ split : coll + "", middle : { _id : 0 } })); -printjson(admin.runCommand({ split : coll + "", middle : { _id : 1 } })); -printjson(admin.runCommand({ split : coll + "", middle : { _id : 2 } })); - -//Put one part on each shard -printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 1 }, to : shards[1]._id })); -printjson(admin.runCommand({ moveChunk : coll + "", find : { _id : 2 }, to : shards[2]._id })); - -//Split the first chunk into two parts, so all shards will have chunks after one migrate -printjson(admin.runCommand({ split : coll + "", middle : { _id : 0.5 } })); - -// Make sure mongos22B is up-to-date with the correct version -mongos22B.getCollection("foo.bar").findOne(); - -printjson(config.chunks.find().toArray()); - -// -// Upgrade cluster to new version -// - -jsTest.log("Starting v2.4 mongos in 2.2 cluster....") - -var mongos = MongoRunner.runMongos({ binVersion : "2.4", configdb : configConnStr, upgrade : "" }) -assert.neq(null, mongos); -MongoRunner.stopMongos(mongos); - -jsTest.log("2.4 mongos upgraded cluster.") - -// -// Do more metadata operations with 2.2 mongoses without updating versions -// - -jsTest.log("Doing more metadata operations with un-refreshed 2.2 mongoses and ds"); - -// Do a move operation -printjson(mongos22A.getDB("admin").runCommand({ moveChunk : coll + "", - find : { _id : 0 }, to : shards[1]._id })); - -// Do a split operation -printjson(mongos22B.getDB("admin").runCommand({ split : coll + "", - middle : { _id : 3 } })); - -printjson(config.chunks.find().toArray()); - -// -// Check that this didn't screw up our upgrade -// - -jsTest.log("Was upgraded?") - -var checkUpgraded = function() { - - // - // Verify that all collections have correct epochs in new cluster - // - var collections = config.collections.find().toArray(); - - var chunks = config.chunks.find().toArray(); - for (var i = 0; i < collections.length; i++) { - - var collection = collections[i]; - if (collection.dropped) continue; - - var epoch = collection.lastmodEpoch - assert(epoch); - - for (var j = 0; j < chunks.length; j++) { - var chunk = chunks[j]; - if (chunk.ns != collection._id) continue; - assert.eq(chunk.lastmodEpoch, epoch); - } - } - - // - // Verify cluster version is correct - // - - var version = config.getMongo().getCollection("config.version").findOne(); - printjson(version) - - assert.eq(version.version, 3); - assert.eq(version.minCompatibleVersion, 3); - assert.eq(version.currentVersion, 4); - assert(version.clusterId); - assert.eq(version.excluding, undefined); -} - -checkUpgraded(); - -jsTest.log("DONE!") - -st.stop(); - - - - - - - - diff --git a/jstests/noPassthroughWithMongod/dur_remove_old_journals.js b/jstests/noPassthroughWithMongod/dur_remove_old_journals.js index e309eee7b2d..334d6f5ffae 100644 --- a/jstests/noPassthroughWithMongod/dur_remove_old_journals.js +++ b/jstests/noPassthroughWithMongod/dur_remove_old_journals.js @@ -1,52 +1,62 @@ // this test makes sure that old journal files are removed -// tunables -STRING_SIZE = 1024*1024; -NUM_TO_INSERT = 2.5*1024; -PATH = MongoRunner.dataDir + "/dur_remove_old_journals"; -SYNC_DELAY = 5; // must be a number - -conn = startMongodEmpty("--port", 30001, "--dbpath", PATH, "--dur", "--smallfiles", "--syncdelay", ''+SYNC_DELAY); +var dbpath = MongoRunner.dataDir + "/dur_remove_old_journals"; +var conn = MongoRunner.runMongod({ + journal: "", + smallfiles: "", + syncdelay: 5, // seconds between fsyncs. + dbpath: dbpath, +}); db = conn.getDB("test"); -longString = 'x'; -while (longString.length < STRING_SIZE) - longString += longString; +// Returns true if j._0 exists. +function firstJournalFileExists() { + var files = listFiles(dbpath + "/journal"); + for (var i = 0; i < files.length; i++) { + if (files[i].baseName === "j._0") { + return true; + } + } + return false; +} + +// Represents the cummulative total of the number of journal files created. +function getLatestJournalFileNum() { + var files = listFiles(dbpath + "/journal"); + var latest = 0; + files.forEach(function(file) { + if (file.baseName !== "lsn") { + var fileNum = NumberInt(file.baseName[file.baseName.length - 1]); + latest = Math.max(latest, fileNum); + } + }); + return latest; +} + +var stringSize = 1024*1024; +var longString = new Array(stringSize).join("x"); + +// Insert some data to create the first journal file. +var numInserted = 0; +while (numInserted < 100) { + db.foo.insert({_id: numInserted++, s: longString}); +} +assert.soon(firstJournalFileExists, "Should have created a journal file"); -numInserted = 0; -while (numInserted < NUM_TO_INSERT){ - db.foo.insert({_id: numInserted++, s:longString}); +// Do writes until the first journal file is deleted, or we give up waiting. +var maxJournalFiles = 10; +while (firstJournalFileExists() && getLatestJournalFileNum() < maxJournalFiles) { + db.foo.insert({_id: numInserted++, s: longString}); if (numInserted % 100 == 0){ - print("numInserted: " + numInserted); + jsTestLog("numInserted: " + numInserted); db.adminCommand({fsync:1}); db.foo.remove({}); db.adminCommand({fsync:1}); } } -sleepSecs = SYNC_DELAY + 15 // long enough for data file flushing and journal keep time -print("\nWaiting " + sleepSecs + " seconds...\n"); -sleep(sleepSecs*1000); - - -files = listFiles(PATH + "/journal") -printjson(files); - -var nfiles = 0; -files.forEach(function (file) { - assert.eq('string', typeof (file.name)); // sanity checking - if (/prealloc/.test(file.name)) { - ; - } - else { - nfiles++; - } -}) - -assert.eq(2, nfiles); // latest journal file and lsn - -stopMongod(30001); - +assert(!firstJournalFileExists(), "Expected to have deleted the first journal file by now"); +MongoRunner.stopMongod(conn); print("*** success ***"); diff --git a/jstests/noPassthroughWithMongod/indexbg_restart_sigkill_secondary_noretry.js b/jstests/noPassthroughWithMongod/indexbg_restart_sigkill_secondary_noretry.js index 38cced11bb9..e2eb5d36719 100644 --- a/jstests/noPassthroughWithMongod/indexbg_restart_sigkill_secondary_noretry.js +++ b/jstests/noPassthroughWithMongod/indexbg_restart_sigkill_secondary_noretry.js @@ -33,7 +33,7 @@ // Set up replica set var replTest = new ReplSetTest({ name: 'bgIndexNoRetry', nodes: 3, - nodeOptions : {noIndexBuildRetry:""} }); + nodeOptions : {noIndexBuildRetry:"", syncdelay:1} }); var nodenames = replTest.nodeList(); // We can't use an arbiter as the third node because the -auth test tries to log on there diff --git a/jstests/sharding/find_and_modify_after_multi_write.js b/jstests/sharding/find_and_modify_after_multi_write.js new file mode 100644 index 00000000000..abce2e026a1 --- /dev/null +++ b/jstests/sharding/find_and_modify_after_multi_write.js @@ -0,0 +1,56 @@ +(function() { +"use strict"; + +/** + * Test that a targetted findAndModify will be properly routed after executing a write that + * does not perform any shard version checks. + */ +var runTest = function(writeFunc) { + var st = new ShardingTest({ shards: 2, mongos: 2 }); + + var testDB = st.s.getDB('test'); + testDB.dropDatabase(); + + testDB.adminCommand({ enableSharding: 'test' }); + st.ensurePrimaryShard('test', 'shard0000'); + testDB.adminCommand({ shardCollection: 'test.user', key: { x: 1 }}); + testDB.adminCommand({ split: 'test.user', middle: { x: 0 }}); + testDB.adminCommand({ moveChunk: 'test.user', find: { x: 0 }, to: 'shard0001' }); + + var testDB2 = st.s1.getDB('test'); + testDB2.user.insert({ x: 123456 }); + + // Force ssv initialization of 'test.user' ns for this mongos. + var doc = testDB2.user.findOne({ x: 123456 }); + assert.neq(null, doc); + + // Move chunk to bump version on a different mongos. + testDB.adminCommand({ moveChunk: 'test.user', find: { x: 0 }, to: 'shard0000' }); + + writeFunc(testDB2); + + // Issue a targetted findAndModify and check that it was upserted to the right shard. + var res = testDB2.runCommand({ + findAndModify: 'user', + query: { x: 100 }, + update: { $set: { y: 1 }}, + upsert: true + }); + + assert.commandWorked(res); + + assert.neq(null, st.d0.getDB('test').user.findOne({ x: 100 })); + assert.eq(null, st.d1.getDB('test').user.findOne({ x: 100 })); + + st.stop(); +}; + +runTest(function(db) { + db.user.update({}, { $inc: { y: 987654 }}, false, true); +}); + +runTest(function(db) { + db.user.remove({ y: 'noMatch' }, false); +}); + +})(); diff --git a/jstests/sharding/query_after_write.js b/jstests/sharding/query_after_write.js new file mode 100644 index 00000000000..b46cb40a19c --- /dev/null +++ b/jstests/sharding/query_after_write.js @@ -0,0 +1,47 @@ +(function() { +"use strict"; + +/** + * Test that queries will be properly routed after executing a write that does not + * perform any shard version checks. + */ +var runTest = function(writeFunc) { + var st = new ShardingTest({ shards: 2, mongos: 2 }); + + var testDB = st.s.getDB('test'); + testDB.dropDatabase(); + + testDB.adminCommand({ enableSharding: 'test' }); + st.ensurePrimaryShard('test', 'shard0000'); + testDB.adminCommand({ shardCollection: 'test.user', key: { x: 1 }}); + testDB.adminCommand({ split: 'test.user', middle: { x: 0 }}); + testDB.adminCommand({ moveChunk: 'test.user', find: { x: 0 }, to: 'shard0001' }); + + var testDB2 = st.s1.getDB('test'); + testDB2.user.insert({ x: 123456 }); + + // Force ssv initialization of 'test.user' ns for this mongos. + var doc = testDB2.user.findOne({ x: 123456 }); + assert.neq(null, doc); + + // Move chunk to bump version on a different mongos. + testDB.adminCommand({ moveChunk: 'test.user', find: { x: 0 }, to: 'shard0000' }); + + writeFunc(testDB2); + + // Issue a query and make sure it gets routed to the right shard. + doc = testDB2.user.findOne({ x: 123456 }); + assert.neq(null, doc); + + st.stop(); +}; + +runTest(function(db) { + db.user.update({}, { $inc: { y: 987654 }}, false, true); +}); + +runTest(function(db) { + db.user.remove({ y: 'noMatch' }, false); +}); + +})(); diff --git a/jstests/slow2/cursor_timeout.js b/jstests/slow2/cursor_timeout.js index 3e80e6c6fa7..d453713bb9d 100644 --- a/jstests/slow2/cursor_timeout.js +++ b/jstests/slow2/cursor_timeout.js @@ -1,4 +1,19 @@ -var st = new ShardingTest({ shards: 2, other: { chunkSize: 1 }}); +// Basic integration tests for the background job that periodically kills idle cursors, in both +// mongod and mongos. This test creates the following four cursors: +// +// 1. A no-timeout cursor through mongos. +// 2. A no-timeout cursor through mongod. +// 3. A normal cursor through mongos. +// 4. A normal cursor through mongod. +// +// After a period of inactivity, the test asserts that cursors #1 and #2 are still alive, and that +// #3 and #4 have been killed. + +var st = + new ShardingTest( { shards: 2, + other: { chunkSize: 1, + shardOptions: { setParameter: "cursorTimeoutMillis=1000" }, + mongosOptions: { setParameter: "cursorTimeoutMillis=1000" } } } ); st.stopBalancer(); var adminDB = st.admin; @@ -20,7 +35,7 @@ for( x = 0; x < 200; x++ ){ var chunkDoc = configDB.chunks.findOne(); var chunkOwner = chunkDoc.shard; var toShard = configDB.shards.findOne({ _id: { $ne: chunkOwner }})._id; -var cmd = { moveChunk: coll.getFullName(), find: chunkDoc.min, to: toShard }; +var cmd = { moveChunk: coll.getFullName(), find: chunkDoc.min, to: toShard, _waitForDelete: true }; var res = adminDB.runCommand( cmd ); jsTest.log( 'move result: ' + tojson( res )); @@ -44,21 +59,18 @@ shardedCursorWithNoTimeout.next(); cursorWithTimeout.next(); cursorWithNoTimeout.next(); -// Cursor cleanup is 10 minutes, but give a 8 min allowance -- -// NOTE: Due to inaccurate timing on non-Linux platforms, mongos tries -// to timeout after 10 minutes but in fact is 15+ minutes; -// SERVER-8381 -sleep( 1000 * 60 * 17 ); +// Wait until the idle cursor background job has killed the cursors that do not have the "no +// timeout" flag set. We use the "cursorTimeoutMillis" setParameter above to reduce the amount of +// time we need to wait here. +sleep( 5000 ); assert.throws( function(){ shardedCursorWithTimeout.itcount(); } ); assert.throws( function(){ cursorWithTimeout.itcount(); } ); -var freshShardedItCount = coll.find().itcount(); // +1 because we already advanced once -assert.eq( freshShardedItCount, shardedCursorWithNoTimeout.itcount() + 1 ); +assert.eq( coll.count(), shardedCursorWithNoTimeout.itcount() + 1 ); -var freshItCount = shardColl.find().itcount(); -assert.eq( freshItCount, cursorWithNoTimeout.itcount() + 1 ); +assert.eq( shardColl.count(), cursorWithNoTimeout.itcount() + 1 ); st.stop(); diff --git a/rpm/mongo.mdv.spec b/rpm/mongo.mdv.spec index 66bbc809a64..96067aa6527 100644 --- a/rpm/mongo.mdv.spec +++ b/rpm/mongo.mdv.spec @@ -1,5 +1,5 @@ %define name mongodb -%define version 2.6.11 +%define version 2.6.12 %define release %mkrel 1 Name: %{name} diff --git a/rpm/mongodb-enterprise-unstable.spec b/rpm/mongodb-enterprise-unstable.spec index af8296b7652..989c6990582 100644 --- a/rpm/mongodb-enterprise-unstable.spec +++ b/rpm/mongodb-enterprise-unstable.spec @@ -1,10 +1,10 @@ Name: mongodb-enterprise-unstable Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongodb-enterprise-unstable,mongo-enterprise-unstable -Version: 2.6.11 +Version: 2.6.12 Release: 1%{?dist} Summary: MongoDB open source document-oriented database system (enterprise metapackage) -License: AGPL 3.0 +License: Commercial URL: http://www.mongodb.org Group: Applications/Databases Requires: mongodb-enterprise-unstable-server = %{version}, mongodb-enterprise-unstable-shell = %{version}, mongodb-enterprise-unstable-mongos = %{version}, mongodb-enterprise-unstable-tools = %{version} diff --git a/rpm/mongodb-enterprise.spec b/rpm/mongodb-enterprise.spec index 7a18be7ad97..53924969bba 100644 --- a/rpm/mongodb-enterprise.spec +++ b/rpm/mongodb-enterprise.spec @@ -2,10 +2,10 @@ Name: mongodb-enterprise Conflicts: mongo-10gen, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools Obsoletes: mongodb-enterprise-unstable, mongo-enterprise-unstable, mongo-10gen-enterprise Provides: mongo-10gen-enterprise -Version: 2.6.11 +Version: 2.6.12 Release: 1%{?dist} Summary: MongoDB open source document-oriented database system (enterprise metapackage) -License: AGPL 3.0 +License: Commercial URL: http://www.mongodb.org Group: Applications/Databases Requires: mongodb-enterprise-server = %{version}, mongodb-enterprise-shell = %{version}, mongodb-enterprise-mongos = %{version}, mongodb-enterprise-tools = %{version} diff --git a/rpm/mongodb-org-unstable.spec b/rpm/mongodb-org-unstable.spec index 1b1e0b344c4..6b0bb7f2bdc 100644 --- a/rpm/mongodb-org-unstable.spec +++ b/rpm/mongodb-org-unstable.spec @@ -1,6 +1,6 @@ Name: mongodb-org-unstable Conflicts: mongo-10gen, mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org, mongodb-org-mongos, mongodb-org-server, mongodb-org-shell, mongodb-org-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools -Version: 2.6.11 +Version: 2.6.12 Release: 1%{?dist} Summary: MongoDB open source document-oriented database system (metapackage) License: AGPL 3.0 diff --git a/rpm/mongodb-org.spec b/rpm/mongodb-org.spec index 738650153e1..a2d4854d036 100644 --- a/rpm/mongodb-org.spec +++ b/rpm/mongodb-org.spec @@ -2,7 +2,7 @@ Name: mongodb-org Conflicts: mongo-10gen-enterprise, mongo-10gen-enterprise-server, mongo-10gen-unstable, mongo-10gen-unstable-enterprise, mongo-10gen-unstable-enterprise-mongos, mongo-10gen-unstable-enterprise-server, mongo-10gen-unstable-enterprise-shell, mongo-10gen-unstable-enterprise-tools, mongo-10gen-unstable-mongos, mongo-10gen-unstable-server, mongo-10gen-unstable-shell, mongo-10gen-unstable-tools, mongo18-10gen, mongo18-10gen-server, mongo20-10gen, mongo20-10gen-server, mongodb, mongodb-server, mongodb-dev, mongodb-clients, mongodb-10gen, mongodb-10gen-enterprise, mongodb-10gen-unstable, mongodb-10gen-unstable-enterprise, mongodb-10gen-unstable-enterprise-mongos, mongodb-10gen-unstable-enterprise-server, mongodb-10gen-unstable-enterprise-shell, mongodb-10gen-unstable-enterprise-tools, mongodb-10gen-unstable-mongos, mongodb-10gen-unstable-server, mongodb-10gen-unstable-shell, mongodb-10gen-unstable-tools, mongodb-enterprise, mongodb-enterprise-mongos, mongodb-enterprise-server, mongodb-enterprise-shell, mongodb-enterprise-tools, mongodb-nightly, mongodb-org-unstable, mongodb-org-unstable-mongos, mongodb-org-unstable-server, mongodb-org-unstable-shell, mongodb-org-unstable-tools, mongodb-stable, mongodb18-10gen, mongodb20-10gen, mongodb-enterprise-unstable, mongodb-enterprise-unstable-mongos, mongodb-enterprise-unstable-server, mongodb-enterprise-unstable-shell, mongodb-enterprise-unstable-tools Obsoletes: mongo-10gen Provides: mongo-10gen -Version: 2.6.11 +Version: 2.6.12 Release: 1%{?dist} Summary: MongoDB open source document-oriented database system (metapackage) License: AGPL 3.0 diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index ac900d23d27..d45ff64dd3d 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -87,6 +87,7 @@ error_code("IndexOptionsConflict", 85 ) error_code("IndexKeySpecsConflict", 86 ) error_code("OutdatedClient", 101) error_code("IncompatibleAuditMetadata", 102) +error_code("CappedPositionLost", 103) # Non-sequential error codes (for compatibility only) error_code("NetworkTimeout", 89) diff --git a/src/mongo/bson/mutable/document.cpp b/src/mongo/bson/mutable/document.cpp index 6ef13e1de43..acd09f8df79 100644 --- a/src/mongo/bson/mutable/document.cpp +++ b/src/mongo/bson/mutable/document.cpp @@ -1898,6 +1898,30 @@ namespace mutablebson { } } + Status Element::setValueElement(ConstElement setFrom) { + verify(ok()); + + // Can't set to your own root element, since this would create a circular document. + if (_doc->root() == setFrom) { + return Status(ErrorCodes::IllegalOperation, + "Attempt to set an element to its own document's root"); + } + + // Setting to self is a no-op. + // + // Setting the root is always an error so we want to fall through to the error handling in + // this case. + if (*this == setFrom && _repIdx != kRootRepIdx) { + return Status::OK(); + } + + Document::Impl& impl = getDocument().getImpl(); + ElementRep thisRep = impl.getElementRep(_repIdx); + const StringData fieldName = impl.getFieldNameForNewElement(thisRep); + Element newValue = getDocument().makeElementWithNewFieldName(fieldName, setFrom); + return setValue(newValue._repIdx); + } + BSONType Element::getType() const { verify(ok()); const Document::Impl& impl = getDocument().getImpl(); diff --git a/src/mongo/bson/mutable/element.h b/src/mongo/bson/mutable/element.h index 189b840068c..284c265f223 100644 --- a/src/mongo/bson/mutable/element.h +++ b/src/mongo/bson/mutable/element.h @@ -454,6 +454,12 @@ namespace mutablebson { */ Status setValueSafeNum(const SafeNum value); + /** Set the value of this Element to the value from another Element. + * + * The name of this Element is not modified. + */ + Status setValueElement(ConstElement setFrom); + // // Accessors diff --git a/src/mongo/bson/mutable/mutable_bson_test.cpp b/src/mongo/bson/mutable/mutable_bson_test.cpp index 3f8b2f40c74..82adff8cbfc 100644 --- a/src/mongo/bson/mutable/mutable_bson_test.cpp +++ b/src/mongo/bson/mutable/mutable_bson_test.cpp @@ -1379,6 +1379,168 @@ namespace { ASSERT_EQUALS(mongo::fromjson(outJson), doc.getObject()); } + TEST(Document, SetValueElementFromSeparateDocument) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ b : 5 }"); + const mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root().leftChild(); + mmb::ConstElement setFrom = doc2.root().leftChild(); + ASSERT_OK(setTo.setValueElement(setFrom)); + + ASSERT_EQUALS(mongo::fromjson("{ a : 5 }"), doc1.getObject()); + + // Doc containing the 'setFrom' element should be unchanged. + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + + TEST(Document, SetValueElementIsNoopWhenSetToSelf) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc(inObj); + + mmb::Element element = doc.root().leftChild(); + ASSERT_OK(element.setValueElement(element)); + + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementIsNoopWhenSetToSelfFromCopy) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc(inObj); + + mmb::Element element = doc.root().leftChild(); + mmb::ConstElement elementCopy = element; + ASSERT_OK(element.setValueElement(elementCopy)); + + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementIsNoopWhenSetToSelfNonRootElement) { + mongo::BSONObj inObj = mongo::fromjson("{ a : { b : { c: 4 } } }"); + mmb::Document doc(inObj); + + mmb::Element element = doc.root().leftChild().leftChild().leftChild(); + ASSERT_EQUALS("c", element.getFieldName()); + ASSERT_OK(element.setValueElement(element)); + + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementSetToNestedObject) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ b : { c : 5, d : 6 } }"); + const mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root().leftChild(); + mmb::ConstElement setFrom = doc2.root().leftChild(); + ASSERT_OK(setTo.setValueElement(setFrom)); + + ASSERT_EQUALS(mongo::fromjson("{ a : { c : 5, d : 6 } }"), doc1.getObject()); + + // Doc containing the 'setFrom' element should be unchanged. + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + + TEST(Document, SetValueElementNonRootElements) { + mongo::BSONObj inObj = mongo::fromjson("{ a : { b : 5, c : 6 } }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ d : { e : 8, f : 9 } }"); + const mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root().leftChild().rightChild(); + ASSERT_EQUALS("c", setTo.getFieldName()); + mmb::ConstElement setFrom = doc2.root().leftChild().leftChild(); + ASSERT_EQUALS("e", setFrom.getFieldName()); + ASSERT_OK(setTo.setValueElement(setFrom)); + + ASSERT_EQUALS(mongo::fromjson("{ a : { b : 5, c : 8 } }"), doc1.getObject()); + + // Doc containing the 'setFrom' element should be unchanged. + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + + TEST(Document, SetValueElementSetRootToSelfErrors) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc(inObj); + + mmb::Element element = doc.root(); + ASSERT_NOT_OK(element.setValueElement(element)); + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementSetRootToAnotherDocRootErrors) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ b : 5 }"); + const mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root(); + mmb::ConstElement setFrom = doc2.root(); + ASSERT_NOT_OK(setTo.setValueElement(setFrom)); + + ASSERT_EQUALS(inObj, doc1.getObject()); + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + + TEST(Document, SetValueElementSetRootToNotRootInSelfErrors) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc(inObj); + + mmb::Element setTo = doc.root(); + mmb::ConstElement setFrom = doc.root().leftChild(); + ASSERT_NOT_OK(setTo.setValueElement(setFrom)); + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementSetRootToNotRootInAnotherDocErrors) { + mongo::BSONObj inObj = mongo::fromjson("{ a : 4 }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ b : 5 }"); + const mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root(); + mmb::ConstElement setFrom = doc2.root().leftChild(); + ASSERT_NOT_OK(setTo.setValueElement(setFrom)); + + ASSERT_EQUALS(inObj, doc1.getObject()); + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + + TEST(Document, SetValueElementSetToOwnRootErrors) { + mongo::BSONObj inObj = mongo::fromjson("{ a : { b : 4 } }"); + mmb::Document doc(inObj); + + mmb::Element setTo = doc.root().leftChild().leftChild(); + ASSERT_EQUALS("b", setTo.getFieldName()); + mmb::ConstElement setFrom = doc.root(); + + ASSERT_NOT_OK(setTo.setValueElement(setFrom)); + ASSERT_EQUALS(inObj, doc.getObject()); + } + + TEST(Document, SetValueElementSetToOtherDocRoot) { + mongo::BSONObj inObj = mongo::fromjson("{ a : { b : 4 } }"); + mmb::Document doc1(inObj); + + mongo::BSONObj inObj2 = mongo::fromjson("{ c : 5 } }"); + mmb::Document doc2(inObj2); + + mmb::Element setTo = doc1.root().leftChild().leftChild(); + ASSERT_EQUALS("b", setTo.getFieldName()); + mmb::ConstElement setFrom = doc2.root(); + + ASSERT_OK(setTo.setValueElement(setFrom)); + ASSERT_EQUALS(mongo::fromjson("{ a : { b : { c : 5 } } }"), doc1.getObject()); + ASSERT_EQUALS(inObj2, doc2.getObject()); + } + TEST(Document, CreateElementWithEmptyFieldName) { mmb::Document doc; mmb::Element noname = doc.makeElementObject(mongo::StringData()); diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index 8ef983abc53..30cee0d99e7 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -1527,6 +1527,11 @@ namespace mongo { if ( comp < 0 ) continue; + uassert(28841, + str::stream() << "server " << _cursors[i].raw()->originalHost() + << " returned an error: " << me, + !_cursors[i].raw()->hasResultFlag(ResultFlag_ErrSet)); + best = me; bestFrom = i; } diff --git a/src/mongo/db/commands/index_filter_commands.cpp b/src/mongo/db/commands/index_filter_commands.cpp index 20fb72fcdfd..24073994ee0 100644 --- a/src/mongo/db/commands/index_filter_commands.cpp +++ b/src/mongo/db/commands/index_filter_commands.cpp @@ -26,6 +26,8 @@ * it in the license file. */ +#include "mongo/platform/basic.h" + #include <string> #include <sstream> @@ -39,6 +41,7 @@ #include "mongo/db/commands/index_filter_commands.h" #include "mongo/db/commands/plan_cache_commands.h" #include "mongo/db/catalog/collection.h" +#include "mongo/util/log.h" namespace { @@ -257,6 +260,9 @@ namespace mongo { // Remove entry from plan cache planCache->remove(*cq); + + LOG(0) << "Removed index filter on " << ns << " " << cq->toStringShort(); + return Status::OK(); } @@ -301,6 +307,8 @@ namespace mongo { planCache->remove(*cq); } + LOG(0) << "Removed all index filters for collection: " << ns; + return Status::OK(); } @@ -363,6 +371,8 @@ namespace mongo { // Remove entry from plan cache. planCache->remove(*cq); + LOG(0) << "Index filter set on " << ns << " " << cq->toStringShort() << " " << indexesElt; + return Status::OK(); } diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp index 61cb9f7326f..3263c25923d 100644 --- a/src/mongo/db/commands/user_management_commands.cpp +++ b/src/mongo/db/commands/user_management_commands.cpp @@ -1257,6 +1257,13 @@ namespace mongo { "Cannot create roles in the $external database")); } + if (RoleGraph::isBuiltinRole(args.roleName)) { + return appendCommandStatus( + result, + Status(ErrorCodes::BadValue, + "Cannot create roles with the same name as a built-in role")); + } + if (!args.hasRoles) { return appendCommandStatus( result, diff --git a/src/mongo/db/commands/write_commands/batch_executor.cpp b/src/mongo/db/commands/write_commands/batch_executor.cpp index eb66b396d17..f5cf279882e 100644 --- a/src/mongo/db/commands/write_commands/batch_executor.cpp +++ b/src/mongo/db/commands/write_commands/batch_executor.cpp @@ -83,6 +83,31 @@ namespace mongo { std::auto_ptr<WriteErrorDetail> _error; }; + /** + * Stores the shard version of a namespace on creation and restores it + * back on destruction if the version was changed to ignored. + */ + class UndoShardVersionIgnore { + public: + UndoShardVersionIgnore(const std::string& ns, ShardedConnectionInfo* info) + : _ns(ns), _info(info) { + if (_info) { + _originalVersion = _info->getVersion(_ns); + } + } + + ~UndoShardVersionIgnore() { + if (_info && ChunkVersion::isIgnoredVersion(_info->getVersion(_ns))) { + _info->setVersion(_ns, _originalVersion); + } + } + + private: + std::string _ns; + ChunkVersion _originalVersion; + ShardedConnectionInfo* _info; + }; + } // namespace // TODO: Determine queueing behavior we want here @@ -179,7 +204,6 @@ namespace mongo { Status wcStatus = Status::OK(); if ( wcDoc.isEmpty() ) { - // The default write concern if empty is w : 1 // Specifying w : 0 is/was allowed, but is interpreted identically to w : 1 @@ -235,6 +259,9 @@ namespace mongo { OwnedPointerVector<BatchedUpsertDetail> upsertedOwned; vector<BatchedUpsertDetail*>& upserted = upsertedOwned.mutableVector(); + UndoShardVersionIgnore undoShardVersionIgnore(request.getTargetingNS(), + ShardedConnectionInfo::get(false)); + // // Apply each batch item, possibly bulking some items together in the write lock. // Stops on error if batch is ordered. diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 90a8013dba2..05347255f52 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -53,6 +53,7 @@ #include "mongo/db/dbmessage.h" #include "mongo/db/dbwebserver.h" #include "mongo/db/dur.h" +#include "mongo/db/dur_journal.h" #include "mongo/db/index_names.h" #include "mongo/db/index_rebuilder.h" #include "mongo/db/initialize_server_global_state.h" @@ -549,7 +550,9 @@ namespace mongo { } Date_t start = jsTime(); + dur::notifyPreDataFileFlush(); int numFiles = MemoryMappedFile::flushAll( true ); + dur::notifyPostDataFileFlush(); time_flushing = (int) (jsTime() - start); _flushed(time_flushing); diff --git a/src/mongo/db/dur_journal.cpp b/src/mongo/db/dur_journal.cpp index bb466139f2e..0f72a3901c1 100644 --- a/src/mongo/db/dur_journal.cpp +++ b/src/mongo/db/dur_journal.cpp @@ -187,9 +187,10 @@ namespace mongo { _nextFileNumber = 0; _curLogFile = 0; _curFileId = 0; - _preFlushTime = 0; - _lastFlushTime = 0; - _writeToLSNNeeded = false; + _lastSeqNumberWrittenToSharedView.store(0); + _preFlushTime.store(0); + _lastFlushTime.store(0); + _writeToLSNNeeded.store(false); } boost::filesystem::path Journal::getFilePathFor(int filenumber) const { @@ -545,12 +546,9 @@ namespace mongo { void Journal::init() { verify( _curLogFile == 0 ); - MongoFile::notifyPreFlush = preFlush; - MongoFile::notifyPostFlush = postFlush; } void Journal::open() { - verify( MongoFile::notifyPreFlush == preFlush ); SimpleMutex::scoped_lock lk(_curLogFileMutex); _open(); } @@ -603,48 +601,78 @@ namespace mongo { return 0; } - unsigned long long getLastDataFileFlushTime() { - return j.lastFlushTime(); - } - /** remember "last sequence number" to speed recoveries concurrency: called by durThread only. */ - void Journal::updateLSNFile() { + void Journal::updateLSNFile(unsigned long long lsnOfCurrentJournalEntry) { RACECHECK - if( !_writeToLSNNeeded ) + if (!_writeToLSNNeeded.load()) return; - _writeToLSNNeeded = false; + _writeToLSNNeeded.store(false); try { + // Don't read from _lastFlushTime again in this function since it may change. + const uint64_t copyOfLastFlushTime = _lastFlushTime.load(); + + // Only write an LSN that is older than the journal entry we are in the middle of writing. + // If this trips, it means that _lastFlushTime got ahead of what is actually in the data + // files because lsnOfCurrentJournalEntry includes data that hasn't yet been written to the + // data files. + if (copyOfLastFlushTime >= lsnOfCurrentJournalEntry) { + severe() << "Attempting to update LSNFile to " << copyOfLastFlushTime + << " which is not older than the current journal sequence number " + << lsnOfCurrentJournalEntry; + fassertFailed(34370); + } + // os can flush as it likes. if it flushes slowly, we will just do extra work on recovery. // however, given we actually close the file, that seems unlikely. File f; f.open(lsnPath().string().c_str()); - if( !f.is_open() ) { + if (!f.is_open()) { // can get 0 if an i/o error log() << "warning: open of lsn file failed" << endl; return; } - LOG(1) << "lsn set " << _lastFlushTime << endl; + LOG(1) << "lsn set " << copyOfLastFlushTime << endl; LSNFile lsnf; - lsnf.set(_lastFlushTime); + lsnf.set(copyOfLastFlushTime); f.write(0, (char*)&lsnf, sizeof(lsnf)); // do we want to fsync here? if we do it probably needs to be async so the durthread // is not delayed. - } - catch(std::exception& e) { + } catch (std::exception& e) { log() << "warning: write to lsn file failed " << e.what() << endl; // keep running (ignore the error). recovery will be slow. } } - void Journal::preFlush() { - j._preFlushTime = Listener::getElapsedTimeMillis(); + namespace { + SimpleMutex lastGeneratedSeqNumberMutex("lastGeneratedSeqNumberMutex"); + uint64_t lastGeneratedSeqNumber = 0; + } + + uint64_t generateNextSeqNumber() { + const uint64_t now = Listener::getElapsedTimeMillis(); + SimpleMutex::scoped_lock lock(lastGeneratedSeqNumberMutex); + if (now > lastGeneratedSeqNumber) { + lastGeneratedSeqNumber = now; + } else { + // Make sure we return unique monotonically increasing numbers. + lastGeneratedSeqNumber++; + } + return lastGeneratedSeqNumber; + } + + void setLastSeqNumberWrittenToSharedView(uint64_t seqNumber) { + j._lastSeqNumberWrittenToSharedView.store(seqNumber); + } + + void notifyPreDataFileFlush() { + j._preFlushTime.store(j._lastSeqNumberWrittenToSharedView.load()); } - void Journal::postFlush() { - j._lastFlushTime = j._preFlushTime; - j._writeToLSNNeeded = true; + void notifyPostDataFileFlush() { + j._lastFlushTime.store(j._preFlushTime.load()); + j._writeToLSNNeeded.store(true); } // call from within _curLogFileMutex @@ -654,7 +682,7 @@ namespace mongo { JFile jf; jf.filename = _curLogFile->_name; - jf.lastEventTimeMs = Listener::getElapsedTimeMillis(); + jf.lastEventTimeMs = generateNextSeqNumber(); _oldJournalFiles.push_back(jf); delete _curLogFile; // close @@ -669,7 +697,11 @@ namespace mongo { while( !_oldJournalFiles.empty() ) { JFile f = _oldJournalFiles.front(); - if( f.lastEventTimeMs < _lastFlushTime + ExtraKeepTimeMs ) { + // 'f.lastEventTimeMs' is the timestamp of the last thing in the journal file. + // '_lastFlushTime' is the start time of the last successful flush of the data + // files to disk. We can't delete this journal file until the last successful + // flush time is at least 10 seconds after 'f.lastEventTimeMs'. + if (f.lastEventTimeMs + ExtraKeepTimeMs < _lastFlushTime.load()) { // eligible for deletion boost::filesystem::path p( f.filename ); log() << "old journal file will be removed: " << f.filename << endl; @@ -683,7 +715,7 @@ namespace mongo { } } - void Journal::_rotate() { + void Journal::_rotate(unsigned long long lsnOfCurrentJournalEntry) { RACECHECK; @@ -692,7 +724,7 @@ namespace mongo { if ( inShutdown() || !_curLogFile ) return; - j.updateLSNFile(); + j.updateLSNFile(lsnOfCurrentJournalEntry); if( _curLogFile && _written < DataLimitPerJournalFile ) return; @@ -781,7 +813,7 @@ namespace mongo { verify( w <= L ); stats.curr->_journaledBytes += L; _curLogFile->synchronousAppend((const void *) b.buf(), L); - _rotate(); + _rotate(h.seqNumber); } catch(std::exception& e) { log() << "error exception in dur::journal " << e.what() << endl; diff --git a/src/mongo/db/dur_journal.h b/src/mongo/db/dur_journal.h index 9e7fb5bf4b6..f595f05c3ee 100644 --- a/src/mongo/db/dur_journal.h +++ b/src/mongo/db/dur_journal.h @@ -30,6 +30,8 @@ #pragma once +#include <mongo/platform/cstdint.h> + namespace mongo { class AlignedBuilder; @@ -49,12 +51,17 @@ namespace mongo { /** assure journal/ dir exists. throws */ void journalMakeDir(); - /** check if time to rotate files; assure a file is open. - done separately from the journal() call as we can do this part - outside of lock. - only called by durThread. + /** + * Generates the next sequence number for use in the journal, guaranteed to be greater than all + * prior sequence numbers. + */ + uint64_t generateNextSeqNumber(); + + /** + * Informs the journaling system that all writes on or before the passed in sequence number have + * been written to the data files' shared mmap view. */ - void journalRotate(); + void setLastSeqNumberWrittenToSharedView(uint64_t seqNumber); /** flag that something has gone wrong during writing to the journal (not for recovery mode) @@ -64,8 +71,6 @@ namespace mongo { /** read lsn from disk from the last run before doing recovery */ unsigned long long journalReadLSN(); - unsigned long long getLastDataFileFlushTime(); - /** never throws. @param anyFiles by default we only look at j._* files. If anyFiles is true, return true if there are any files in the journal directory. acquirePathLock() uses this to @@ -79,5 +84,11 @@ namespace mongo { const unsigned JournalCommitIntervalDefault = 100; + /** + * Call these before (pre) and after (post) the datafiles are flushed to disk by the DataFileSync + * thread. These should not be called for any other flushes. + */ + void notifyPreDataFileFlush(); + void notifyPostDataFileFlush(); } } diff --git a/src/mongo/db/dur_journalimpl.h b/src/mongo/db/dur_journalimpl.h index bafc0570f19..0c12a6677d5 100644 --- a/src/mongo/db/dur_journalimpl.h +++ b/src/mongo/db/dur_journalimpl.h @@ -30,7 +30,12 @@ #pragma once +#include <boost/filesystem/path.hpp> + #include "mongo/db/dur_journalformat.h" +#include "mongo/platform/atomic_word.h" +#include "mongo/util/alignedbuilder.h" +#include "mongo/util/concurrency/mutex.h" #include "mongo/util/logfile.h" namespace mongo { @@ -59,7 +64,6 @@ namespace mongo { boost::filesystem::path getFilePathFor(int filenumber) const; - unsigned long long lastFlushTime() const { return _lastFlushTime; } void cleanup(bool log); // closes and removes journal files unsigned long long curFileId() const { return _curFileId; } @@ -77,7 +81,7 @@ namespace mongo { /** check if time to rotate files. assure a file is open. * internally called with every commit */ - void _rotate(); + void _rotate(unsigned long long lsnOfCurrentJournalEntry); void _open(); void closeCurrentJournalFile(); @@ -101,12 +105,17 @@ namespace mongo { list<JFile> _oldJournalFiles; // use _curLogFileMutex // lsn related - static void preFlush(); - static void postFlush(); - unsigned long long _preFlushTime; - unsigned long long _lastFlushTime; // data < this time is fsynced in the datafiles (unless hard drive controller is caching) - bool _writeToLSNNeeded; - void updateLSNFile(); + friend void setLastSeqNumberWrittenToSharedView(uint64_t seqNumber); + friend void notifyPreDataFileFlush(); + friend void notifyPostDataFileFlush(); + void updateLSNFile(unsigned long long lsnOfCurrentJournalEntry); + // data <= this time is in the shared view + AtomicUInt64 _lastSeqNumberWrittenToSharedView; + // data <= this time was in the shared view when the last flush to start started + AtomicUInt64 _preFlushTime; + // data <= this time is fsynced in the datafiles (unless hard drive controller is caching) + AtomicUInt64 _lastFlushTime; + AtomicInt32 _writeToLSNNeeded; }; } diff --git a/src/mongo/db/dur_preplogbuffer.cpp b/src/mongo/db/dur_preplogbuffer.cpp index 9686c21a526..c2f3b09adfa 100644 --- a/src/mongo/db/dur_preplogbuffer.cpp +++ b/src/mongo/db/dur_preplogbuffer.cpp @@ -168,7 +168,7 @@ namespace mongo { bb.reset(); h.setSectionLen(0xffffffff); // total length, will fill in later - h.seqNumber = getLastDataFileFlushTime(); + h.seqNumber = generateNextSeqNumber(); h.fileId = j.curFileId(); } diff --git a/src/mongo/db/dur_recover.cpp b/src/mongo/db/dur_recover.cpp index 2e972b45d86..cd177222760 100644 --- a/src/mongo/db/dur_recover.cpp +++ b/src/mongo/db/dur_recover.cpp @@ -385,27 +385,46 @@ namespace mongo { scoped_lock lk(_mx); RACECHECK - // Check the footer checksum before doing anything else. if (_recovering) { + // Check the footer checksum before doing anything else. verify( ((const char *)h) + sizeof(JSectHeader) == p ); if (!f->checkHash(h, len + sizeof(JSectHeader))) { log() << "journal section checksum doesn't match"; throw JournalSectionCorruptException(); } - } - if( _recovering && _lastDataSyncedFromLastRun > h->seqNumber + ExtraKeepTimeMs ) { - if( h->seqNumber != _lastSeqMentionedInConsoleLog ) { - static int n; - if( ++n < 10 ) { - log() << "recover skipping application of section seq:" << h->seqNumber << " < lsn:" << _lastDataSyncedFromLastRun << endl; + static uint64_t numJournalSegmentsSkipped = 0; + static const uint64_t kMaxSkippedSectionsToLog = 10; + if (_lastDataSyncedFromLastRun > h->seqNumber + ExtraKeepTimeMs) { + if (_appliedAnySections) { + severe() << "Journal section sequence number " << h->seqNumber + << " is lower than the threshold for applying (" + << h->seqNumber + ExtraKeepTimeMs + << ") but we have already applied some journal sections. " + << "This implies a corrupt journal file."; + fassertFailed(34369); } - else if( n == 10 ) { + + if (++numJournalSegmentsSkipped < kMaxSkippedSectionsToLog) { + log() << "recover skipping application of section seq:" << h->seqNumber + << " < lsn:" << _lastDataSyncedFromLastRun << endl; + } else if (numJournalSegmentsSkipped == kMaxSkippedSectionsToLog) { log() << "recover skipping application of section more..." << endl; } - _lastSeqMentionedInConsoleLog = h->seqNumber; + _lastSeqSkipped = h->seqNumber; + return; + } + + if (!_appliedAnySections) { + _appliedAnySections = true; + if (numJournalSegmentsSkipped >= kMaxSkippedSectionsToLog) { + // Log the last skipped section's sequence number if it hasn't been logged before. + log() << "recover final skipped journal section had sequence number " + << _lastSeqSkipped; + } + log() << "recover applying initial journal section with sequence number " + << h->seqNumber; } - return; } auto_ptr<JournalSectionIterator> i; @@ -553,6 +572,12 @@ namespace mongo { } } + if (_lastSeqSkipped && !_appliedAnySections) { + log() << "recover journal replay completed without applying any sections. " + << "This can happen if there were no writes after the last fsync of the data " + << "files. Last skipped sections had sequence number " << _lastSeqSkipped; + } + close(); if (storageGlobalParams.durOptions & StorageGlobalParams::DurScanOnly) { diff --git a/src/mongo/db/dur_recover.h b/src/mongo/db/dur_recover.h index b36c7ea6562..9ddb2550a5a 100644 --- a/src/mongo/db/dur_recover.h +++ b/src/mongo/db/dur_recover.h @@ -57,7 +57,11 @@ namespace mongo { } last; public: RecoveryJob() : _lastDataSyncedFromLastRun(0), - _mx("recovery"), _recovering(false) { _lastSeqMentionedInConsoleLog = 1; } + _lastSeqSkipped(0), + _appliedAnySections(false), + _mx("recovery"), + _recovering(false) {} + void go(vector<boost::filesystem::path>& files); ~RecoveryJob(); @@ -79,7 +83,8 @@ namespace mongo { list<boost::shared_ptr<DurableMappedFile> > _mmfs; unsigned long long _lastDataSyncedFromLastRun; - unsigned long long _lastSeqMentionedInConsoleLog; + unsigned long long _lastSeqSkipped; + bool _appliedAnySections; public: mongo::mutex _mx; // protects _mmfs private: diff --git a/src/mongo/db/dur_writetodatafiles.cpp b/src/mongo/db/dur_writetodatafiles.cpp index d9a2e345115..6d823c7aa07 100644 --- a/src/mongo/db/dur_writetodatafiles.cpp +++ b/src/mongo/db/dur_writetodatafiles.cpp @@ -31,6 +31,7 @@ #include "mongo/pch.h" #include "mongo/db/dur_commitjob.h" +#include "mongo/db/dur_journal.h" #include "mongo/db/dur_recover.h" #include "mongo/db/dur_stats.h" #include "mongo/util/concurrency/mutex.h" @@ -100,6 +101,7 @@ namespace mongo { WRITETODATAFILES_Impl1(h, uncompressed); unsigned long long m = t.micros(); stats.curr->_writeToDataFilesMicros += m; + setLastSeqNumberWrittenToSharedView(h.seqNumber); LOG(2) << "journal WRITETODATAFILES " << m / 1000.0 << "ms" << endl; } diff --git a/src/mongo/db/exec/collection_scan.cpp b/src/mongo/db/exec/collection_scan.cpp index 1a29d8b76fb..e9f6992ba94 100644 --- a/src/mongo/db/exec/collection_scan.cpp +++ b/src/mongo/db/exec/collection_scan.cpp @@ -32,6 +32,7 @@ #include "mongo/db/exec/collection_scan_common.h" #include "mongo/db/exec/filter.h" #include "mongo/db/exec/working_set.h" +#include "mongo/db/exec/working_set_common.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/structure/collection_iterator.h" #include "mongo/util/fail_point_service.h" @@ -64,7 +65,7 @@ namespace mongo { : _workingSet(workingSet), _filter(filter), _params(params), - _nsDropped(false) { + _isDead(false) { // We pre-allocate a WSID and use it to pass up fetch requests. It is only // used to pass up fetch requests and we should never use it for anything else. @@ -77,13 +78,21 @@ namespace mongo { PlanStage::StageState CollectionScan::work(WorkingSetID* out) { ++_commonStats.works; - if (_nsDropped) { return PlanStage::DEAD; } + if (_isDead) { + Status status(ErrorCodes::CappedPositionLost, str::stream() + << "CollectionScan died due to position in capped collection being deleted."); + *out = WorkingSetCommon::allocateStatusMember(_workingSet, status); + return PlanStage::DEAD; + } // Do some init if we haven't already. if (NULL == _iter) { Collection* collection = cc().database()->getCollection( _params.ns ); if ( collection == NULL ) { - _nsDropped = true; + _isDead = true; + Status status(ErrorCodes::InternalError, + str::stream() << "CollectionScan died due to NULL collection."); + *out = WorkingSetCommon::allocateStatusMember(_workingSet, status); return PlanStage::DEAD; } @@ -152,7 +161,7 @@ namespace mongo { if ((0 != _params.maxScan) && (_specificStats.docsTested >= _params.maxScan)) { return true; } - if (_nsDropped) { return true; } + if (_isDead) { return true; } if (NULL == _iter) { return false; } return _iter->isEOF(); } @@ -190,8 +199,7 @@ namespace mongo { ++_commonStats.unyields; if (NULL != _iter) { if (!_iter->recoverFromYield()) { - warning() << "Collection dropped or state deleted during yield of CollectionScan"; - _nsDropped = true; + _isDead = true; } } } diff --git a/src/mongo/db/exec/collection_scan.h b/src/mongo/db/exec/collection_scan.h index e0d4eb945f8..94edd43c7b2 100644 --- a/src/mongo/db/exec/collection_scan.h +++ b/src/mongo/db/exec/collection_scan.h @@ -75,8 +75,7 @@ namespace mongo { CollectionScanParams _params; - // True if nsdetails(_ns) == NULL on our first call to work. - bool _nsDropped; + bool _isDead; // If we want to return a DiskLoc and it points at something that's not in memory, we return // a a "please page this in" result. We allocate one WSM for this purpose at construction diff --git a/src/mongo/db/ops/modifier_rename.cpp b/src/mongo/db/ops/modifier_rename.cpp index c2a86e5cb0d..3942d11cea7 100644 --- a/src/mongo/db/ops/modifier_rename.cpp +++ b/src/mongo/db/ops/modifier_rename.cpp @@ -235,10 +235,8 @@ namespace mongo { (_preparedState->toIdxFound == (_toFieldRef.numParts()-1)); if (destExists) { - removeStatus = _preparedState->toElemFound.remove(); - if (!removeStatus.isOK()) { - return removeStatus; - } + // Set destination element to the value of the source element. + return _preparedState->toElemFound.setValueElement(_preparedState->fromElemFound); } // Creates the final element that's going to be the in 'doc'. diff --git a/src/mongo/db/ops/modifier_rename_test.cpp b/src/mongo/db/ops/modifier_rename_test.cpp index 606337bdd7c..6d889bb9c58 100644 --- a/src/mongo/db/ops/modifier_rename_test.cpp +++ b/src/mongo/db/ops/modifier_rename_test.cpp @@ -153,6 +153,13 @@ namespace { ModifierInterface::Options::normal())); } + TEST(MoveOnSamePath, MoveToSelf) { + ModifierRename mod; + ASSERT_NOT_OK( + mod.init(fromjson("{'b.a':'b.a'}").firstElement(), + ModifierInterface::Options::normal())); + } + TEST(MissingTo, SimpleNumberAtRoot) { Document doc(fromjson("{a: 2}")); Mod setMod(fromjson("{$rename: {'a':'b'}}")); @@ -219,6 +226,48 @@ namespace { ASSERT_EQUALS(logDoc, logObj); } + TEST(SimpleReplace, RenameToExistingFieldDoesNotReorderFields) { + Document doc(fromjson("{a: 1, b: 2, c: 3}")); + Mod setMod(fromjson("{$rename: {a: 'b'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "a"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "b"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{b: 1, c: 3}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set: {b: 1}, $unset: {a: true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + + TEST(SimpleReplace, RenameToExistingNestedFieldDoesNotReorderFields) { + Document doc(fromjson("{a: {b: {c: 1, d: 2}}, b: 3, c: {d: 4}}")); + Mod setMod(fromjson("{$rename: {'c.d': 'a.b.c'}}")); + + ModifierInterface::ExecInfo execInfo; + ASSERT_OK(setMod.prepare(doc.root(), "", &execInfo)); + ASSERT_EQUALS(execInfo.fieldRef[0]->dottedField(), "c.d"); + ASSERT_EQUALS(execInfo.fieldRef[1]->dottedField(), "a.b.c"); + ASSERT_FALSE(execInfo.noOp); + + ASSERT_OK(setMod.apply()); + ASSERT_FALSE(doc.isInPlaceModeEnabled()); + ASSERT_EQUALS(doc, fromjson("{a: {b: {c: 4, d: 2}}, b: 3, c: {}}")); + + Document logDoc; + LogBuilder logBuilder(logDoc.root()); + BSONObj logObj = fromjson("{$set: {'a.b.c': 4}, $unset: {'c.d': true}}"); + ASSERT_OK(setMod.log(&logBuilder)); + ASSERT_EQUALS(logDoc, logObj); + } + TEST(DottedTo, MissingCompleteTo) { Document doc(fromjson("{a: 2, b: 1, c: {}}")); Mod setMod(fromjson("{$rename: {'a':'c.r.d'}}")); diff --git a/src/mongo/db/query/idhack_runner.cpp b/src/mongo/db/query/idhack_runner.cpp index d587323d5f0..af365e0b37c 100644 --- a/src/mongo/db/query/idhack_runner.cpp +++ b/src/mongo/db/query/idhack_runner.cpp @@ -146,6 +146,14 @@ namespace { _done = true; return Runner::RUNNER_DEAD; } + + // Even if we're not dead due to something like a db or collection drop, the + // document we're looking for may have already been deleted out from under + // us. + if (_locFetching.isNull()) { + _done = true; + return Runner::RUNNER_EOF; + } } } @@ -233,7 +241,6 @@ namespace { if (_done || _killed) { return; } if (_locFetching == dl && (type == INVALIDATION_DELETION)) { _locFetching = DiskLoc(); - _killed = true; } } diff --git a/src/mongo/db/query/multi_plan_runner.cpp b/src/mongo/db/query/multi_plan_runner.cpp index 002b3ebac62..e14d03cc984 100644 --- a/src/mongo/db/query/multi_plan_runner.cpp +++ b/src/mongo/db/query/multi_plan_runner.cpp @@ -149,35 +149,19 @@ namespace mongo { if (NULL != _bestPlan) { _bestPlan->invalidate(dl, type); for (list<WorkingSetID>::iterator it = _alreadyProduced.begin(); - it != _alreadyProduced.end();) { + it != _alreadyProduced.end(); ++it) { WorkingSetMember* member = _bestPlan->getWorkingSet()->get(*it); if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; WorkingSetCommon::fetchAndInvalidateLoc(member); - _bestPlan->getWorkingSet()->flagForReview(*it); - _alreadyProduced.erase(it); - it = next; - } - else { - it++; } } if (NULL != _backupPlan) { _backupPlan->invalidate(dl, type); for (list<WorkingSetID>::iterator it = _backupAlreadyProduced.begin(); - it != _backupAlreadyProduced.end();) { + it != _backupAlreadyProduced.end(); ++it) { WorkingSetMember* member = _backupPlan->getWorkingSet()->get(*it); if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; WorkingSetCommon::fetchAndInvalidateLoc(member); - _backupPlan->getWorkingSet()->flagForReview(*it); - _backupAlreadyProduced.erase(it); - it = next; - } - else { - it++; } } } @@ -186,18 +170,10 @@ namespace mongo { for (size_t i = 0; i < _candidates.size(); ++i) { _candidates[i].root->invalidate(dl, type); for (list<WorkingSetID>::iterator it = _candidates[i].results.begin(); - it != _candidates[i].results.end();) { + it != _candidates[i].results.end(); ++it) { WorkingSetMember* member = _candidates[i].ws->get(*it); if (member->hasLoc() && member->loc == dl) { - list<WorkingSetID>::iterator next = it; - next++; WorkingSetCommon::fetchAndInvalidateLoc(member); - _candidates[i].ws->flagForReview(*it); - _candidates[i].results.erase(it); - it = next; - } - else { - it++; } } } diff --git a/src/mongo/db/query/new_find.cpp b/src/mongo/db/query/new_find.cpp index b2214008a7a..c3b4a129842 100644 --- a/src/mongo/db/query/new_find.cpp +++ b/src/mongo/db/query/new_find.cpp @@ -269,34 +269,20 @@ namespace mongo { // to getNext(...) might just return EOF). bool saveClientCursor = false; - if (Runner::RUNNER_DEAD == state || Runner::RUNNER_ERROR == state) { - // Propagate this error to caller. - if (Runner::RUNNER_ERROR == state) { - // Stats are helpful when errors occur. - TypeExplain* bareExplain; - Status res = runner->getInfo(&bareExplain, NULL); - if (res.isOK()) { - boost::scoped_ptr<TypeExplain> errorExplain(bareExplain); - error() << "Runner error, stats:\n" - << errorExplain->stats.jsonString(Strict, true); - } - - uasserted(17406, "getMore runner error: " + - WorkingSetCommon::toStatusString(obj)); - } - - // If we're dead there's no way to get more results. - saveClientCursor = false; - - // In the old system tailable capped cursors would be killed off at the - // cursorid level. If a tailable capped cursor is nuked the cursorid - // would vanish. - // - // In the new system they die and are cleaned up later (or time out). - // So this is where we get to remove the cursorid. - if (0 == numResults) { - resultFlags = ResultFlag_CursorNotFound; + if (Runner::RUNNER_ERROR == state) { + // Stats are helpful when errors occur. + TypeExplain* bareExplain; + Status res = runner->getInfo(&bareExplain, NULL); + if (res.isOK()) { + boost::scoped_ptr<TypeExplain> errorExplain(bareExplain); + error() << "getMore runner error, stats:\n" + << errorExplain->stats.jsonString(Strict, true); } + uasserted(17406, "getMore runner error: " + + WorkingSetCommon::toStatusString(obj)); + } + else if (Runner::RUNNER_DEAD == state) { + uasserted(28617, "Runner killed during getMore"); } else if (Runner::RUNNER_EOF == state) { // EOF is also end of the line unless it's tailable. @@ -670,17 +656,16 @@ namespace mongo { Status res = runner->getInfo(&bareExplain, NULL); if (res.isOK()) { boost::scoped_ptr<TypeExplain> errorExplain(bareExplain); - error() << "Runner error, stats:\n" + error() << "Runner error during find, stats:\n" << errorExplain->stats.jsonString(Strict, true); } - uasserted(17144, "Runner error: " + WorkingSetCommon::toStatusString(obj)); + uasserted(17144, "Runner error during find: " + WorkingSetCommon::toStatusString(obj)); } - - // Why save a dead runner? - if (Runner::RUNNER_DEAD == state) { - saveClientCursor = false; + else if (Runner::RUNNER_DEAD == state) { + uasserted(28616, "Runner killed during find"); } - else if (pq.hasOption(QueryOption_CursorTailable)) { + + if (pq.hasOption(QueryOption_CursorTailable)) { // If we're tailing a capped collection, we don't bother saving the cursor if the // collection is empty. Otherwise, the semantics of the tailable cursor is that the // client will keep trying to read from it. So we'll keep it around. diff --git a/src/mongo/db/query/plan_enumerator.cpp b/src/mongo/db/query/plan_enumerator.cpp index 35c629771ea..c5ec77dfd44 100644 --- a/src/mongo/db/query/plan_enumerator.cpp +++ b/src/mongo/db/query/plan_enumerator.cpp @@ -590,41 +590,55 @@ namespace mongo { // For each FIRST, we assign nodes to it. for (IndexToPredMap::const_iterator it = idxToFirst.begin(); it != idxToFirst.end(); ++it) { - // The assignment we're filling out. - OneIndexAssignment indexAssign; - - // This is the index we assign to. - indexAssign.index = it->first; - const IndexEntry& thisIndex = (*_indices)[it->first]; // If the index is multikey, we only assign one pred to it. We also skip // compounding. TODO: is this also true for 2d and 2dsphere indices? can they be // multikey but still compoundable? if (thisIndex.multikey) { - // TODO: could pick better pred than first but not too worried since we should - // really be isecting indices here. Just take the first pred. We don't assign - // any other preds to this index. The planner will intersect the preds and this - // enumeration strategy is just one index at a time. - indexAssign.preds.push_back(it->second[0]); - indexAssign.positions.push_back(0); - - // If there are any preds that could possibly be compounded with this - // index... - IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); - if (compIt != idxToNotFirst.end()) { - const vector<MatchExpression*>& couldCompound = compIt->second; - vector<MatchExpression*> tryCompound; + // Since the index is multikey, we can only use one of the predicates over the + // leading field of the index. However, we do not know which of these predicates is + // most selective. Therefore, we will generate a plan for each so that they can be + // ranked against each other. + vector<MatchExpression*>::const_iterator multikeyIt; + for (multikeyIt = it->second.begin(); multikeyIt != it->second.end(); + ++multikeyIt) { + MatchExpression* pred = *multikeyIt; + + OneIndexAssignment indexAssign; + indexAssign.index = it->first; + + indexAssign.preds.push_back(pred); + indexAssign.positions.push_back(0); - // ...select the predicates that are safe to compound and try to - // compound them. - getMultikeyCompoundablePreds(indexAssign.preds, couldCompound, &tryCompound); - if (tryCompound.size()) { - compound(tryCompound, thisIndex, &indexAssign); + // If there are any preds that could possibly be compounded with this + // index... + IndexToPredMap::const_iterator compIt = idxToNotFirst.find(indexAssign.index); + if (compIt != idxToNotFirst.end()) { + const vector<MatchExpression*>& couldCompound = compIt->second; + vector<MatchExpression*> tryCompound; + + // ...select the predicates that are safe to compound and try to + // compound them. + getMultikeyCompoundablePreds(indexAssign.preds, couldCompound, &tryCompound); + if (tryCompound.size()) { + compound(tryCompound, thisIndex, &indexAssign); + } } + + // Output the assignment. + AndEnumerableState state; + state.assignments.push_back(indexAssign); + andAssignment->choices.push_back(state); } } else { + // The assignment we're filling out. + OneIndexAssignment indexAssign; + + // This is the index we assign to. + indexAssign.index = it->first; + // The index isn't multikey. Assign all preds to it. The planner will // intersect the bounds. indexAssign.preds = it->second; @@ -639,11 +653,12 @@ namespace mongo { if (compIt != idxToNotFirst.end()) { compound(compIt->second, thisIndex, &indexAssign); } - } - AndEnumerableState state; - state.assignments.push_back(indexAssign); - andAssignment->choices.push_back(state); + // Output the assignment. + AndEnumerableState state; + state.assignments.push_back(indexAssign); + andAssignment->choices.push_back(state); + } } } diff --git a/src/mongo/db/query/query_planner.cpp b/src/mongo/db/query/query_planner.cpp index 2ebd0b51834..11b6c86c925 100644 --- a/src/mongo/db/query/query_planner.cpp +++ b/src/mongo/db/query/query_planner.cpp @@ -697,7 +697,7 @@ namespace mongo { if (QueryPlannerCommon::hasNode(query.root(), MatchExpression::GEO_NEAR, &gnNode)) { // No index for GEO_NEAR? No query. RelevantTag* tag = static_cast<RelevantTag*>(gnNode->getTag()); - if (0 == tag->first.size() && 0 == tag->notFirst.size()) { + if (!tag || (0 == tag->first.size() && 0 == tag->notFirst.size())) { QLOG() << "Unable to find index for $geoNear query." << endl; // Don't leave tags on query tree. query.root()->resetTag(); diff --git a/src/mongo/db/query/query_planner_test.cpp b/src/mongo/db/query/query_planner_test.cpp index b4bdac93a50..2471b6d4369 100644 --- a/src/mongo/db/query/query_planner_test.cpp +++ b/src/mongo/db/query/query_planner_test.cpp @@ -1504,10 +1504,12 @@ namespace { runQuery(fromjson("{'a.b': {$elemMatch: {c: {$all: " "[{$elemMatch: {d: {$gt: 1, $lt: 3}}}]}}}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c.d': 1}, " "bounds: {'a.b.c.d': [[-Infinity,3,true,false]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c.d': 1}, " + "bounds: {'a.b.c.d': [[1,Infinity,false,true]]}}}}}"); } // SERVER-13677 @@ -1516,10 +1518,14 @@ namespace { addIndex(BSON("a.b.c" << 1), true); runQuery(fromjson("{z: 1, 'a.b': {$elemMatch: {c: {$all: [4, 5, 6]}}}}")); - assertNumSolutions(2U); + assertNumSolutions(4U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, " "bounds: {'a.b.c': [[4,4,true,true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, " + "bounds: {'a.b.c': [[5,5,true,true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, " + "bounds: {'a.b.c': [[5,5,true,true]]}}}}}"); } TEST_F(QueryPlannerTest, ElemMatchValueMatch) { @@ -1601,12 +1607,16 @@ namespace { addIndex(BSON("a.b" << 1 << "a.c" << 1), true); runQuery(fromjson("{a: {$elemMatch: {b: {$gte: 2, $lt: 4}, c: 25}}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:{$gte:2,$lt: 4},c:25}}}, node: " "{ixscan: {filter: null, pattern: {'a.b': 1, 'a.c': 1}, " "bounds: {'a.b': [[-Infinity,4,true,false]], " "'a.c': [[25,25,true,true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:{$gte:2,$lt: 4},c:25}}}, node: " + "{ixscan: {filter: null, pattern: {'a.b': 1, 'a.c': 1}, " + "bounds: {'a.b': [[2,Infinity,true,true]], " + "'a.c': [[25,25,true,true]]}}}}}"); } // SERVER-13664 @@ -2045,11 +2055,9 @@ namespace { " {a: {$geoIntersects: {$geometry: " "{type: 'Point', coordinates: [4.0, 1.0]}}}}]}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); - // Bounds of the two 2dsphere geo predicates are combined into - // a single index scan. - assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {a: '2dsphere'}}}}}", 2U); } TEST_F(QueryPlannerTest, And2DSphereWithNearSameField) { @@ -2180,6 +2188,11 @@ namespace { assertSolutionExists("{geoNear2d: {a: '2d', b: 1}}"); } + TEST_F(QueryPlannerTest, NearEmptyPath) { + addIndex(BSON("" << "2dsphere")); + runInvalidQuery(fromjson("{'': {$near: {$geometry: {type:'Point',coordinates:[0,0]}}}}")); + } + // // $in // @@ -2803,10 +2816,13 @@ namespace { addIndex(fromjson("{'a.b': 1}"), true); runQueryHint(fromjson("{'a.b': 1, a: {$elemMatch: {b: 2}}}"), fromjson("{'a.b': 1}")); - assertNumSolutions(1U); + assertNumSolutions(2U); assertSolutionExists("{fetch: {filter: {$and: [{a:{$elemMatch:{b:2}}}, {'a.b': 1}]}, " "node: {ixscan: {filter: null, pattern: {'a.b': 1}, bounds: " "{'a.b': [[2, 2, true, true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {a:{$elemMatch:{b:2}}}, " + "node: {ixscan: {filter: null, pattern: {'a.b': 1}, bounds: " + "{'a.b': [[1, 1, true, true]]}}}}}"); } TEST_F(QueryPlannerTest, HintInvalid) { @@ -3001,10 +3017,17 @@ namespace { addIndex(BSON("a" << 1), true); runQuery(fromjson("{$and: [{a: /0/}, {a: /1/}, {a: /2/}]}")); - ASSERT_EQUALS(getNumSolutions(), 2U); + ASSERT_EQUALS(getNumSolutions(), 4U); assertSolutionExists("{cscan: {filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}, dir: 1}}"); assertSolutionExists("{fetch: {filter: {$and:[{a:/0/},{a:/1/},{a:/2/}]}, node: {ixscan: " - "{pattern: {a: 1}, filter: null}}}}"); + "{pattern: {a: 1}, filter: null, " + "bounds: {a: [['', {}, true, false], [/0/, /0/, true, true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {$and:[{a:/1/},{a:/0/},{a:/2/}]}, node: {ixscan: " + "{pattern: {a: 1}, filter: null, " + "bounds: {a: [['', {}, true, false], [/1/, /1/, true, true]]}}}}}"); + assertSolutionExists("{fetch: {filter: {$and:[{a:/2/},{a:/0/},{a:/1/}]}, node: {ixscan: " + "{pattern: {a: 1}, filter: null, " + "bounds: {a: [['', {}, true, false], [/2/, /2/, true, true]]}}}}}"); } // @@ -3353,15 +3376,12 @@ namespace { addIndex(BSON("a" << 1), true); runQuery(fromjson("{a: {$gt: 0, $lt: 5}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {filter: {$and: [{a: {$lt: 5}}, {a: {$gt: 0}}]}, dir: 1}}"); - - vector<string> alternates; - alternates.push_back("{fetch: {filter: {a: {$lt: 5}}, node: {ixscan: {filter: null, " + assertSolutionExists("{fetch: {filter: {a: {$lt: 5}}, node: {ixscan: {filter: null, " "pattern: {a: 1}, bounds: {a: [[0, Infinity, false, true]]}}}}}"); - alternates.push_back("{fetch: {filter: {a: {$gt: 0}}, node: {ixscan: {filter: null, " + assertSolutionExists("{fetch: {filter: {a: {$gt: 0}}, node: {ixscan: {filter: null, " "pattern: {a: 1}, bounds: {a: [[-Infinity, 5, true, false]]}}}}}"); - assertHasOneSolutionOf(alternates); } /** @@ -3593,10 +3613,12 @@ namespace { addIndex(BSON("a.b.c" << 1), true); runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1, $lte: 1}}}}}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " "{'a.b.c': [[-Infinity, 1, true, true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " + "{'a.b.c': [[1, Infinity, true, true]]}}}}}"); } // The index bounds cannot be intersected because the index is multikey. @@ -3608,10 +3630,12 @@ namespace { addIndex(BSON("a.b.c" << 1), true); runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {c: {$gte: 1, $in:[2]}}}}}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " "{'a.b.c': [[1, Infinity, true, true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b.c': 1}, bounds: " + "{'a.b.c': [[2, 2, true, true]]}}}}}"); } // The bounds can be compounded because the index is not multikey. @@ -3659,7 +3683,7 @@ namespace { // We can intersect the bounds for all three predicates because // the index is not multikey. - TEST_F(QueryPlannerTest, ElemMatchInterectBoundsNotMultikey) { + TEST_F(QueryPlannerTest, ElemMatchIntersectBoundsNotMultikey) { addIndex(BSON("a.b" << 1)); runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {$gte: 1, $lte: 4}}}}," "'a.b': {$in: [2,5]}}")); @@ -3675,16 +3699,18 @@ namespace { // from the $in predicate are not intersected with the bounds from the // remaining to predicates because the $in is not joined to the other // predicates with an $elemMatch. - TEST_F(QueryPlannerTest, ElemMatchInterectBoundsMultikey) { + TEST_F(QueryPlannerTest, ElemMatchIntersectBoundsMultikey) { // true means multikey addIndex(BSON("a.b" << 1), true); runQuery(fromjson("{a: {$elemMatch: {b: {$elemMatch: {$gte: 1, $lte: 4}}}}," "'a.b': {$in: [2,5]}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " "{'a.b': [[1, 4, true, true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b': 1}, bounds: " + "{'a.b': [[2,2,true,true], [5,5,true,true]]}}}}}"); } // Bounds can be intersected because the predicates are joined by an @@ -3766,11 +3792,14 @@ namespace { addIndex(BSON("a.b" << 1 << "a.c" << 1), true); runQuery(fromjson("{'a.b': 1, a: {$elemMatch: {b: {$gt: 0}, c: 1}}}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " "{'a.b': [[0,Infinity,false,true]], " " 'a.c': [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); } // Bounds for the predicates joined by the $elemMatch over the shared prefix @@ -3781,11 +3810,14 @@ namespace { addIndex(BSON("a.b" << 1 << "a.c" << 1), true); runQuery(fromjson("{a: {$elemMatch: {b: 1, c: 1}}, 'a.b': 1}")); - assertNumSolutions(2U); + assertNumSolutions(3U); assertSolutionExists("{cscan: {dir: 1}}"); assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " "{'a.b': [[1,1,true,true]], " " 'a.c': [[1,1,true,true]]}}}}}"); + assertSolutionExists("{fetch: {node: {ixscan: {pattern: {'a.b':1,'a.c':1}, bounds: " + "{'a.b': [[1,1,true,true]], " + " 'a.c': [['MinKey','MaxKey',true,true]]}}}}}"); } // Bounds for the predicates joined by the $elemMatch over the shared prefix diff --git a/src/mongo/dbtests/querytests.cpp b/src/mongo/dbtests/querytests.cpp index a804908887e..20ac1d3cfca 100644 --- a/src/mongo/dbtests/querytests.cpp +++ b/src/mongo/dbtests/querytests.cpp @@ -456,19 +456,11 @@ namespace QueryTests { ASSERT( !c->more() ); insert( ns, BSON( "a" << 2 ) ); insert( ns, BSON( "a" << 3 ) ); - ASSERT( !c->more() ); - // Inserting a document into a capped collection can force another document out. - // In this case, the capped collection has 2 documents, so inserting two more clobbers - // whatever DiskLoc that the underlying cursor had as its state. - // - // In the Cursor world, the ClientCursor was responsible for manipulating cursors. It - // would detect that the cursor's "refloc" (translation: diskloc required to maintain - // iteration state) was being clobbered and it would kill the cursor. - // - // In the Runner world there is no notion of a "refloc" and as such the invalidation - // broadcast code doesn't know enough to know that the underlying collection iteration - // can't proceed. - // ASSERT_EQUALS( 0, c->getCursorId() ); + + // We have overwritten the previous cursor position and should encounter a dead cursor. + if (c->more()) { + ASSERT_THROWS(c->nextSafe(), AssertionException); + } } }; diff --git a/src/mongo/platform/random.cpp b/src/mongo/platform/random.cpp index d1a620d6a74..10c197f6d6a 100644 --- a/src/mongo/platform/random.cpp +++ b/src/mongo/platform/random.cpp @@ -35,8 +35,8 @@ namespace mongo { // ---- PseudoRandom ----- - int32_t PseudoRandom::nextInt32() { - int32_t t = _x ^ (_x << 11); + uint32_t PseudoRandom::nextUInt32() { + uint32_t t = _x ^ (_x << 11); _x = _y; _y = _z; _z = _w; @@ -44,13 +44,13 @@ namespace mongo { } namespace { - const int32_t default_y = 362436069; - const int32_t default_z = 521288629; - const int32_t default_w = 88675123; + const uint32_t default_y = 362436069; + const uint32_t default_z = 521288629; + const uint32_t default_w = 88675123; } PseudoRandom::PseudoRandom( int32_t seed ) { - _x = seed; + _x = static_cast<uint32_t>(seed); _y = default_y; _z = default_z; _w = default_w; @@ -58,7 +58,7 @@ namespace mongo { PseudoRandom::PseudoRandom( uint32_t seed ) { - _x = static_cast<int32_t>(seed); + _x = seed; _y = default_y; _z = default_z; _w = default_w; @@ -66,8 +66,8 @@ namespace mongo { PseudoRandom::PseudoRandom( int64_t seed ) { - int32_t high = seed >> 32; - int32_t low = seed & 0xFFFFFFFF; + uint32_t high = seed >> 32; + uint32_t low = seed & 0xFFFFFFFF; _x = high ^ low; _y = default_y; @@ -75,9 +75,13 @@ namespace mongo { _w = default_w; } + int32_t PseudoRandom::nextInt32() { + return nextUInt32(); + } + int64_t PseudoRandom::nextInt64() { - int64_t a = nextInt32(); - int64_t b = nextInt32(); + uint64_t a = nextUInt32(); + uint64_t b = nextUInt32(); return ( a << 32 ) | b; } diff --git a/src/mongo/platform/random.h b/src/mongo/platform/random.h index 823ea6a08db..da52d05248f 100644 --- a/src/mongo/platform/random.h +++ b/src/mongo/platform/random.h @@ -39,17 +39,21 @@ namespace mongo { /** * @return a number between 0 and max */ - int32_t nextInt32( int32_t max ) { return nextInt32() % max; } + int32_t nextInt32( int32_t max ) { + return static_cast<uint32_t>(nextInt32()) % static_cast<uint32_t>(max); + } /** * @return a number between 0 and max */ - int64_t nextInt64( int64_t max ) { return nextInt64() % max; } + int64_t nextInt64( int64_t max ) { + return static_cast<uint64_t>(nextInt64()) % static_cast<uint64_t>(max); + } /** * @return a number between 0 and max * - * This makes PsuedoRandom instances passable as the third argument to std::random_shuffle + * This makes PseudoRandom instances passable as the third argument to std::random_shuffle */ intptr_t operator()(intptr_t max) { if (sizeof(intptr_t) == 4) @@ -58,10 +62,12 @@ namespace mongo { } private: - int32_t _x; - int32_t _y; - int32_t _z; - int32_t _w; + uint32_t nextUInt32(); + + uint32_t _x; + uint32_t _y; + uint32_t _z; + uint32_t _w; }; /** diff --git a/src/mongo/platform/random_test.cpp b/src/mongo/platform/random_test.cpp index 1f93e78f31f..4e081ac92de 100644 --- a/src/mongo/platform/random_test.cpp +++ b/src/mongo/platform/random_test.cpp @@ -17,6 +17,7 @@ */ #include <set> +#include <vector> #include "mongo/platform/random.h" @@ -86,6 +87,72 @@ namespace mongo { ASSERT_EQUALS( 100U, s.size() ); } + TEST(RandomTest, NextInt32SanityCheck) { + // Generate 1000 int32s and assert that each bit is set between 40% and 60% of the time. + // This is a bare minimum sanity check, not an attempt to ensure quality random numbers. + + PseudoRandom a(11); + std::vector<int32_t> nums; + for (int i = 0; i < 1000; i++) { + nums.push_back(a.nextInt32()); + } + + for (int bit = 0; bit < 32; bit++) { + int onesCount = 0; + for (size_t i=0; i < nums.size(); i++) { + bool isSet = (nums[i] >> bit) & 1; + if (isSet) + onesCount++; + } + + if (onesCount < 400 || onesCount > 600) + FAIL(str::stream() << "bit " << bit << " was set " << (onesCount / 10.) + << "% of the time."); + } + } + + TEST(RandomTest, NextInt64SanityCheck) { + // Generate 1000 int64s and assert that each bit is set between 40% and 60% of the time. + // This is a bare minimum sanity check, not an attempt to ensure quality random numbers. + + PseudoRandom a(11); + std::vector<int64_t> nums; + for (int i = 0; i < 1000; i++) { + nums.push_back(a.nextInt64()); + } + + for (int bit = 0; bit < 64; bit++) { + int onesCount = 0; + for (size_t i=0; i < nums.size(); i++) { + bool isSet = (nums[i] >> bit) & 1; + if (isSet) + onesCount++; + } + + if (onesCount < 400 || onesCount > 600) + FAIL(str::stream() << "bit " << bit << " was set " << (onesCount / 10.) + << "% of the time."); + } + } + + TEST(RandomTest, NextInt32InRange) { + PseudoRandom a(11); + for (int i = 0; i < 1000; i++) { + int32_t res = a.nextInt32(10); + ASSERT_GREATER_THAN_OR_EQUALS(res, 0); + ASSERT_LESS_THAN(res, 10); + } + } + + TEST(RandomTest, NextInt64InRange) { + PseudoRandom a(11); + for (int i = 0; i < 1000; i++) { + int64_t res = a.nextInt64(10); + ASSERT_GREATER_THAN_OR_EQUALS(res, 0); + ASSERT_LESS_THAN(res, 10); + } + } + TEST( RandomTest, Secure1 ) { SecureRandom* a = SecureRandom::create(); diff --git a/src/mongo/s/d_migrate.cpp b/src/mongo/s/d_migrate.cpp index b492c0e8b18..75fa610e5e0 100644 --- a/src/mongo/s/d_migrate.cpp +++ b/src/mongo/s/d_migrate.cpp @@ -54,6 +54,7 @@ #include "mongo/db/commands.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/dur.h" +#include "mongo/db/exec/working_set_common.h" #include "mongo/db/field_parser.h" #include "mongo/db/hasher.h" #include "mongo/db/jsobj.h" @@ -434,8 +435,10 @@ namespace mongo { // we want the number of records to better report, in that case bool isLargeChunk = false; unsigned long long recCount = 0;; + BSONObj obj; DiskLoc dl; - while (Runner::RUNNER_ADVANCED == runner->getNext(NULL, &dl)) { + Runner::RunnerState state; + while (Runner::RUNNER_ADVANCED == (state = runner->getNext(&obj, &dl))) { if ( ! isLargeChunk ) { scoped_spinlock lk( _trackerLocks ); _cloneLocs.insert( dl ); @@ -445,6 +448,18 @@ namespace mongo { isLargeChunk = true; } } + + if (Runner::RUNNER_DEAD == state) { + errmsg = "Runner error while scanning for documents belonging to chunk."; + return false; + } + + if (Runner::RUNNER_ERROR == state) { + errmsg = "Runner error while scanning for documents belonging to chunk: " + + WorkingSetCommon::toStatusString(obj); + return false; + } + runner.reset(); if ( isLargeChunk ) { diff --git a/src/mongo/s/request.cpp b/src/mongo/s/request.cpp index 25219eb0893..ccaca188db1 100644 --- a/src/mongo/s/request.cpp +++ b/src/mongo/s/request.cpp @@ -90,7 +90,7 @@ namespace mongo { int msgId = (int)(_m.header()->id); Timer t; - LOG(3) << "Request::process begin ns: " << getns() + LOG(3) << "Request::process begin ns: " << getnsIfPresent() << " msg id: " << msgId << " op: " << op << " attempt: " << attempt @@ -130,7 +130,7 @@ namespace mongo { // globalOpCounters are handled by write commands. } - LOG(3) << "Request::process end ns: " << getns() + LOG(3) << "Request::process end ns: " << getnsIfPresent() << " msg id: " << msgId << " op: " << op << " attempt: " << attempt diff --git a/src/mongo/s/request.h b/src/mongo/s/request.h index a8230f36b4c..97107bf29d9 100644 --- a/src/mongo/s/request.h +++ b/src/mongo/s/request.h @@ -52,6 +52,9 @@ namespace mongo { const char * getns() const { return _d.getns(); } + const char* getnsIfPresent() const { + return _d.messageShouldHaveNs() ? _d.getns() : ""; + } int op() const { return _m.operation(); } diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 9c4f2acea2c..787fbd64c80 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -145,7 +145,7 @@ namespace mongo { LOG( ex.isUserAssertion() ? 1 : 0 ) << "Assertion failed" << " while processing " << opToString( m.operation() ) << " op" - << " for " << r.getns() << causedBy( ex ) << endl; + << " for " << r.getnsIfPresent() << causedBy( ex ) << endl; if ( r.expectResponse() ) { m.header()->id = r.id(); @@ -159,7 +159,7 @@ namespace mongo { log() << "Exception thrown" << " while processing " << opToString( m.operation() ) << " op" - << " for " << r.getns() << causedBy( ex ) << endl; + << " for " << r.getnsIfPresent() << causedBy( ex ) << endl; if ( r.expectResponse() ) { m.header()->id = r.id(); diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js index 944306fbb19..5b27c1216ba 100644 --- a/src/mongo/shell/shardingtest.js +++ b/src/mongo/shell/shardingtest.js @@ -665,8 +665,13 @@ printShardingStatus = function( configDB , verbose ){ output( "\t\t\tshard key: " + tojson(coll.key) ); output( "\t\t\tchunks:" ); - res = configDB.chunks.group( { cond : { ns : coll._id } , key : { shard : 1 }, - reduce : function( doc , out ){ out.nChunks++; } , initial : { nChunks : 0 } } ); + res = configDB.chunks.aggregate( { $match : { ns : coll._id } } , + { $group : { _id : "$shard" , + cnt : { $sum : 1 } } } , + { $project : { _id : 0 , + shard : "$_id" , + nChunks : "$cnt" } } , + { $sort : { shard : 1 } } ).toArray(); var totalChunks = 0; res.forEach( function(z){ totalChunks += z.nChunks; @@ -1009,3 +1014,12 @@ ShardingTest.prototype.restartMongos = function(n) { } }; +/** + * Helper method for setting primary shard of a database and making sure that it was successful. + * Note: first mongos needs to be up. + */ +ShardingTest.prototype.ensurePrimaryShard = function(dbName, shardName) { + var db = this.s0.getDB('admin'); + var res = db.adminCommand({ movePrimary: dbName, to: shardName }); + assert(res.ok || res.errmsg == "it is already the primary", tojson(res)); +}; diff --git a/src/mongo/util/mmap.cpp b/src/mongo/util/mmap.cpp index b4b9435a7f5..f021cd55022 100644 --- a/src/mongo/util/mmap.cpp +++ b/src/mongo/util/mmap.cpp @@ -138,17 +138,8 @@ namespace { return total; } - void nullFunc() { } - - // callback notifications - void (*MongoFile::notifyPreFlush)() = nullFunc; - void (*MongoFile::notifyPostFlush)() = nullFunc; - /*static*/ int MongoFile::flushAll( bool sync ) { - if ( sync ) notifyPreFlush(); - int x = _flushAll(sync); - if ( sync ) notifyPostFlush(); - return x; + return _flushAll(sync); } /*static*/ int MongoFile::_flushAll( bool sync ) { diff --git a/src/mongo/util/mmap.h b/src/mongo/util/mmap.h index 9d24da27c9a..33df258ad28 100644 --- a/src/mongo/util/mmap.h +++ b/src/mongo/util/mmap.h @@ -102,10 +102,6 @@ namespace mongo { */ static std::set<MongoFile*>& getAllFiles(); - // callbacks if you need them - static void (*notifyPreFlush)(); - static void (*notifyPostFlush)(); - static int flushAll( bool sync ); // returns n flushed static long long totalMappedLength(); static void closeAllFiles( std::stringstream &message ); diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp index 0759a6da5fd..8533f4260ca 100644 --- a/src/mongo/util/net/ssl_options.cpp +++ b/src/mongo/util/net/ssl_options.cpp @@ -58,8 +58,7 @@ namespace mongo { .hidden(); options->addOptionChaining("net.ssl.disabledProtocols", "sslDisabledProtocols", moe::String, - "Comma separated list of disabled protocols") - .hidden(); + "Comma separated list of TLS protocols to disable [TLS1_0,TLS1_1,TLS1_2]"); options->addOptionChaining("net.ssl.weakCertificateValidation", "sslWeakCertificateValidation", moe::Switch, "allow client to connect without " @@ -97,11 +96,6 @@ namespace mongo { .requires("ssl") .requires("ssl.CAFile"); - options->addOptionChaining("net.ssl.disabledProtocols", "sslDisabledProtocols", moe::String, - "Comma separated list of disabled protocols") - .requires("ssl") - .hidden(); - options->addOptionChaining("net.ssl.allowInvalidHostnames", "sslAllowInvalidHostnames", moe::Switch, "allow connections to servers with non-matching hostnames") .requires("ssl"); @@ -189,13 +183,22 @@ namespace mongo { } if (params.count("net.ssl.disabledProtocols")) { + // The disabledProtocols field is composed of a comma separated list of protocols to + // disable. First, tokenize the field. std::vector<std::string> tokens = StringSplitter::split( params["net.ssl.disabledProtocols"].as<string>(), ","); + // All accepted tokens, and their corresponding enum representation. The noTLS* tokens + // exist for backwards compatibility. std::map<std::string, SSLGlobalParams::Protocols> validConfigs; + validConfigs["TLS1_0"] = SSLGlobalParams::TLS1_0; validConfigs["noTLS1_0"] = SSLGlobalParams::TLS1_0; + validConfigs["TLS1_1"] = SSLGlobalParams::TLS1_1; validConfigs["noTLS1_1"] = SSLGlobalParams::TLS1_1; + validConfigs["TLS1_2"] = SSLGlobalParams::TLS1_2; validConfigs["noTLS1_2"] = SSLGlobalParams::TLS1_2; + + // Map the tokens to their enum values, and push them onto the list of disabled protocols. for (std::vector<std::string>::iterator it = tokens.begin(); it != tokens.end(); ++it) { std::map<std::string, SSLGlobalParams::Protocols>::iterator mappedToken = validConfigs.find(*it); diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp index 54a166ddadc..3f7b6480262 100644 --- a/src/mongo/util/version.cpp +++ b/src/mongo/util/version.cpp @@ -31,7 +31,7 @@ namespace mongo { * 1.2.3-rc4-pre- * If you really need to do something else you'll need to fix _versionArray() */ - const char versionString[] = "2.6.11"; + const char versionString[] = "2.6.12"; // See unit test for example outputs BSONArray toVersionArray(const char* version){ |