diff options
| author | Antonin Kral <a.kral@bobek.cz> | 2012-11-28 09:18:54 +0100 |
|---|---|---|
| committer | Antonin Kral <a.kral@bobek.cz> | 2012-11-28 09:18:54 +0100 |
| commit | 8648b72bdffc6c730fda77dc2e6c83ec6cfa1166 (patch) | |
| tree | 03c1a6e21eb5de70dea89552707ba9fac75e2933 | |
| parent | 83957b73f9177f6e38bd5375bd93ca1f6a47188c (diff) | |
Imported Upstream version 2.2.2
134 files changed, 4448 insertions, 2020 deletions
diff --git a/SConstruct b/SConstruct index 1792804ab17..d564045232c 100644 --- a/SConstruct +++ b/SConstruct @@ -861,7 +861,7 @@ def doConfigure(myenv): # discover modules (subdirectories of db/modules/), and # load the (python) module for each module's build.py - modules = moduleconfig.discover_modules('.') + modules = moduleconfig.discover_modules('src/mongo/') # ask each module to configure itself, and return a # dictionary of name => list_of_sources for each module. @@ -939,6 +939,9 @@ def getSystemInstallName(): if nix and os.uname()[2].startswith( "8." ): n += "-tiger" + if len(env.get("MONGO_MODULES", None)): + n += "-" + "-".join(env["MONGO_MODULES"].keys()) + try: findSettingsSetup() import settings diff --git a/distsrc/THIRD-PARTY-NOTICES b/distsrc/THIRD-PARTY-NOTICES index a27586e59e5..f3f4436f7e2 100644 --- a/distsrc/THIRD-PARTY-NOTICES +++ b/distsrc/THIRD-PARTY-NOTICES @@ -263,4 +263,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -End + +6) License notice for SpiderMonkey 1.7 +--------------------------------- +For applicable files: + + Version: MPL 1.1/GPL 2.0/LGPL 2.1 + + The contents of this file are subject to the Mozilla Public License Version + 1.1 (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" basis, + WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + for the specific language governing rights and limitations under the + License. + + The Original Code is Mozilla Communicator client code, released + March 31, 1998. + + The Initial Developer of the Original Code is + Netscape Communications Corporation. + Portions created by the Initial Developer are Copyright (C) 1998 + the Initial Developer. All Rights Reserved. + + Contributor(s): + + Alternatively, the contents of this file may be used under the terms of + either of the GNU General Public License Version 2 or later (the "GPL"), + or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), + in which case the provisions of the GPL or the LGPL are applicable instead + of those above. If you wish to allow use of your version of this file only + under the terms of either the GPL or the LGPL, and not to allow others to + use your version of this file under the terms of the MPL, indicate your + decision by deleting the provisions above and replace them with the notice + and other provisions required by the GPL or the LGPL. If you do not delete + the provisions above, a recipient may use your version of this file under + the terms of any one of the MPL, the GPL or the LGPL. + +End
\ No newline at end of file diff --git a/doxygenConfig b/doxygenConfig index 14f062243ef..33dbf86c489 100644 --- a/doxygenConfig +++ b/doxygenConfig @@ -3,7 +3,7 @@ #--------------------------------------------------------------------------- DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = MongoDB -PROJECT_NUMBER = 2.2.0 +PROJECT_NUMBER = 2.2.2 OUTPUT_DIRECTORY = docs/doxygen CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/jstests/aggregation/testshard1.js b/jstests/aggregation/testshard1.js index 3bd0ff5f1bf..8d5d4f376c4 100644 --- a/jstests/aggregation/testshard1.js +++ b/jstests/aggregation/testshard1.js @@ -131,5 +131,27 @@ for(i = 0; i < 6; ++i) { 'agg sharded test simple match failed'); } +function testSkipLimit(ops, expectedCount) { + if (expectedCount > 10) { + // make shard -> mongos intermediate results less than 16MB + ops.unshift({$project: {_id:1}}) + } + + ops.push({$group: {_id:1, count: {$sum: 1}}}); + + var out = db.runCommand({aggregate:"ts1", pipeline:ops}); + assert.commandWorked(out); + assert.eq(out.result[0].count, expectedCount); +} + +testSkipLimit([], nItems); // control +testSkipLimit([{$skip:10}], nItems - 10); +testSkipLimit([{$limit:10}], 10); +testSkipLimit([{$skip:5}, {$limit:10}], 10); +testSkipLimit([{$limit:10}, {$skip:5}], 10 - 5); +testSkipLimit([{$skip:5}, {$skip: 3}, {$limit:10}], 10); +testSkipLimit([{$skip:5}, {$limit:10}, {$skip: 3}], 10 - 3); +testSkipLimit([{$limit:10}, {$skip:5}, {$skip: 3}], 10 - 3 - 5); + // shut everything down shardedAggTest.stop(); diff --git a/jstests/cursorb.js b/jstests/cursorb.js new file mode 100644 index 00000000000..65e356e89cb --- /dev/null +++ b/jstests/cursorb.js @@ -0,0 +1,17 @@ +// The 'cursor not found in map -1' warning is not logged when get more exhausts a client cursor. +// SERVER-6931 + +t = db.jstests_cursorb; +t.drop(); + +// Exhaust a client cursor in get more. +for( i = 0; i < 200; ++i ) { + t.save( { a:i } ); +} +t.find().itcount(); + +// Check that the 'cursor not found in map -1' message is not printed. This message indicates an +// attempt to look up a cursor with an invalid id and should never appear in the log. +log = db.adminCommand( { getLog:'global' } ).log +log.forEach( function( line ) { assert( !line.match( /cursor not found in map -1 / ), + 'Cursor map lookup with id -1.' ); } ); diff --git a/jstests/distinct3.js b/jstests/distinct3.js index c2678838af8..d15cba00fe3 100644 --- a/jstests/distinct3.js +++ b/jstests/distinct3.js @@ -16,8 +16,15 @@ for( i = 0; i < 1000; ++i ) { } db.getLastError(); -// The idea here is to try and remove the last match for the {a:1} index scan while distinct is yielding. -p = startParallelShell( 'for( i = 0; i < 2500; ++i ) { db.jstests_distinct3.remove({a:49}); for( j = 0; j < 20; ++j ) { db.jstests_distinct3.save({a:49,c:49,d:j}) } }' ); +// Attempt to remove the last match for the {a:1} index scan while distinct is yielding. +p = startParallelShell( 'for( i = 0; i < 2500; ++i ) { ' + + ' db.jstests_distinct3.remove( { a:49 } ); ' + + ' for( j = 0; j < 20; ++j ) { ' + + ' db.jstests_distinct3.save( { a:49, c:49, d:j } ); ' + + ' } ' + + '} ' + + '// Wait for the above writes to complete. ' + + 'db.getLastError(); ' ); for( i = 0; i < 100; ++i ) { count = t.distinct( 'c', {$or:[{a:{$gte:0},d:0},{b:{$gte:0}}]} ).length; diff --git a/jstests/evalb.js b/jstests/evalb.js index 0aa9f47f1f9..aa56e972384 100644 --- a/jstests/evalb.js +++ b/jstests/evalb.js @@ -1,17 +1,40 @@ +// Check the return value of a db.eval function running a database query, and ensure the function's +// contents are logged in the profile log. -t = db.evalb; -t.drop(); +// Use a reserved database name to avoid a conflict in the parallel test suite. +var stddb = db; +var db = db.getSisterDB( 'evalb' ); -t.save( { x : 3 } ); +function profileCursor() { + return db.system.profile.find( { user:username } ); +} -assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "A" ); +function lastOp() { + return profileCursor().sort( { $natural:-1 } ).next(); +} -db.setProfilingLevel( 2 ); +try { -assert.eq( 3, db.eval( function(){ return db.evalb.findOne().x; } ) , "B" ); + username = 'jstests_evalb_user'; + db.addUser( username, 'password', false, 1 ); + db.auth( username, 'password' ); -o = db.system.profile.find( { "command.$eval" : { $exists : true } } ).sort( { $natural : -1 } ).limit(1).next(); -assert( tojson(o).indexOf( "findOne().x" ) > 0 , "C : " + tojson( o ) ) + t = db.evalb; + t.drop(); -db.setProfilingLevel( 0 ); + t.save( { x:3 } ); + assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'A' ); + + db.setProfilingLevel( 2 ); + + assert.eq( 3, db.eval( function() { return db.evalb.findOne().x; } ), 'B' ); + + o = lastOp(); + assert( tojson( o ).indexOf( 'findOne().x' ) > 0, 'C : ' + tojson( o ) ); +} +finally { + + db.setProfilingLevel(0); + db = stddb; +} diff --git a/jstests/find_and_modify_server6909.js b/jstests/find_and_modify_server6909.js new file mode 100644 index 00000000000..2f688459698 --- /dev/null +++ b/jstests/find_and_modify_server6909.js @@ -0,0 +1,21 @@ +c = db.find_and_modify_server6906; + + +c.drop(); + +c.insert( { _id : 5 , a:{ b:1 } } ); +ret = c.findAndModify( { query:{ 'a.b':1 }, + update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches. + new:true } ); +assert.eq( 5, ret._id ); +assert.eq( 2, ret.a.b ); + + +c.drop(); + +c.insert( { _id : null , a:{ b:1 } } ); +ret = c.findAndModify( { query:{ 'a.b':1 }, + update:{ $set:{ 'a.b':2 } }, // Ensure the query on 'a.b' no longer matches. + new:true } ); +assert.eq( 2, ret.a.b ); + diff --git a/jstests/find_and_modify_server6993.js b/jstests/find_and_modify_server6993.js new file mode 100644 index 00000000000..b8a31915372 --- /dev/null +++ b/jstests/find_and_modify_server6993.js @@ -0,0 +1,9 @@ + +c = db.find_and_modify_server6993; +c.drop(); + +c.insert( { a:[ 1, 2 ] } ); + +c.findAndModify( { query:{ a:1 }, update:{ $set:{ 'a.$':5 } } } ); + +assert.eq( 5, c.findOne().a[ 0 ] ); diff --git a/jstests/replsets/no_chaining.js b/jstests/replsets/no_chaining.js new file mode 100644 index 00000000000..c937dbed1eb --- /dev/null +++ b/jstests/replsets/no_chaining.js @@ -0,0 +1,68 @@ + +function myprint( x ) { + print( "chaining output: " + x ); +} + +var replTest = new ReplSetTest({name: 'testSet', nodes: 3}); +var nodes = replTest.startSet(); +var hostnames = replTest.nodeList(); +replTest.initiate( + { + "_id" : "testSet", + "members" : [ + {"_id" : 0, "host" : hostnames[0], "priority" : 2}, + {"_id" : 1, "host" : hostnames[1]}, + {"_id" : 2, "host" : hostnames[2]} + ], + "settings" : { + "chainingAllowed" : false + } + } +); + +var master = replTest.getMaster(); +replTest.awaitReplication(); + + +var breakNetwork = function() { + replTest.bridge(); + replTest.partition(0, 2); + master = replTest.getMaster(); +}; + +var checkNoChaining = function() { + master.getDB("test").foo.insert({x:1}); + + assert.soon( + function() { + return nodes[1].getDB("test").foo.findOne() != null; + } + ); + + var endTime = (new Date()).getTime()+10; + while ((new Date()).getTime() < endTime) { + assert(nodes[2].getDB("test").foo.findOne() == null, + 'Check that 2 does not catch up'); + } +}; + +var forceSync = function() { + assert.soon( + function() { + nodes[2].getDB("admin").runCommand({replSetSyncFrom : hostnames[1]}); + return nodes[2].getDB("test").foo.findOne() != null; + }, + 'Check force sync still works' + ); +}; + +if (!_isWindows()) { + print("break the network so that node 2 cannot replicate"); + breakNetwork(); + + print("make sure chaining is not happening"); + checkNoChaining(); + + print("check that forcing sync target still works"); + forceSync(); +} diff --git a/jstests/replsets/replset9.js b/jstests/replsets/replset9.js new file mode 100644 index 00000000000..e3912877405 --- /dev/null +++ b/jstests/replsets/replset9.js @@ -0,0 +1,52 @@ + + +var rt = new ReplSetTest( { name : "replset9tests" , nodes: 1, oplogSize: 400 } ); + +var nodes = rt.startSet(); +rt.initiate(); +var master = rt.getMaster(); +var bigstring = "a"; +var md = master.getDB( 'd' ); +var mdc = md[ 'c' ]; + +// idea: while cloner is running, update some docs and then immediately remove them. +// oplog will have ops referencing docs that no longer exist. + +var doccount = 20000; +// Avoid empty extent issues +mdc.insert( { _id:-1, x:"dummy" } ); + +// Make this db big so that cloner takes a while. +print ("inserting bigstrings"); +for( i = 0; i < doccount; ++i ) { + mdc.insert( { _id:i, x:bigstring } ); + bigstring += "a"; +} +md.getLastError(); + +// Insert some docs to update and remove +print ("inserting x"); +for( i = doccount; i < doccount*2; ++i ) { + mdc.insert( { _id:i, bs:bigstring, x:i } ); +} +md.getLastError(); + +// add a secondary; start cloning +var slave = rt.add(); +rt.reInitiate(); +print ("initiation complete!"); +var sc = slave.getDB( 'd' )[ 'c' ]; +slave.setSlaveOk(); + +print ("updating and deleting documents"); +for (i = doccount*2; i > doccount; --i) { + mdc.update( { _id:i }, { $inc: { x : 1 } } ); + md.getLastError(); + mdc.remove( { _id:i } ); + md.getLastError(); + mdc.insert( { bs:bigstring } ); + md.getLastError(); +} +print ("finished"); +// Wait for replication to catch up. +rt.awaitReplication(640000); diff --git a/jstests/sharding/addshard5.js b/jstests/sharding/addshard5.js index b272ae4ab1e..a62dbe43c98 100644 --- a/jstests/sharding/addshard5.js +++ b/jstests/sharding/addshard5.js @@ -36,8 +36,8 @@ coll.insert({ hello : "world" }) assert.eq( null, coll.getDB().getLastError() ) // Migrate the collection to and from shard2 so shard1 loads the shard2 host -printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id }) ) -printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[0]._id }) ) +printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[1]._id, _waitForDelete : true }) ) +printjson( admin.runCommand({ moveChunk : coll + "", find : { _id : 0 }, to : shards[0]._id, _waitForDelete : true }) ) // // Drop and re-add shard with last shard's host diff --git a/jstests/sharding/authCommands.js b/jstests/sharding/authCommands.js index 37348bf163f..fd9e0118960 100644 --- a/jstests/sharding/authCommands.js +++ b/jstests/sharding/authCommands.js @@ -16,9 +16,12 @@ assert.eq(0, db.runCommand({dbStats : 1}).ok); assert( db.getSiblingDB('local').auth('__system', 'foopdedoop'), "Failed to authenticate as system user" ); -assert.eq(0, db.runCommand({dbStats : 1}).ok); +// Because of SERVER-6897, commands sent without an $auth table are assumed to have full access +// to preserve compatibility with 2.0 +// assert.eq(0, db.runCommand({dbStats : 1}).ok); // SERVER-6897 assert.eq(1, db.runCommand({dbStats : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok ); -assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary. + // SERVER-6897 +// assert.eq(0, db.runCommand({dbStats : 1}).ok); // Make sure the credentials are temporary. assert.eq(0, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(1) } } } ).ok ); assert.eq(1, db.runCommand({dropDatabase : 1, $auth : { test : { userName : NumberInt(2) } } } ).ok ); diff --git a/jstests/sharding/auth_slaveok_routing.js b/jstests/sharding/auth_slaveok_routing.js index 242966e8d6e..c775228179b 100644 --- a/jstests/sharding/auth_slaveok_routing.js +++ b/jstests/sharding/auth_slaveok_routing.js @@ -84,6 +84,12 @@ coll.setSlaveOk( true ); */ ReplSetTest.awaitRSClientHosts( mongos, replTest.getSecondaries(), { ok : true, secondary : true }); +// +// We also need to wait for the primary, it's possible that the mongos may think a node is a +// secondary but it actually changed to a primary before we send our final query. +// +ReplSetTest.awaitRSClientHosts( mongos, replTest.getPrimary(), + { ok : true, ismaster : true }); // Recheck if we can still query secondaries after refreshing connections. jsTest.log( 'Final query to SEC' ); diff --git a/jstests/sharding/count1.js b/jstests/sharding/count1.js index 4af42caf7a1..c7994c22712 100644 --- a/jstests/sharding/count1.js +++ b/jstests/sharding/count1.js @@ -63,7 +63,7 @@ assert.eq( 6 , db.foo.find().count() , "basic count after split " ); assert.eq( 6 , db.foo.find().sort( { name : 1 } ).count() , "basic count after split sorted " ); // part 4 -s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : secondary.getMongo().name } ); +s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : secondary.getMongo().name , _waitForDelete : true } ); assert.eq( 3 , primary.foo.find().toArray().length , "primary count" ); assert.eq( 3 , secondary.foo.find().toArray().length , "secondary count" ); diff --git a/jstests/sharding/count2.js b/jstests/sharding/count2.js index ae0559ddb1a..edac3b692ec 100644 --- a/jstests/sharding/count2.js +++ b/jstests/sharding/count2.js @@ -26,7 +26,7 @@ assert.eq( 3, db2.count( { name : { $gte: "aaa" , $lt: "ddd" } } ) , "initial co s1.printChunks( "test.foo" ) -s1.adminCommand( { movechunk : "test.foo" , find : { name : "aaa" } , to : s1.getOther( s1.getServer( "test" ) ).name } ); +s1.adminCommand( { movechunk : "test.foo" , find : { name : "aaa" } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true }); assert.eq( 3, db1.count( { name : { $gte: "aaa" , $lt: "ddd" } } ) , "post count mongos1" ); diff --git a/jstests/sharding/delete_during_migrate.js b/jstests/sharding/delete_during_migrate.js new file mode 100644 index 00000000000..4b01d16734a --- /dev/null +++ b/jstests/sharding/delete_during_migrate.js @@ -0,0 +1,42 @@ +// Test migrating a big chunk while deletions are happening within that chunk. +// Test is slightly non-deterministic, since removes could happen before migrate +// starts. Protect against that by making chunk very large. + +// start up a new sharded cluster +var st = new ShardingTest({ shards : 2, mongos : 1 }); + +// stop balancer since we want manual control for this +st.stopBalancer(); + +var dbname = "testDB"; +var coll = "foo"; +var ns = dbname + "." + coll; +var s = st.s0; +var t = s.getDB( dbname ).getCollection( coll ); + +// Create fresh collection with lots of docs +t.drop(); +for ( i=0; i<200000; i++ ){ + t.insert( { a : i } ); +} + +// enable sharding of the collection. Only 1 chunk. +t.ensureIndex( { a : 1 } ); +s.adminCommand( { enablesharding : dbname } ); +s.adminCommand( { shardcollection : ns , key: { a : 1 } } ); + +// start a parallel shell that deletes things +startMongoProgramNoConnect( "mongo" , + "--host" , getHostName() , + "--port" , st.s0.port , + "--eval" , "db." + coll + ".remove({});" , + dbname ); + +// migrate while deletions are happening +var moveResult = s.adminCommand( { moveChunk : ns , + find : { a : 1 } , + to : st.getOther( st.getServer( dbname ) ).name } ); +// check if migration worked +assert( moveResult.ok , "migration didn't work while doing deletes" ); + +st.stop(); diff --git a/jstests/sharding/error1.js b/jstests/sharding/error1.js index b60ffa82226..97614c8addf 100644 --- a/jstests/sharding/error1.js +++ b/jstests/sharding/error1.js @@ -24,7 +24,7 @@ db.foo2.save( { _id : 3 , num : 15 } ); db.foo2.save( { _id : 4 , num : 20 } ); s.adminCommand( { split : "test.foo2" , middle : { num : 10 } } ); -s.adminCommand( { movechunk : "test.foo2" , find : { num : 20 } , to : s.getOther( s.getServer( "test" ) ).name } ); +s.adminCommand( { movechunk : "test.foo2" , find : { num : 20 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } ); print( "a: " + a.foo2.count() ); print( "b: " + b.foo2.count() ); diff --git a/jstests/sharding/findandmodify2.js b/jstests/sharding/findandmodify2.js index 1ca794bfd7e..542818c7167 100644 --- a/jstests/sharding/findandmodify2.js +++ b/jstests/sharding/findandmodify2.js @@ -103,10 +103,10 @@ s.printChunks(); print("---------- Verifying that both codepaths resulted in splits..."); -assert.gt( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" ); -assert.gt( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" ); -assert.gt( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" ); -assert.gt( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" ); +assert.gte( s.config.chunks.count({ "ns": "test." + col_fam }), minChunks, "findAndModify update code path didn't result in splits" ); +assert.gte( s.config.chunks.count({ "ns": "test." + col_fam_upsert }), minChunks, "findAndModify upsert code path didn't result in splits" ); +assert.gte( s.config.chunks.count({ "ns": "test." + col_update }), minChunks, "update code path didn't result in splits" ); +assert.gte( s.config.chunks.count({ "ns": "test." + col_update_upsert }), minChunks, "upsert code path didn't result in splits" ); printjson( db[col_update].stats() ); diff --git a/jstests/sharding/geo_near_random2.js b/jstests/sharding/geo_near_random2.js index b9800274bb9..71b0406cf7a 100644 --- a/jstests/sharding/geo_near_random2.js +++ b/jstests/sharding/geo_near_random2.js @@ -18,7 +18,7 @@ test.insertPts(5000); for (var i = (test.nPts/10); i < test.nPts; i+= (test.nPts/10)){ s.adminCommand({split: ('test.' + testName), middle: {_id: i} }); try { - s.adminCommand({moveChunk: ('test.' + testName), find: {_id: i-1}, to: ('shard000' + (i%3))}); + s.adminCommand({moveChunk: ('test.' + testName), find: {_id: i-1}, to: ('shard000' + (i%3)), _waitForDelete : true }); } catch (e) { // ignore this error if (! e.match(/that chunk is already on that shard/)){ diff --git a/jstests/sharding/key_many.js b/jstests/sharding/key_many.js index 42cacddb76b..75a9784096f 100644 --- a/jstests/sharding/key_many.js +++ b/jstests/sharding/key_many.js @@ -98,7 +98,7 @@ for ( var i=0; i<types.length; i++ ){ s.adminCommand( { split : longName , find : makeObjectDotted( curT.values[3] ) } ); s.adminCommand( { split : longName , find : makeObjectDotted( curT.values[3] ) } ); - s.adminCommand( { movechunk : longName , find : makeObjectDotted( curT.values[0] ) , to : secondary.getMongo().name } ); + s.adminCommand( { movechunk : longName , find : makeObjectDotted( curT.values[0] ) , to : secondary.getMongo().name, _waitForDelete : true } ); s.printChunks(); diff --git a/jstests/sharding/key_string.js b/jstests/sharding/key_string.js index bbc5dfb49ec..63ed7771db8 100644 --- a/jstests/sharding/key_string.js +++ b/jstests/sharding/key_string.js @@ -24,7 +24,7 @@ s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // [Minkey - s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // * [allan -> sara) , [sara -> Maxkey) s.adminCommand( { split : "test.foo" , find : { name : "joe" } } ); // [alan -> joe) , [joe -> sara] -s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : seconday.getMongo().name } ); +s.adminCommand( { movechunk : "test.foo" , find : { name : "allan" } , to : seconday.getMongo().name, _waitForDelete : true } ); s.printChunks(); diff --git a/jstests/sharding/limit_push.js b/jstests/sharding/limit_push.js index 75ad271deb3..b508e307eb7 100644 --- a/jstests/sharding/limit_push.js +++ b/jstests/sharding/limit_push.js @@ -3,6 +3,9 @@ s = new ShardingTest( "limit_push", 2, 1, 1 ); +// Stop balancer since we do manual moves. +s.stopBalancer(); + db = s.getDB( "test" ); // Create some data @@ -16,7 +19,7 @@ s.adminCommand( { shardcollection : "test.limit_push" , key : { x : 1 } } ); // Now split the and move the data between the shards s.adminCommand( { split : "test.limit_push", middle : { x : 50 }} ); -s.adminCommand( { moveChunk: "test.limit_push", find : { x : 51}, to : "shard0000" }) +s.adminCommand( { moveChunk: "test.limit_push", find : { x : 51}, to : "shard0000", _waitForDelete : true }) // Check that the chunck have split correctly assert.eq( 2 , s.config.chunks.count() , "wrong number of chunks"); diff --git a/jstests/sharding/mongos_validate_backoff.js b/jstests/sharding/mongos_validate_backoff.js index 9b1ecfc5c50..c4f7e16f864 100644 --- a/jstests/sharding/mongos_validate_backoff.js +++ b/jstests/sharding/mongos_validate_backoff.js @@ -39,7 +39,7 @@ for( var test = 0; test < 3; test++ ){ // Kind a heuristic test, we want to make sure that the error wait after sleeping is much less // than the error wait after a lot of errors - assert.gt( lastWait, firstWait * 2 * 2 * 2 * 2 ) + assert.gt( lastWait, firstWait * 2 * 2 ) // Sleeping for long enough to reset our exponential counter sleep( 3000 ) diff --git a/jstests/sharding/mrShardedOutput.js b/jstests/sharding/mrShardedOutput.js index 4655f2d8fd3..e711b9f4940 100644 --- a/jstests/sharding/mrShardedOutput.js +++ b/jstests/sharding/mrShardedOutput.js @@ -21,16 +21,59 @@ function reduce2(key, values) { return values[0]; } var numdocs = 0; var numbatch = 100000; var nchunks = 0; -for ( iter=0; iter<2; iter++ ){ + +var numIterations = 2; + +for (var it = 0; it < numIterations; it++) { + + jsTest.log("Starting new insert batch..."); + // add some more data for input so that chunks will get split further - for (i=0; i<numbatch; i++){ db.foo.save({a: Math.random() * 1000, y:str})} - db.getLastError(); + for (i=0; i<numbatch; i++){ db.foo.save({a: Math.random() * 1000, y:str, i : numdocs + i})} + + assert.eq(null, db.getLastError()); + + jsTest.log("No errors on insert batch.") + numdocs += numbatch var isBad = db.foo.find().itcount() != numdocs + if (isBad) jsTest.log("Insert count is smaller than full count!") + + if (isBad) { + + jsTest.log( "Showing document distribution because documents missed..." ) + + // Stop balancing + s.stopBalancer(); + + // Wait for writebacks + sleep( 10000 ); + + s.printShardingStatus(true); + + var shards = config.shards.find().toArray(); + + for (var i = 0; i < shards.length; i++){ + + var shard = new Mongo(shards[i].host) + + var partialColl = shard.getCollection(db.foo + "").find(); + + while (partialColl.hasNext()) { + var obj = partialColl.next(); + delete obj.y; + print(tojson(obj)); + } + } + + jsTest.log( "End document distribution." ) + } + // Verify that wbl weirdness isn't causing this assert.soon( function(){ var c = db.foo.find().itcount(); print( "Count is " + c ); return c == numdocs } ) + assert( ! isBad ) //assert.eq( numdocs, db.foo.find().itcount(), "Not all data was saved!" ) diff --git a/jstests/sharding/multi_mongos1.js b/jstests/sharding/multi_mongos1.js index 9778ff6059d..79b6dc08a48 100644 --- a/jstests/sharding/multi_mongos1.js +++ b/jstests/sharding/multi_mongos1.js @@ -43,11 +43,11 @@ s1.adminCommand( { split : "test.foo" , middle : { num : 1 } } ); s1.adminCommand( { split : "test.foo" , middle : { num : N } } ); // s2 is now stale w.r.t boundaires around { num: 1 } -res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name } ); +res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true } ); assert.eq( 0 , res.ok , "a move with stale boundaries should not have succeeded" + tojson(res) ); // s2 must have reloaded as a result of a failed move; retrying should work -res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name } ); +res = s2.getDB( "admin" ).runCommand( { movechunk : "test.foo" , find : { num : 1 } , to : s1.getOther( s1.getServer( "test" ) ).name, _waitForDelete : true } ); assert.eq( 1 , res.ok , "mongos did not reload after a failed migrate" + tojson(res) ); // s1 is not stale about the boundaries of [MinKey->1) diff --git a/jstests/sharding/no_empty_reset.js b/jstests/sharding/no_empty_reset.js index 3f37e6f956f..62ef74cc6b0 100644 --- a/jstests/sharding/no_empty_reset.js +++ b/jstests/sharding/no_empty_reset.js @@ -29,7 +29,7 @@ var fullShard = st.getShard( coll, { _id : 1 } ) var emptyShard = st.getShard( coll, { _id : -1 } ) var admin = st.s.getDB( "admin" ) -printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : -1 }, to : fullShard.shardName }) ) +printjson( admin.runCommand({ moveChunk : "" + coll, find : { _id : -1 }, to : fullShard.shardName, _waitForDelete : true }) ) jsTestLog( "Resetting shard version via first mongos..." ) diff --git a/jstests/sharding/read_pref.js b/jstests/sharding/read_pref.js index 7168d269eb0..4fc4cb87b68 100755 --- a/jstests/sharding/read_pref.js +++ b/jstests/sharding/read_pref.js @@ -112,12 +112,14 @@ assert.eq( primaryNode.name, explain.server ); assert.eq( 1, explain.n ); // Kill all members except one +var stoppedNodes = []; for ( var x = 0; x < NODES - 1; x++ ){ replTest.stop( x ); + stoppedNodes.push( replTest.nodes[x] ); } // Wait for ReplicaSetMonitor to realize nodes are down -ReplSetTest.awaitRSClientHosts( conn, replTest.nodes[0], { ok: false }, replTest.name ); +ReplSetTest.awaitRSClientHosts( conn, stoppedNodes, { ok: false }, replTest.name ); // Wait for the last node to be in steady state -> secondary (not recovering) var lastNode = replTest.nodes[NODES - 1]; diff --git a/jstests/sharding/read_pref_multi_mongos_stale_config.js b/jstests/sharding/read_pref_multi_mongos_stale_config.js new file mode 100644 index 00000000000..1556adef9e8 --- /dev/null +++ b/jstests/sharding/read_pref_multi_mongos_stale_config.js @@ -0,0 +1,34 @@ +var st = new ShardingTest({ shards: { rs0: { quiet: '' }, rs1: { quiet: '' }}, mongos: 2 }); + +var testDB1 = st.s0.getDB('test'); +var testDB2 = st.s1.getDB('test'); + +// Trigger a query on mongos 1 so it will have a view of test.user as being unsharded. +testDB1.user.findOne(); + +testDB2.adminCommand({ enableSharding: 'test' }); +testDB2.adminCommand({ shardCollection: 'test.user', key: { x: 1 }}); + +testDB2.adminCommand({ split: 'test.user', middle: { x: 100 }}); + +var configDB2 = st.s1.getDB('config'); +var chunkToMove = configDB2.chunks.find().sort({ min: 1 }).next(); +var toShard = configDB2.shards.findOne({ _id: { $ne: chunkToMove.shard }})._id; +testDB2.adminCommand({ moveChunk: 'test.user', to: toShard, find: { x: 50 }}); + +for (var x = 0; x < 200; x++) { + testDB2.user.insert({ x: x }); +} + +testDB2.runCommand({ getLastError: 1 }); + +var cursor = testDB1.user.find({ x: 30 }).readPref('primary'); +assert(cursor.hasNext()); +assert.eq(30, cursor.next().x); + +cursor = testDB1.user.find({ x: 130 }).readPref('primary'); +assert(cursor.hasNext()); +assert.eq(130, cursor.next().x); + +st.stop(); + diff --git a/jstests/sharding/read_pref_rs_client.js b/jstests/sharding/read_pref_rs_client.js index df9a7ee1327..af7eac5d09a 100644 --- a/jstests/sharding/read_pref_rs_client.js +++ b/jstests/sharding/read_pref_rs_client.js @@ -180,12 +180,7 @@ function noPriSecOkTest() { coll.find().readPref('primary').explain(); }); - // Needs to restart server, otherwise the js Mongo constructor - // would throw because it can't find the primary - replTest.start(0, {}, true); - replTest.awaitSecondaryNodes(); replConn = new Mongo(replTest.getURL()); - replTest.stop(0); coll = replConn.getDB('test').user; var dest = coll.find().readPref('primaryPreferred').explain().server; assert.eq(SEC_HOST, dest); diff --git a/jstests/sharding/repl_monitor_refresh.js b/jstests/sharding/repl_monitor_refresh.js index 7214cb2081b..de2d7ae2347 100644 --- a/jstests/sharding/repl_monitor_refresh.js +++ b/jstests/sharding/repl_monitor_refresh.js @@ -5,7 +5,7 @@ var NODE_COUNT = 3; var st = new ShardingTest({ shards: { rs0: { nodes: NODE_COUNT, oplogSize: 10 }}, - separateConfig: true }); + separateConfig: true, config : 3 }); var replTest = st.rs0; var mongos = st.s; diff --git a/jstests/sharding/return_partial_shards_down.js b/jstests/sharding/return_partial_shards_down.js new file mode 100644 index 00000000000..e5dcd33ef6a --- /dev/null +++ b/jstests/sharding/return_partial_shards_down.js @@ -0,0 +1,96 @@ +// +// Tests that zero results are correctly returned with returnPartial and shards down +// + +var st = new ShardingTest({shards : 3, + mongos : 1, + other : {mongosOptions : {verbose : 2}, + separateConfig : true}}); + +var mongos = st.s; +var config = mongos.getDB("config"); +var admin = mongos.getDB("admin"); +var shards = config.shards.find().toArray(); + +for ( var i = 0; i < shards.length; i++) { + shards[i].conn = new Mongo(shards[i].host); +} + +var collOneShard = mongos.getCollection("foo.collOneShard"); +var collAllShards = mongos.getCollection("foo.collAllShards"); + +printjson(admin.runCommand({enableSharding : collOneShard.getDB() + ""})) +printjson(admin.runCommand({movePrimary : collOneShard.getDB() + "", + to : shards[0]._id})); + +printjson(admin.runCommand({shardCollection : collOneShard + "", + key : {_id : 1}})); +printjson(admin.runCommand({shardCollection : collAllShards + "", + key : {_id : 1}})); + +// Split and move the "both shard" collection to both shards + +printjson(admin.runCommand({split : collAllShards + "", + middle : {_id : 0}})); +printjson(admin.runCommand({split : collAllShards + "", + middle : {_id : 1000}})); +printjson(admin.runCommand({moveChunk : collAllShards + "", + find : {_id : 0}, + to : shards[1]._id})); +printjson(admin.runCommand({moveChunk : collAllShards + "", + find : {_id : 1000}, + to : shards[2]._id})); + +// Collections are now distributed correctly +jsTest.log("Collections now distributed correctly."); +st.printShardingStatus(); + +var inserts = [{_id : -1}, + {_id : 1}, + {_id : 1000}]; + +collOneShard.insert(inserts); +collAllShards.insert(inserts); + +assert.eq(null, collOneShard.getDB().getLastError()); + +var returnPartialFlag = 1 << 7; + +jsTest.log("All shards up!"); + +assert.eq(3, collOneShard.find().itcount()); +assert.eq(3, collAllShards.find().itcount()); + +assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); +assert.eq(3, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); + +jsTest.log("One shard down!") + +MongoRunner.stopMongod(st.shard2) + +jsTest.log("done.") + +assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); +assert.eq(2, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); + +jsTest.log("Two shards down!") + +MongoRunner.stopMongod(st.shard1) + +jsTest.log("done.") + +assert.eq(3, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); +assert.eq(1, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); + +jsTest.log("All shards down!") + +MongoRunner.stopMongod(st.shard0) + +jsTest.log("done.") + +assert.eq(0, collOneShard.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); +assert.eq(0, collAllShards.find({}, {}, 0, 0, 0, returnPartialFlag).itcount()); + +jsTest.log("DONE!"); + +st.stop(); diff --git a/jstests/sharding/shard2.js b/jstests/sharding/shard2.js index d7eeb764967..ff03bf7b24b 100644 --- a/jstests/sharding/shard2.js +++ b/jstests/sharding/shard2.js @@ -57,10 +57,10 @@ placeCheck( 2 ); // NOTE: at this point we have 2 shard on 1 server // test move shard -assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : primary.getMongo().name } ); } ); -assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : "adasd" } ) } ); +assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : primary.getMongo().name, _waitForDelete : true } ); } ); +assert.throws( function(){ s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : "adasd", _waitForDelete : true } ) } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : secondary.getMongo().name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : secondary.getMongo().name, _waitForDelete : true } ); assert.eq( 2 , secondary.foo.find().length() , "secondary should have 2 after move shard" ); assert.eq( 1 , primary.foo.find().length() , "primary should only have 1 after move shard" ); @@ -221,10 +221,10 @@ assert.eq( 2 , s.onNumShards( "foo" ) , "on 2 shards" ); secondary.foo.insert( { num : -3 } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : secondary.getMongo().name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : secondary.getMongo().name, _waitForDelete : true } ); assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shards" ); -s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : primary.getMongo().name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : -2 } , to : primary.getMongo().name, _waitForDelete : true } ); assert.eq( 2 , s.onNumShards( "foo" ) , "on 2 shards again" ); assert.eq( 3 , s.config.chunks.count() , "only 3 chunks" ); diff --git a/jstests/sharding/shard3.js b/jstests/sharding/shard3.js index 7b9cc33ccd1..785bcae1a40 100644 --- a/jstests/sharding/shard3.js +++ b/jstests/sharding/shard3.js @@ -37,7 +37,7 @@ assert.eq( 0 , secondary.count() , "s1" ) assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shards" ); s.adminCommand( { split : "test.foo" , middle : { num : 2 } } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } ); assert( primary.find().toArray().length > 0 , "blah 1" ); assert( secondary.find().toArray().length > 0 , "blah 2" ); @@ -89,7 +89,7 @@ assert( a.findOne( { num : 1 } ) , "pre move 1" ) s.printCollectionInfo( "test.foo" ); myto = s.getOther( s.getServer( "test" ) ).name print( "counts before move: " + tojson( s.shardCounts( "foo" ) ) ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : myto } ) +s.adminCommand( { movechunk : "test.foo" , find : { num : 1 } , to : myto, _waitForDelete : true } ) print( "counts after move: " + tojson( s.shardCounts( "foo" ) ) ); s.printCollectionInfo( "test.foo" ); assert.eq( 1 , s.onNumShards( "foo" ) , "on 1 shard again" ); @@ -132,7 +132,7 @@ s.adminCommand( { shardcollection : "test.foo" , key : { num : 1 } } ); a.save( { num : 2 } ); a.save( { num : 3 } ); s.adminCommand( { split : "test.foo" , middle : { num : 2 } } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } ); s.printShardingStatus(); s.printCollectionInfo( "test.foo" , "after dropDatabase setup" ); @@ -165,7 +165,7 @@ assert.eq( 3 , dba.foo.count() , "Ba" ); assert.eq( 3 , dbb.foo.count() , "Bb" ); s.adminCommand( { split : "test2.foo" , middle : { num : 2 } } ); -s.adminCommand( { movechunk : "test2.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test2" ) ).name } ); +s.adminCommand( { movechunk : "test2.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test2" ) ).name, _waitForDelete : true } ); assert.eq( 2 , s.onNumShards( "foo" , "test2" ) , "B on 2 shards" ); diff --git a/jstests/sharding/shard4.js b/jstests/sharding/shard4.js index 2d7a0dff6ad..6b48b02bc2f 100644 --- a/jstests/sharding/shard4.js +++ b/jstests/sharding/shard4.js @@ -1,6 +1,7 @@ // shard4.js s = new ShardingTest( "shard4" , 2 , 50 , 2 ); +s.stopBalancer() s2 = s._mongos[1]; @@ -19,7 +20,7 @@ assert.eq( 7 , s.getDB( "test" ).foo.find().toArray().length , "normal A" ); assert.eq( 7 , s2.getDB( "test" ).foo.find().toArray().length , "other A" ); s.adminCommand( { split : "test.foo" , middle : { num : 4 } } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } ); assert( s._connections[0].getDB( "test" ).foo.find().toArray().length > 0 , "blah 1" ); assert( s._connections[1].getDB( "test" ).foo.find().toArray().length > 0 , "blah 2" ); diff --git a/jstests/sharding/shard5.js b/jstests/sharding/shard5.js index 050a7d70281..ba40014dad6 100644 --- a/jstests/sharding/shard5.js +++ b/jstests/sharding/shard5.js @@ -21,7 +21,7 @@ assert.eq( 7 , s.getDB( "test" ).foo.find().toArray().length , "normal A" ); assert.eq( 7 , s2.getDB( "test" ).foo.find().toArray().length , "other A" ); s.adminCommand( { split : "test.foo" , middle : { num : 4 } } ); -s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name } ); +s.adminCommand( { movechunk : "test.foo" , find : { num : 3 } , to : s.getOther( s.getServer( "test" ) ).name, _waitForDelete : true } ); assert( s._connections[0].getDB( "test" ).foo.find().toArray().length > 0 , "blah 1" ); assert( s._connections[1].getDB( "test" ).foo.find().toArray().length > 0 , "blah 2" ); diff --git a/jstests/sharding/shard6.js b/jstests/sharding/shard6.js index 55ae710ab44..dd93ff6e792 100644 --- a/jstests/sharding/shard6.js +++ b/jstests/sharding/shard6.js @@ -2,7 +2,7 @@ summary = ""; -s = new ShardingTest( "shard6" , 2 , 0 , 2 ); +s = new ShardingTest( "shard6" , 2 , 0 , 2, { separateConfig : true } ); s.config.settings.update( { _id: "balancer" }, { $set : { stopped: true } } , true ); diff --git a/jstests/sharding/writeback_bulk_insert.js b/jstests/sharding/writeback_bulk_insert.js new file mode 100644 index 00000000000..0d27eee28e8 --- /dev/null +++ b/jstests/sharding/writeback_bulk_insert.js @@ -0,0 +1,116 @@ +// +// Tests whether a writeback error during bulk insert hangs GLE +// + +jsTest.log("Starting sharded cluster...") + +var st = new ShardingTest({shards : 1, + mongos : 3, + verbose : 2, + other : {separateConfig : true, + mongosOptions : {noAutoSplit : ""}}}) + +st.stopBalancer() + +var mongosA = st.s0 +var mongosB = st.s1 +var mongosC = st.s2 + +jsTest.log("Adding new collection...") + +var collA = mongosA.getCollection(jsTestName() + ".coll") +collA.insert({hello : "world"}) +assert.eq(null, collA.getDB().getLastError()) + +var collB = mongosB.getCollection("" + collA) +collB.insert({hello : "world"}) +assert.eq(null, collB.getDB().getLastError()) + +var collC = mongosC.getCollection("" + collA) +collC.insert({hello : "world"}) +assert.eq(null, collC.getDB().getLastError()) + +jsTest.log("Enabling sharding...") + +printjson(mongosA.getDB("admin").runCommand({enableSharding : collA.getDB() + + ""})) +printjson(mongosA.getDB("admin").runCommand({shardCollection : collA + "", + key : {_id : 1}})) + +// MongoD doesn't know about the config shard version *until* MongoS tells it +collA.findOne() + +jsTest.log("Preparing bulk insert...") + +var data1MB = "x" +while (data1MB.length < 1024 * 1024) + data1MB += data1MB; + +var data7MB = "" +// Data now at 7MB +for ( var i = 0; i < 7; i++) + data7MB += data1MB; + +print("7MB object size is : " + Object.bsonsize({_id : 0, + d : data7MB})) + +var dataCloseTo8MB = data7MB; +// WARNING - MAGIC NUMBERS HERE +// The idea is to exceed the 16MB limit by just enough so that the message gets +// passed in the +// shell, but adding additional writeback information fails. +for ( var i = 0; i < 1031 * 1024 + 862; i++) { + dataCloseTo8MB += "x" +} + +print("Object size is: " + Object.bsonsize([{_id : 0, + d : dataCloseTo8MB}, + {_id : 1, + d : dataCloseTo8MB}])) + +jsTest.log("Trigger wbl for mongosB...") + +collB.insert([{_id : 0, + d : dataCloseTo8MB}, + {_id : 1, + d : dataCloseTo8MB}]) + +// Will hang if overflow is not detected correctly +jsTest.log("Waiting for GLE...") + +assert.neq(null, collB.getDB().getLastError()) + +print("GLE correctly returned error...") + +assert.eq(3, collA.find().itcount()) +assert.eq(3, collB.find().itcount()) + +var data8MB = ""; +for ( var i = 0; i < 8; i++) { + data8MB += data1MB; +} + +print("Object size is: " + Object.bsonsize([{_id : 0, + d : data8MB}, + {_id : 1, + d : data8MB}])) + +jsTest.log("Trigger wbl for mongosC...") + +collC.insert([{_id : 0, + d : data8MB}, + {_id : 1, + d : data8MB}]) + +// Should succeed since our insert size is 16MB (plus very small overhead) +jsTest.log("Waiting for GLE...") + +assert.eq(null, collC.getDB().getLastError()) + +print("GLE Successful...") + +assert.eq(5, collA.find().itcount()) +assert.eq(5, collB.find().itcount()) +assert.eq(5, collC.find().itcount()) + +st.stop() diff --git a/jstests/slowNightly/balance_tags1.js b/jstests/slowNightly/balance_tags1.js index 5e45838536b..945f0526b17 100644 --- a/jstests/slowNightly/balance_tags1.js +++ b/jstests/slowNightly/balance_tags1.js @@ -1,5 +1,5 @@ -s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { chunksize : 1 , nopreallocj : true } ) +s = new ShardingTest( "balance_tags1" , 3 , 1 , 1 , { sync:true, chunksize : 1 , nopreallocj : true } ) s.config.settings.update( { _id: "balancer" }, { $set : { stopped: false, _nosleep: true } } , true ); db = s.getDB( "test" ); @@ -11,9 +11,13 @@ db.getLastError(); s.adminCommand( { enablesharding : "test" } ) s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } ); +s.stopBalancer(); + for ( i=0; i<20; i++ ) s.adminCommand( { split : "test.foo" , middle : { _id : i } } ); +s.startBalancer(); + sh.status( true ) assert.soon( function() { counts = s.chunkCounts( "foo" ); @@ -42,6 +46,7 @@ assert.soon( function() { return counts["shard0002"] == 0; } , "balance 2 didn't happen" , 1000 * 60 * 10 , 1000 ) +printjson(sh.status()); s.stop(); diff --git a/jstests/slowNightly/sharding_balance4.js b/jstests/slowNightly/sharding_balance4.js index 43ca23d78bf..33db1db1da9 100644 --- a/jstests/slowNightly/sharding_balance4.js +++ b/jstests/slowNightly/sharding_balance4.js @@ -3,6 +3,7 @@ // check that doing updates done during a migrate all go to the right place s = new ShardingTest( "slow_sharding_balance4" , 2 , 1 , 1 , { chunksize : 1 } ) +s.stopBalancer(); s.adminCommand( { enablesharding : "test" } ); s.adminCommand( { shardcollection : "test.foo" , key : { _id : 1 } } ); @@ -63,7 +64,11 @@ function check( msg , dontAssert ){ print( "not asserting for key failure: " + x + " want: " + e + " got: " + tojson(z) ) return false; } - + + s.s.getDB("admin").runCommand({ setParameter : 1, logLevel : 2 }) + + printjson( db.foo.findOne( { _id : parseInt( x ) } ) ) + // we will assert past this point but wait a bit to see if it is because the missing update // was being held in the writeback roundtrip sleep( 10000 ); @@ -84,6 +89,9 @@ function check( msg , dontAssert ){ } function diff1(){ + + jsTest.log("Running diff1...") + var myid = doUpdate( false ) var le = db.getLastErrorCmd(); @@ -117,6 +125,8 @@ function sum(){ assert.lt( 20 , diff1() ,"initial load" ); print( diff1() ) +s.startBalancer(); + assert.soon( function(){ var d = diff1(); diff --git a/jstests/slowNightly/sharding_migrate_cursor1.js b/jstests/slowNightly/sharding_migrate_cursor1.js index 71981027b09..3054fb322b9 100644 --- a/jstests/slowNightly/sharding_migrate_cursor1.js +++ b/jstests/slowNightly/sharding_migrate_cursor1.js @@ -57,6 +57,11 @@ print( "cursor should be gone" ) join(); +assert.soon( function(){ + print( "Waiting for migrate cleanup to complete..." ); + return t.count() == t.find().itcount(); +}) + //assert.soon( function(){ return numDocs == t.count(); } , "at end 1" ) sleep( 5000 ) assert.eq( numDocs , t.count() , "at end 2" ) diff --git a/jstests/slowNightly/sharding_rs2.js b/jstests/slowNightly/sharding_rs2.js index dde363a52b7..c5a22d95b8f 100644 --- a/jstests/slowNightly/sharding_rs2.js +++ b/jstests/slowNightly/sharding_rs2.js @@ -125,7 +125,7 @@ s.adminCommand( { split : "test.foo" , middle : { x : 50 } } ) db.printShardingStatus() other = s.config.shards.findOne( { _id : { $ne : serverName } } ); -s.adminCommand( { moveChunk : "test.foo" , find : { x : 10 } , to : other._id } ) +s.adminCommand( { moveChunk : "test.foo" , find : { x : 10 } , to : other._id, _waitForDelete : true } ) assert.eq( 100 , t.count() , "C3" ) assert.eq( 50 , rs.test.getMaster().getDB( "test" ).foo.count() , "C4" ) diff --git a/jstests/slowWeekly/minvalid2.js b/jstests/slowWeekly/minvalid2.js new file mode 100644 index 00000000000..f7d5dd534d4 --- /dev/null +++ b/jstests/slowWeekly/minvalid2.js @@ -0,0 +1,70 @@ +/** + * This checks rollback, which shouldn't happen unless we have reached minvalid. + * 1. make 3-member set w/arb (2) + * 2. shut down 1 + * 3. do writes to 0 + * 4. modify 0's minvalid + * 5. shut down 0 + * 6. start up 1 + * 7. writes on 1 + * 8. start up 0 + * 9. check 0 does not rollback + */ + +print("1. make 3-member set w/arb (2)"); +var name = "minvalid" +var replTest = new ReplSetTest({name: name, nodes: 3, oplogSize:1}); +var host = getHostName(); + +var nodes = replTest.startSet(); +replTest.initiate({_id : name, members : [ + {_id : 0, host : host+":"+replTest.ports[0]}, + {_id : 1, host : host+":"+replTest.ports[1]}, + {_id : 2, host : host+":"+replTest.ports[2], arbiterOnly : true} +]}); +var master = replTest.getMaster(); +var mdb = master.getDB("foo"); + +mdb.foo.save({a: 1000}); +replTest.awaitReplication(); + +print("2: shut down 1"); +replTest.stop(1); + +print("3: do writes to 0"); +mdb.foo.save({a: 1001}); + +print("4: modify 0's minvalid"); +var local = master.getDB("local"); +var lastOp = local.oplog.rs.find().sort({$natural:-1}).limit(1).next(); +printjson(lastOp); + +local.replset.minvalid.insert({ts:new Timestamp(lastOp.ts.t, lastOp.ts.i+1), + h:new NumberLong("1234567890")}); +printjson(local.replset.minvalid.findOne()); + +print("5: shut down 0"); +replTest.stop(0); + +print("6: start up 1"); +replTest.restart(1); + +print("7: writes on 1") +master = replTest.getMaster(); +mdb1 = master.getDB("foo"); +mdb1.foo.save({a:1002}); + +print("8: start up 0"); +replTest.restart(0); + +print("9: check 0 does not rollback"); +assert.soon(function(){ + var status = master.adminCommand({replSetGetStatus:1}); + var stateStr = status.members[0].stateStr; + assert(stateStr != "ROLLBACK" && + stateStr != "SECONDARY" && + stateStr != "PRIMARY", tojson(status)); + return stateStr == "FATAL"; +}); + +replTest.stopSet(15); diff --git a/jstests/tool/dumprestore_auth.js b/jstests/tool/dumprestore_auth.js new file mode 100644 index 00000000000..6f0e6c0a05c --- /dev/null +++ b/jstests/tool/dumprestore_auth.js @@ -0,0 +1,28 @@ +// dumprestore_auth.js + +t = new ToolTest("dumprestore_auth", { auth : "" }); + +c = t.startDB("foo"); + +adminDB = c.getDB().getSiblingDB('admin'); +adminDB.addUser('admin', 'password'); +adminDB.auth('admin','password'); + +assert.eq(0 , c.count() , "setup1"); +c.save({ a : 22 }); +assert.eq(1 , c.count() , "setup2"); + +t.runTool("dump" , "--out" , t.ext, "--username", "admin", "--password", "password"); + +c.drop(); +assert.eq(0 , c.count() , "after drop"); + +t.runTool("restore" , "--dir" , t.ext); // Should fail +assert.eq(0 , c.count() , "after restore without auth"); + +t.runTool("restore" , "--dir" , t.ext, "--username", "admin", "--password", "password"); +assert.soon("c.findOne()" , "no data after sleep"); +assert.eq(1 , c.count() , "after restore 2"); +assert.eq(22 , c.findOne().a , "after restore 2"); + +t.stop(); diff --git a/jstests/tool/exportimport1.js b/jstests/tool/exportimport1.js index 451078e1b95..f4dcbee6b46 100644 --- a/jstests/tool/exportimport1.js +++ b/jstests/tool/exportimport1.js @@ -19,7 +19,11 @@ assert.eq( 1 , c.count() , "after restore 2" ); var doc = c.findOne(); assert.eq( 22 , doc.a , "after restore 2" ); for (var i=0; i<arr.length; i++) { - assert.eq( arr[i], doc.b[i] , "after restore array: "+i ); + if (typeof arr[i] == 'undefined') { + assert.eq( { "$undefined" : true }, doc.b[i] , "after restore array: "+i ); + } else { + assert.eq( arr[i], doc.b[i] , "after restore array: "+i ); + } } // now with --jsonArray @@ -49,7 +53,11 @@ assert.soon( "c.findOne()" , "no data after sleep" ); assert.eq( 1 , c.count() , "after restore 2" ); var doc = c.findOne(); for (var i=0; i<arr.length; i++) { - assert.eq( arr[i], doc.a[i] , "after restore array: "+i ); + if (typeof arr[i] == 'undefined') { + assert.eq( { "$undefined" : true }, doc.a[i] , "after restore array: "+i ); + } else { + assert.eq( arr[i], doc.a[i] , "after restore array: "+i ); + } } diff --git a/jstests/tool/restorewithauth.js b/jstests/tool/restorewithauth.js index 685301bdd94..15439c4fc03 100644 --- a/jstests/tool/restorewithauth.js +++ b/jstests/tool/restorewithauth.js @@ -23,22 +23,25 @@ var conn = startMongod( "--port", port, "--dbpath", "/data/db/" + baseName, "--n var foo = conn.getDB( "foo" ); for( var i = 0; i < 4; i++ ) { foo["bar"].save( { "x": i } ); + foo["baz"].save({"x": i}); } // make sure the collection exists assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 1 ) //make sure it has no index except _id -assert.eq( foo.system.indexes.count(), 1 ) +assert.eq(foo.system.indexes.count(), 2); + +foo.bar.createIndex({x:1}); +assert.eq(foo.system.indexes.count(), 3); // get data dump var dumpdir = "/data/db/restorewithauth-dump1/"; resetDbpath( dumpdir ); -x = runMongoProgram( "mongodump", "--db", "foo", "-h", "127.0.0.1:"+port, "--collection", "bar", - "--out", dumpdir ); +x = runMongoProgram("mongodump", "--db", "foo", "-h", "127.0.0.1:"+port, "--out", dumpdir); -// now drop the collection -foo.bar.drop(); +// now drop the db +foo.dropDatabase(); // stop mongod stopMongod( port ); @@ -55,22 +58,45 @@ admin.auth( "admin" , "admin" ); var foo = conn.getDB( "foo" ) // make sure no collection with the same name exists -assert.eq( foo.system.namespaces.count( {name: "foo.bar"}), 0 ) +assert.eq(foo.system.namespaces.count( {name: "foo.bar"}), 0); +assert.eq(foo.system.namespaces.count( {name: "foo.baz"}), 0); // now try to restore dump x = runMongoProgram( "mongorestore", "-h", "127.0.0.1:" + port, "--dir" , dumpdir, "-vvvvv" ); // make sure that the collection isn't restored -assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 0 ) +assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 0); +assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 0); // now try to restore dump with correct credentials x = runMongoProgram( "mongorestore", "-h", "127.0.0.1:" + port, "-d", "foo", "-u", "admin", "-p", "admin", "--dir", dumpdir + "foo/", "-vvvvv"); // make sure that the collection was restored -assert.eq( foo.system.namespaces.count({name: "foo.bar"}), 1 ) +assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 1); +assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 1); // make sure the collection has 4 documents -assert.eq( foo.bar.count(), 4 ) +assert.eq(foo.bar.count(), 4); +assert.eq(foo.baz.count(), 4); + +foo.dropDatabase(); + +// make sure that the collection is empty +assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 0); +assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 0); + +foo.addUser('user', 'password'); + +// now try to restore dump with foo database credentials +x = runMongoProgram("mongorestore", "-h", "127.0.0.1:" + port, "-d", "foo", "-u", "user", "-p", + "password", "--dir", dumpdir + "foo/", "-vvvvv"); + +// make sure that the collection was restored +assert.eq(foo.system.namespaces.count({name: "foo.bar"}), 1); +assert.eq(foo.system.namespaces.count({name: "foo.baz"}), 1); +assert.eq(foo.bar.count(), 4); +assert.eq(foo.baz.count(), 4); +assert.eq(foo.system.indexes.count(), 4); // _id on foo, _id on bar, x on foo, _id on system.users stopMongod( port ); diff --git a/rpm/mongo.spec b/rpm/mongo.spec index 5ea653d4694..9f1b49d1113 100644 --- a/rpm/mongo.spec +++ b/rpm/mongo.spec @@ -1,7 +1,7 @@ Name: mongo-10gen Conflicts: mongo, mongo-10gen-unstable Obsoletes: mongo-stable -Version: 2.2.0 +Version: 2.2.2 Release: mongodb_1%{?dist} Summary: mongodb client shell and tools License: AGPL 3.0 diff --git a/site_scons/libdeps.py b/site_scons/libdeps.py index 561b9620244..35ee5b46bde 100644 --- a/site_scons/libdeps.py +++ b/site_scons/libdeps.py @@ -51,6 +51,9 @@ import SCons.Errors import SCons.Scanner import SCons.Util +libdeps_env_var = 'LIBDEPS' +syslibdeps_env_var = 'SYSLIBDEPS' + def sorted_by_str(iterable): """Shorthand for sorting an iterable according to its string representation. @@ -71,19 +74,19 @@ class DependencyCycleError(SCons.Errors.UserError): def __str__(self): return " => ".join(str(n) for n in self.cycle_nodes) -def __get_libdeps(node, env_var): +def __get_libdeps(node): """Given a SCons Node, return its library dependencies. Computes the dependencies if they're not already cached. """ - cached_var_name = env_var + '_cached' + cached_var_name = libdeps_env_var + '_cached' if not hasattr(node.attributes, cached_var_name): - setattr(node.attributes, cached_var_name, __compute_libdeps(node, env_var)) + setattr(node.attributes, cached_var_name, __compute_libdeps(node)) return getattr(node.attributes, cached_var_name) -def __compute_libdeps(node, env_var): +def __compute_libdeps(node): """Recursively identify all library dependencies for a node.""" if getattr(node.attributes, 'libdeps_exploring', False): @@ -94,11 +97,11 @@ def __compute_libdeps(node, env_var): node.attributes.libdeps_exploring = True try: try: - for child in env.Flatten(env.get(env_var, [])): + for child in env.Flatten(env.get(libdeps_env_var, [])): if not child: continue deps.add(child) - deps.update(__get_libdeps(child, env_var)) + deps.update(__get_libdeps(child)) except DependencyCycleError, e: if len(e.cycle_nodes) == 1 or e.cycle_nodes[0] != e.cycle_nodes[-1]: @@ -109,6 +112,20 @@ def __compute_libdeps(node, env_var): return deps +def __get_syslibdeps(node): + """ Given a SCons Node, return its system library dependencies. + + These are the depencencies listed with SYSLIBDEPS, and are linked using -l. + """ + cached_var_name = syslibdeps_env_var + '_cached' + if not hasattr(node.attributes, cached_var_name): + syslibdeps = [] + for lib in __get_libdeps(node): + for syslib in lib.get_env().get(syslibdeps_env_var, []): + syslibdeps.append(syslib) + setattr(node.attributes, cached_var_name, sorted(syslibdeps)) + return getattr(node.attributes, cached_var_name) + def update_scanner(builder): """Update the scanner for "builder" to also scan library dependencies.""" @@ -118,14 +135,12 @@ def update_scanner(builder): path_function = old_scanner.path_function def new_scanner(node, env, path=()): result = set(old_scanner.function(node, env, path)) - result.update(__get_libdeps(node, 'LIBDEPS')) - result.update(__get_libdeps(node, 'SYSLIBDEPS')) + result.update(__get_libdeps(node)) return sorted_by_str(result) else: path_function = None def new_scanner(node, env, path=()): - result = set(__get_libdeps(node, 'LIBDEPS')) - result.update(__get_libdeps(node, 'SYSLIBDEPS')) + result = set(__get_libdeps(node)) return sorted_by_str(result) builder.target_scanner = SCons.Scanner.Scanner(function=new_scanner, @@ -138,21 +153,24 @@ def get_libdeps(source, target, env, for_signature): """ target = env.Flatten([target]) - return list(__get_libdeps(target[0], 'LIBDEPS')) + return sorted_by_str(__get_libdeps(target[0])) def get_libdeps_objs(source, target, env, for_signature): objs = set() for lib in get_libdeps(source, target, env, for_signature): objs.update(lib.sources_set) - return list(objs) + return sorted_by_str(objs) def get_libdeps_special_sun(source, target, env, for_signature): x = get_libdeps(source, target, env, for_signature ) return x + x + x def get_syslibdeps(source, target, env, for_signature): - deps = list(__get_libdeps(target[0], 'SYSLIBDEPS')) - return deps + deps = __get_syslibdeps(target[0]) + lib_link_prefix = env.subst('$LIBLINKPREFIX') + lib_link_suffix = env.subst('$LIBLINKSUFFIX') + result = ''.join([' %s%s%s' % (lib_link_prefix, d, lib_link_suffix) for d in deps]) + return result def libdeps_emitter(target, source, env): """SCons emitter that takes values from the LIBDEPS environment variable and @@ -172,7 +190,7 @@ def libdeps_emitter(target, source, env): libdep_files = [] lib_suffix = env.subst('$LIBSUFFIX', target=target, source=source) lib_prefix = env.subst('$LIBPREFIX', target=target, source=source) - for dep in env.Flatten([env.get('LIBDEPS', [])]): + for dep in env.Flatten([env.get(libdeps_env_var, [])]): full_path = env.subst(str(dep), target=target, source=source) dir_name = os.path.dirname(full_path) file_name = os.path.basename(full_path) @@ -182,7 +200,7 @@ def libdeps_emitter(target, source, env): file_name += '${LIBSUFFIX}' libdep_files.append(env.File(os.path.join(dir_name, file_name))) - env['LIBDEPS'] = libdep_files + env[libdeps_env_var] = libdep_files return target, source @@ -203,11 +221,11 @@ def setup_environment(env): env['_LIBDEPS_LIBS'] = get_libdeps env['_LIBDEPS_OBJS'] = get_libdeps_objs - env['_SYSLIBDEPS'] = ' ${_stripixes(LIBLINKPREFIX, SYSLIBDEPS, LIBLINKSUFFIX, LIBPREFIXES, LIBSUFFIXES, __env__)} ' + env['_SYSLIBDEPS'] = get_syslibdeps env['_SHLIBDEPS'] = '$SHLIBDEP_GROUP_START ${_concat(SHLIBDEPPREFIX, __env__.subst(_LIBDEPS, target=TARGET, source=SOURCE), SHLIBDEPSUFFIX, __env__, target=TARGET, source=SOURCE)} $SHLIBDEP_GROUP_END' - env['LIBDEPS'] = SCons.Util.CLVar() - env['SYSLIBDEPS'] = SCons.Util.CLVar() + env[libdeps_env_var] = SCons.Util.CLVar() + env[syslibdeps_env_var] = SCons.Util.CLVar() env.Append(LIBEMITTER=libdeps_emitter, PROGEMITTER=libdeps_emitter, SHLIBEMITTER=libdeps_emitter) diff --git a/src/mongo/bson/util/builder.h b/src/mongo/bson/util/builder.h index b8027e561a2..9b512dba8b3 100644 --- a/src/mongo/bson/util/builder.h +++ b/src/mongo/bson/util/builder.h @@ -247,13 +247,10 @@ namespace mongo { void decouple(); // not allowed. not implemented. }; - namespace { #if defined(_WIN32) - int (*mongo_snprintf)(char *str, size_t size, const char *format, ...) = &sprintf_s; -#else - int (*mongo_snprintf)(char *str, size_t size, const char *format, ...) = &snprintf; +#pragma push_macro("snprintf") +#define snprintf _snprintf #endif - } /** stringstream deals with locale so this is a lot faster than std::stringstream for UTF8 */ template <typename Allocator> @@ -301,7 +298,7 @@ namespace mongo { const int prev = _buf.l; const int maxSize = 32; char * start = _buf.grow( maxSize ); - int z = mongo_snprintf( start , maxSize , "%.16g" , x ); + int z = snprintf( start , maxSize , "%.16g" , x ); verify( z >= 0 ); verify( z < maxSize ); _buf.l = prev + z; @@ -335,7 +332,7 @@ namespace mongo { template <typename T> StringBuilderImpl& SBNUM(T val,int maxSize,const char *macro) { int prev = _buf.l; - int z = mongo_snprintf( _buf.grow(maxSize) , maxSize , macro , (val) ); + int z = snprintf( _buf.grow(maxSize) , maxSize , macro , (val) ); verify( z >= 0 ); verify( z < maxSize ); _buf.l = prev + z; @@ -346,4 +343,8 @@ namespace mongo { typedef StringBuilderImpl<TrivialAllocator> StringBuilder; typedef StringBuilderImpl<StackAllocator> StackStringBuilder; +#if defined(_WIN32) +#undef snprintf +#pragma pop_macro("snprintf") +#endif } // namespace mongo diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index 5af4c623982..1177cfd1172 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -36,6 +36,8 @@ namespace mongo { + AtomicInt64 DBClientBase::ConnectionIdSequence; + void ConnectionString::_fillServers( string s ) { // @@ -389,6 +391,14 @@ namespace mongo { } BSONObj DBClientWithCommands::getLastErrorDetailed(bool fsync, bool j, int w, int wtimeout) { + return getLastErrorDetailed("admin", fsync, j, w, wtimeout); + } + + BSONObj DBClientWithCommands::getLastErrorDetailed(const std::string& db, + bool fsync, + bool j, + int w, + int wtimeout) { BSONObj info; BSONObjBuilder b; b.append( "getlasterror", 1 ); @@ -407,13 +417,21 @@ namespace mongo { if ( wtimeout > 0 ) b.append( "wtimeout", wtimeout ); - runCommand("admin", b.obj(), info); + runCommand(db, b.obj(), info); return info; } string DBClientWithCommands::getLastError(bool fsync, bool j, int w, int wtimeout) { - BSONObj info = getLastErrorDetailed(fsync, j, w, wtimeout); + return getLastError("admin", fsync, j, w, wtimeout); + } + + string DBClientWithCommands::getLastError(const std::string& db, + bool fsync, + bool j, + int w, + int wtimeout) { + BSONObj info = getLastErrorDetailed(db, fsync, j, w, wtimeout); return getLastErrorString( info ); } diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 8e00958403a..437f75bcf6a 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -46,15 +46,19 @@ namespace mongo { * @param lastHost the last host returned (mainly used for doing round-robin). * Will be overwritten with the newly returned host if not empty. Should * never be NULL. + * @param isPrimarySelected out parameter that is set to true if the returned host + * is a primary. * * @return the host object of the node selected. If none of the nodes are - * eligible, returns an empty host. + * eligible, returns an empty host. Cannot be NULL and valid only if returned + * host is not empty. */ HostAndPort _selectNode(const vector<ReplicaSetMonitor::Node>& nodes, const BSONObj& readPreferenceTag, bool secOnly, int localThresholdMillis, - HostAndPort* lastHost /* in/out */) { + HostAndPort* lastHost /* in/out */, + bool* isPrimarySelected) { HostAndPort fallbackHost; // Implicit: start from index 0 if lastHost doesn't exist anymore @@ -85,6 +89,7 @@ namespace mongo { if (node.matchesTag(readPreferenceTag)) { // found an ok candidate; may not be local. fallbackHost = node.addr; + *isPrimarySelected = node.ismaster; if (node.isLocalSecondary(localThresholdMillis)) { // found a local node. return early. @@ -1006,14 +1011,15 @@ namespace mongo { } HostAndPort ReplicaSetMonitor::selectAndCheckNode(ReadPreference preference, - TagSet* tags) { + TagSet* tags, + bool* isPrimarySelected) { HostAndPort candidate; { scoped_lock lk(_lock); candidate = ReplicaSetMonitor::selectNode(_nodes, preference, tags, - _localThresholdMillis, &_lastReadPrefHost); + _localThresholdMillis, &_lastReadPrefHost, isPrimarySelected); } if (candidate.empty()) { @@ -1022,7 +1028,7 @@ namespace mongo { scoped_lock lk(_lock); return ReplicaSetMonitor::selectNode(_nodes, preference, tags, _localThresholdMillis, - &_lastReadPrefHost); + &_lastReadPrefHost, isPrimarySelected); } return candidate; @@ -1033,11 +1039,15 @@ namespace mongo { ReadPreference preference, TagSet* tags, int localThresholdMillis, - HostAndPort* lastHost) { + HostAndPort* lastHost, + bool* isPrimarySelected) { + *isPrimarySelected = false; + switch (preference) { case ReadPreference_PrimaryOnly: for (vector<Node>::const_iterator iter = nodes.begin(); iter != nodes.end(); ++iter) { if (iter->ismaster && iter->ok) { + *isPrimarySelected = true; return iter->addr; } } @@ -1047,14 +1057,14 @@ namespace mongo { case ReadPreference_PrimaryPreferred: { HostAndPort candidatePri = selectNode(nodes, ReadPreference_PrimaryOnly, tags, - localThresholdMillis, lastHost); + localThresholdMillis, lastHost, isPrimarySelected); if (!candidatePri.empty()) { return candidatePri; } return selectNode(nodes, ReadPreference_SecondaryOnly, tags, - localThresholdMillis, lastHost); + localThresholdMillis, lastHost, isPrimarySelected); } case ReadPreference_SecondaryOnly: @@ -1063,7 +1073,7 @@ namespace mongo { while (!tags->isExhausted()) { candidate = _selectNode(nodes, tags->getCurrentTag(), true, localThresholdMillis, - lastHost); + lastHost, isPrimarySelected); if (candidate.empty()) { tags->next(); @@ -1079,14 +1089,14 @@ namespace mongo { case ReadPreference_SecondaryPreferred: { HostAndPort candidateSec = selectNode(nodes, ReadPreference_SecondaryOnly, tags, - localThresholdMillis, lastHost); + localThresholdMillis, lastHost, isPrimarySelected); if (!candidateSec.empty()) { return candidateSec; } return selectNode(nodes, ReadPreference_PrimaryOnly, tags, - localThresholdMillis, lastHost); + localThresholdMillis, lastHost, isPrimarySelected); } case ReadPreference_Nearest: @@ -1095,7 +1105,7 @@ namespace mongo { while (!tags->isExhausted()) { candidate = _selectNode(nodes, tags->getCurrentTag(), false, localThresholdMillis, - lastHost); + lastHost, isPrimarySelected); if (candidate.empty()) { tags->next(); @@ -1163,6 +1173,19 @@ namespace mongo { _check(true); } + bool ReplicaSetMonitor::isAnyNodeOk() const { + scoped_lock lock(_lock); + + for (vector<Node>::const_iterator iter = _nodes.begin(); + iter != _nodes.end(); ++iter) { + if (iter->ok) { + return true; + } + } + + return false; + } + bool ReplicaSetMonitor::Node::matchesTag(const BSONObj& tag) const { if (tag.isEmpty()) { return true; @@ -1347,19 +1370,7 @@ namespace mongo { } bool DBClientReplicaSet::connect() { - try { - checkMaster(); - } - catch (AssertionException&) { - // Can't use _getMonitor because that will create a new monitor from the cached seed if - // the monitor doesn't exist. - ReplicaSetMonitorPtr monitor = ReplicaSetMonitor::get(_setName); - if (_master && monitor ) { - monitor->notifyFailure(_masterHost); - } - return false; - } - return true; + return _getMonitor()->isAnyNodeOk(); } bool DBClientReplicaSet::auth(const string &dbname, const string &username, const string &pwd, string& errmsg, bool digestPassword, Auth::Level * level) { @@ -1558,12 +1569,24 @@ namespace mongo { } ReplicaSetMonitorPtr monitor = _getMonitor(); - _lastSlaveOkHost = monitor->selectAndCheckNode(preference, tags); + bool isPrimarySelected = false; + _lastSlaveOkHost = monitor->selectAndCheckNode(preference, tags, &isPrimarySelected); if ( _lastSlaveOkHost.empty() ){ return NULL; } + // Primary connection is special because it is the only connection that is + // versioned in mongos. Therefore, we have to make sure that this object + // maintains only one connection to the primary and use that connection + // every time we need to talk to the primary. + if (isPrimarySelected) { + checkMaster(); + _lastSlaveOkConn = _master; + _lastSlaveOkHost = _masterHost; // implied, but still assign just to be safe + return _master.get(); + } + _lastSlaveOkConn.reset(new DBClientConnection(true , this , _so_timeout)); _lastSlaveOkConn->connect(_lastSlaveOkHost); diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index a74fa7902a3..d8a52382512 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -147,6 +147,8 @@ namespace mongo { * robin, starting from the node next to this lastHost. This will be overwritten * with the newly chosen host if not empty, not primary and when preference * is not Nearest. + * @param isPrimarySelected out parameter that is set to true if the returned host + * is a primary. Cannot be NULL and valid only if returned host is not empty. * * @return the host object of the node selected. If none of the nodes are * eligible, returns an empty host. @@ -155,7 +157,8 @@ namespace mongo { ReadPreference preference, TagSet* tags, int localThresholdMillis, - HostAndPort* lastHost); + HostAndPort* lastHost, + bool* isPrimarySelected); /** * Selects the right node given the nodes to pick from and the preference. This @@ -163,14 +166,17 @@ namespace mongo { * if the primary node needs to be returned but is not currently available (except * for ReadPrefrence_Nearest). * - * @param preference the read mode to use - * @param tags the tags used for filtering nodes + * @param preference the read mode to use. + * @param tags the tags used for filtering nodes. + * @param isPrimarySelected out parameter that is set to true if the returned host + * is a primary. Cannot be NULL and valid only if returned host is not empty. * * @return the host object of the node selected. If none of the nodes are * eligible, returns an empty host. */ HostAndPort selectAndCheckNode(ReadPreference preference, - TagSet* tags); + TagSet* tags, + bool* isPrimarySelected); /** * Creates a new ReplicaSetMonitor, if it doesn't already exist. @@ -262,6 +268,14 @@ namespace mongo { bool isHostCompatible(const HostAndPort& host, ReadPreference readPreference, const TagSet* tagSet) const; + /** + * Performs a quick check if at least one node is up based on the cached + * view of the set. + * + * @return true if any node is ok + */ + bool isAnyNodeOk() const; + private: /** * This populates a list of hosts from the list of seeds (discarding the @@ -405,10 +419,10 @@ namespace mongo { DBClientReplicaSet( const string& name , const vector<HostAndPort>& servers, double so_timeout=0 ); virtual ~DBClientReplicaSet(); - /** Returns false if nomember of the set were reachable, or neither is - * master, although, - * when false returned, you can still try to use this connection object, it will - * try reconnects. + /** + * Returns false if no member of the set were reachable. This object + * can still be used even when false was returned as it will try to + * reconnect when you use it later. */ bool connect(); @@ -448,6 +462,12 @@ namespace mongo { // ---- access raw connections ---- + /** + * WARNING: this method is very dangerous - this object can decide to free the + * returned master connection any time. + * + * @return the reference to the address that points to the master connection. + */ DBClientConnection& masterConn(); DBClientConnection& slaveConn(); @@ -542,12 +562,18 @@ namespace mongo { string _setName; HostAndPort _masterHost; - scoped_ptr<DBClientConnection> _master; + // Note: reason why this is a shared_ptr is because we want _lastSlaveOkConn to + // keep a reference of the _master connection when it selected a primary node. + // This is because the primary connection is special in mongos - it is the only + // connection that is versioned. + // WARNING: do not assign this variable (which will increment the internal ref + // counter) to any other variable other than _lastSlaveOkConn. + boost::shared_ptr<DBClientConnection> _master; // Last used host in a slaveOk query (can be a primary) HostAndPort _lastSlaveOkHost; // Last used connection in a slaveOk query (can be a primary) - scoped_ptr<DBClientConnection> _lastSlaveOkConn; + boost::shared_ptr<DBClientConnection> _lastSlaveOkConn; double _so_timeout; diff --git a/src/mongo/client/dbclientcursor.cpp b/src/mongo/client/dbclientcursor.cpp index e7e67a23a60..d0469839ae5 100644 --- a/src/mongo/client/dbclientcursor.cpp +++ b/src/mongo/client/dbclientcursor.cpp @@ -157,10 +157,11 @@ namespace mongo { verify( !haveLimit ); auto_ptr<Message> response(new Message()); verify( _client ); - if ( _client->recv(*response) ) { - batch.m = response; - dataReceived(); + if (!_client->recv(*response)) { + uasserted(16465, "recv failed while exhausting cursor"); } + batch.m = response; + dataReceived(); } void DBClientCursor::dataReceived( bool& retry, string& host ) { diff --git a/src/mongo/client/dbclientinterface.h b/src/mongo/client/dbclientinterface.h index 75dba2afcfc..e615e6abbaf 100644 --- a/src/mongo/client/dbclientinterface.h +++ b/src/mongo/client/dbclientinterface.h @@ -25,6 +25,7 @@ #include "mongo/client/authlevel.h" #include "mongo/client/authentication_table.h" #include "mongo/db/jsobj.h" +#include "mongo/platform/atomic_word.h" #include "mongo/util/net/message.h" #include "mongo/util/net/message_port.h" @@ -631,16 +632,30 @@ namespace mongo { bool createCollection(const string &ns, long long size = 0, bool capped = false, int max = 0, BSONObj *info = 0); /** Get error result from the last write operation (insert/update/delete) on this connection. + db doesn't change the command's behavior - it is just for auth checks. @return error message text, or empty string if no error. */ + string getLastError(const std::string& db, + bool fsync = false, + bool j = false, + int w = 0, + int wtimeout = 0); + // Same as above but defaults to using admin DB string getLastError(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0); /** Get error result from the last write operation (insert/update/delete) on this connection. + db doesn't change the command's behavior - it is just for auth checks. @return full error object. If "w" is -1, wait for propagation to majority of nodes. If "wtimeout" is 0, the operation will block indefinitely if needed. */ + virtual BSONObj getLastErrorDetailed(const std::string& db, + bool fsync = false, + bool j = false, + int w = 0, + int wtimeout = 0); + // Same as above but defaults to using admin DB virtual BSONObj getLastErrorDetailed(bool fsync = false, bool j = false, int w = 0, int wtimeout = 0); /** Can be called with the returned value from getLastErrorDetailed to extract an error string. @@ -907,13 +922,17 @@ namespace mongo { */ class DBClientBase : public DBClientWithCommands, public DBConnector { protected: + static AtomicInt64 ConnectionIdSequence; + long long _connectionId; // unique connection id for this connection WriteConcern _writeConcern; - public: DBClientBase() { _writeConcern = W_NORMAL; + _connectionId = ConnectionIdSequence.fetchAndAdd(1); } + long long getConnectionId() const { return _connectionId; } + WriteConcern getWriteConcern() const { return _writeConcern; } void setWriteConcern( WriteConcern w ) { _writeConcern = w; } diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index a2461ed7ee3..83178dfcb95 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -1127,9 +1127,26 @@ namespace mongo { throw; } catch( DBException& e ){ - warning() << "db exception when finishing on " << shard << ", current connection state is " << mdata.toBSON() << causedBy( e ) << endl; - mdata.errored = true; - throw; + // NOTE: RECV() WILL NOT THROW A SOCKET EXCEPTION - WE GET THIS AS ERROR 15988 FROM + // ABOVE + if (e.getCode() == 15988) { + + warning() << "exception when receiving data from " << shard + << ", current connection state is " << mdata.toBSON() + << causedBy( e ) << endl; + + mdata.errored = true; + if (returnPartial) { + mdata.cleanup(); + continue; + } + throw; + } + else { + warning() << "db exception when finishing on " << shard << ", current connection state is " << mdata.toBSON() << causedBy( e ) << endl; + mdata.errored = true; + throw; + } } catch( std::exception& e){ warning() << "exception when finishing on " << shard << ", current connection state is " << mdata.toBSON() << causedBy( e ) << endl; @@ -1228,6 +1245,7 @@ namespace mongo { if( ! isVersioned() ) return false; if( _cursorMap.size() > 1 ) return true; + if( _cursorMap.size() == 0 ) return true; if( _cursorMap.begin()->second.pcState->manager ) return true; return false; } diff --git a/src/mongo/client/syncclusterconnection.cpp b/src/mongo/client/syncclusterconnection.cpp index d09c4be3d02..b674a6a9cce 100644 --- a/src/mongo/client/syncclusterconnection.cpp +++ b/src/mongo/client/syncclusterconnection.cpp @@ -141,9 +141,17 @@ namespace mongo { } BSONObj SyncClusterConnection::getLastErrorDetailed(bool fsync, bool j, int w, int wtimeout) { + return getLastErrorDetailed("admin", fsync, j, w, wtimeout); + } + + BSONObj SyncClusterConnection::getLastErrorDetailed(const std::string& db, + bool fsync, + bool j, + int w, + int wtimeout) { if ( _lastErrors.size() ) return _lastErrors[0]; - return DBClientBase::getLastErrorDetailed(fsync,j,w,wtimeout); + return DBClientBase::getLastErrorDetailed(db,fsync,j,w,wtimeout); } void SyncClusterConnection::_connect( string host ) { diff --git a/src/mongo/client/syncclusterconnection.h b/src/mongo/client/syncclusterconnection.h index 3907f7e50a0..b1d7439e020 100644 --- a/src/mongo/client/syncclusterconnection.h +++ b/src/mongo/client/syncclusterconnection.h @@ -91,6 +91,11 @@ namespace mongo { virtual bool isFailed() const { return false; } virtual string toString() { return _toString(); } + virtual BSONObj getLastErrorDetailed(const std::string& db, + bool fsync=false, + bool j=false, + int w=0, + int wtimeout=0); virtual BSONObj getLastErrorDetailed(bool fsync=false, bool j=false, int w=0, int wtimeout=0); virtual bool callRead( Message& toSend , Message& response ); diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index 894927f3c86..905456047af 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -792,7 +792,7 @@ namespace mongo { if ( ! cc().getAuthenticationInfo()->isAuthorizedReads( nsToDatabase( cursor->ns() ) ) ) return false; - // mustn't have an active ClientCursor::Pointer + // Must not have an active ClientCursor::Pin. massert( 16089, str::stream() << "Cannot kill active cursor " << id, cursor->_pinValue < 100 ); diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 9b478170e5c..aebd23d4607 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -100,6 +100,9 @@ namespace mongo { } } void release() { + if ( _cursorid == INVALID_CURSOR_ID ) { + return; + } ClientCursor *cursor = c(); _cursorid = INVALID_CURSOR_ID; if ( cursor ) { diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp index 2923dbb5c4b..0ad78239de2 100644 --- a/src/mongo/db/commands/find_and_modify.cpp +++ b/src/mongo/db/commands/find_and_modify.cpp @@ -78,7 +78,7 @@ namespace mongo { return runNoDirectClient( ns , query , fields , update , upsert , returnNew , remove , - result ); + result , errmsg ); } catch ( PageFaultException& e ) { e.touch(); @@ -108,7 +108,7 @@ namespace mongo { bool runNoDirectClient( const string& ns , const BSONObj& queryOriginal , const BSONObj& fields , const BSONObj& update , bool upsert , bool returnNew , bool remove , - BSONObjBuilder& result ) { + BSONObjBuilder& result , string& errmsg ) { Lock::DBWrite lk( ns ); @@ -123,6 +123,30 @@ namespace mongo { // we're going to re-write the query to be more efficient // we have to be a little careful because of positional operators // maybe we can pass this all through eventually, but right now isn't an easy way + + bool hasPositionalUpdate = false; + { + // if the update has a positional piece ($) + // then we need to pull all query parts in + // so here we check for $ + // a little hacky + BSONObjIterator i( update ); + while ( i.more() ) { + const BSONElement& elem = i.next(); + + if ( elem.fieldName()[0] != '$' || elem.type() != Object ) + continue; + + BSONObjIterator j( elem.Obj() ); + while ( j.more() ) { + if ( str::contains( j.next().fieldName(), ".$" ) ) { + hasPositionalUpdate = true; + break; + } + } + } + } + BSONObjBuilder b( queryOriginal.objsize() + 10 ); b.append( doc["_id"] ); @@ -137,7 +161,7 @@ namespace mongo { continue; } - if ( ! str::contains( elem.fieldName() , '.' ) ) { + if ( ! hasPositionalUpdate ) { // if there is a dotted field, accept we may need more query parts continue; } @@ -177,11 +201,21 @@ namespace mongo { UpdateResult res = updateObjects( ns.c_str() , update , queryModified , upsert , false , true , cc().curop()->debug() ); if ( returnNew ) { - if ( ! res.existing && res.upserted.isSet() ) { + if ( res.upserted.isSet() ) { queryModified = BSON( "_id" << res.upserted ); } - log() << "queryModified: " << queryModified << endl; - verify( Helpers::findOne( ns.c_str() , queryModified , doc ) ); + else if ( queryModified["_id"].type() ) { + // we do this so that if the update changes the fields, it still matches + queryModified = queryModified["_id"].wrap(); + } + if ( ! Helpers::findOne( ns.c_str() , queryModified , doc ) ) { + errmsg = str::stream() << "can't find object after modification " + << " ns: " << ns + << " queryModified: " << queryModified + << " queryOriginal: " << queryOriginal; + log() << errmsg << endl; + return false; + } _appendHelper( result , doc , true , fields ); } diff --git a/src/mongo/db/compact.cpp b/src/mongo/db/compact.cpp index 4c258ef62a7..847a77b01e0 100644 --- a/src/mongo/db/compact.cpp +++ b/src/mongo/db/compact.cpp @@ -63,7 +63,7 @@ namespace mongo { Extent *e = diskloc.ext(); e->assertOk(); - verify( e->validates() ); + verify( e->validates(diskloc) ); unsigned skipped = 0; { diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 6a80fe9e8b4..3c1daa278b9 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -1290,9 +1290,8 @@ namespace mongo { return TRUE; case CTRL_LOGOFF_EVENT: - rawOut( "CTRL_LOGOFF_EVENT signal" ); - consoleTerminate( "CTRL_LOGOFF_EVENT" ); - return TRUE; + // only sent to services, and only in pre-Vista Windows; FALSE means ignore + return FALSE; case CTRL_SHUTDOWN_EVENT: rawOut( "CTRL_SHUTDOWN_EVENT signal" ); @@ -1384,14 +1383,11 @@ namespace mongo { printWindowsStackTrace( *excPointers->ContextRecord ); doMinidump(excPointers); - // In release builds, let dbexit() try to shut down cleanly -#if !defined(_DEBUG) - dbexit( EXIT_UNCAUGHT, "unhandled exception" ); -#endif + // Don't go through normal shutdown procedure. It may make things worse. + log() << "*** immediate exit due to unhandled exception" << endl; + ::_exit(EXIT_ABRUPT); - // In debug builds, give debugger a chance to run - if( filtLast ) - return filtLast( excPointers ); + // We won't reach here return EXCEPTION_EXECUTE_HANDLER; } diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index a7fcf760173..d448c9aecb0 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -1909,12 +1909,11 @@ namespace mongo { BSONObj authObj = cmdObj[AuthenticationTable::fieldName].Obj(); ai->setTemporaryAuthorization( authObj ); } else { - result.append( "errmsg" , - "unauthorized: no auth credentials provided for command and " - "authenticated using internal user. This is most likely because " - "you are using an old version of mongos" ); - log() << "command denied: " << cmdObj.toString() << endl; - return false; + SOMETIMES ( noAuthTableCounter, 1000 ) { + warning() << "Received command without $auth table. This is probably because " + "you are running with 1 or more mongod or mongos nodes that are running a " + "version prior to 2.2. Command object: " << cmdObj.toString() << endl; + } } } diff --git a/src/mongo/db/dbcommands_admin.cpp b/src/mongo/db/dbcommands_admin.cpp index b1b46e8c0b0..804926f43c5 100644 --- a/src/mongo/db/dbcommands_admin.cpp +++ b/src/mongo/db/dbcommands_admin.cpp @@ -178,7 +178,10 @@ namespace mongo { } private: - void validateNS(const char *ns, NamespaceDetails *d, const BSONObj& cmdObj, BSONObjBuilder& result) { + void validateNS(const char *ns, + NamespaceDetails *d, + const BSONObj& cmdObj, + BSONObjBuilder& result) { const bool full = cmdObj["full"].trueValue(); const bool scanData = full || cmdObj["scandata"].trueValue(); @@ -189,54 +192,109 @@ namespace mongo { result.appendNumber("max", d->maxCappedDocs()); } - result.append("firstExtent", str::stream() << d->firstExtent.toString() << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString()); - result.append( "lastExtent", str::stream() << d->lastExtent.toString() << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString()); - - BSONArrayBuilder extentData; + result.append("firstExtent", str::stream() << d->firstExtent.toString() + << " ns:" << d->firstExtent.ext()->nsDiagnostic.toString()); + result.append( "lastExtent", str::stream() << d->lastExtent.toString() + << " ns:" << d->lastExtent.ext()->nsDiagnostic.toString()); + BSONArrayBuilder extentData; + int extentCount = 0; try { d->firstExtent.ext()->assertOk(); d->lastExtent.ext()->assertOk(); - DiskLoc el = d->firstExtent; - int ne = 0; - while( !el.isNull() ) { - Extent *e = el.ext(); - e->assertOk(); - el = e->xnext; - ne++; - if ( full ) - extentData << e->dump(); - + DiskLoc extentDiskLoc = d->firstExtent; + while (!extentDiskLoc.isNull()) { + Extent* thisExtent = extentDiskLoc.ext(); + if (full) { + extentData << thisExtent->dump(); + } + if (!thisExtent->validates(extentDiskLoc, &errors)) { + valid = false; + } + DiskLoc nextDiskLoc = thisExtent->xnext; + if (extentCount > 0 && !nextDiskLoc.isNull() + && nextDiskLoc.ext()->xprev != extentDiskLoc) { + StringBuilder sb; + sb << "'xprev' pointer " << nextDiskLoc.ext()->xprev.toString() + << " in extent " << nextDiskLoc.toString() + << " does not point to extent " << extentDiskLoc.toString(); + errors << sb.str(); + valid = false; + } + if (nextDiskLoc.isNull() && extentDiskLoc != d->lastExtent) { + StringBuilder sb; + sb << "'lastExtent' pointer " << d->lastExtent.toString() + << " does not point to last extent in list " << extentDiskLoc.toString(); + errors << sb.str(); + valid = false; + } + extentDiskLoc = nextDiskLoc; + extentCount++; killCurrentOp.checkForInterrupt(); } - result.append("extentCount", ne); } - catch (...) { - valid=false; - errors << "extent asserted"; + catch (const DBException& e) { + StringBuilder sb; + sb << "exception validating extent " << extentCount + << ": " << e.what(); + errors << sb.str(); + valid = false; } + result.append("extentCount", extentCount); if ( full ) result.appendArray( "extents" , extentData.arr() ); - result.appendNumber("datasize", d->stats.datasize); result.appendNumber("nrecords", d->stats.nrecords); result.appendNumber("lastExtentSize", d->lastExtentSize); result.appendNumber("padding", d->paddingFactor()); - try { + bool testingLastExtent = false; try { - result.append("firstExtentDetails", d->firstExtent.ext()->dump()); - - valid = valid && d->firstExtent.ext()->validates() && - d->firstExtent.ext()->xprev.isNull(); + if (d->firstExtent.isNull()) { + errors << "'firstExtent' pointer is null"; + valid=false; + } + else { + result.append("firstExtentDetails", d->firstExtent.ext()->dump()); + if (!d->firstExtent.ext()->xprev.isNull()) { + StringBuilder sb; + sb << "'xprev' pointer in 'firstExtent' " << d->firstExtent.toString() + << " is " << d->firstExtent.ext()->xprev.toString() + << ", should be null"; + errors << sb.str(); + valid=false; + } + } + testingLastExtent = true; + if (d->lastExtent.isNull()) { + errors << "'lastExtent' pointer is null"; + valid=false; + } + else { + if (d->firstExtent != d->lastExtent) { + result.append("lastExtentDetails", d->lastExtent.ext()->dump()); + if (!d->lastExtent.ext()->xnext.isNull()) { + StringBuilder sb; + sb << "'xnext' pointer in 'lastExtent' " << d->lastExtent.toString() + << " is " << d->lastExtent.ext()->xnext.toString() + << ", should be null"; + errors << sb.str(); + valid = false; + } + } + } } - catch (...) { - errors << "exception firstextent"; + catch (const DBException& e) { + StringBuilder sb; + sb << "exception processing '" + << (testingLastExtent ? "lastExtent" : "firstExtent") + << "': " << e.what(); + errors << sb.str(); valid = false; } diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 6cf21f7997d..bb26d132141 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -268,7 +268,12 @@ namespace mongo { bool secondaryThrottle , RemoveCallback * callback, bool fromMigrate ) { - + + Timer rangeRemoveTimer; + + LOG(1) << "begin removal of " << min << " to " << max << " in " << ns + << (secondaryThrottle ? " (waiting for secondaries)" : "" ) << endl; + Client& c = cc(); long long numDeleted = 0; @@ -349,6 +354,9 @@ namespace mongo { log() << "Helpers::removeRangeUnlocked time spent waiting for replication: " << millisWaitingForReplication << "ms" << endl; + LOG(1) << "end removal of " << min << " to " << max << " in " << ns + << " (took " << rangeRemoveTimer.millis() << "ms)" << endl; + return numDeleted; } diff --git a/src/mongo/db/dbwebserver.cpp b/src/mongo/db/dbwebserver.cpp index dac1927f111..f427d24cf80 100644 --- a/src/mongo/db/dbwebserver.cpp +++ b/src/mongo/db/dbwebserver.cpp @@ -91,7 +91,7 @@ namespace mongo { pcrecpp::StringPiece input( auth ); string name, val; - pcrecpp::RE re("(\\w+)=\"?(.*?)\"?, "); + pcrecpp::RE re("(\\w+)=\"?(.*?)\"?,\\s*"); while ( re.Consume( &input, &name, &val) ) { parms[name] = val; } diff --git a/src/mongo/db/dur.cpp b/src/mongo/db/dur.cpp index 978c4bea9eb..0b6bf376db7 100644 --- a/src/mongo/db/dur.cpp +++ b/src/mongo/db/dur.cpp @@ -262,42 +262,51 @@ namespace mongo { } bool NOINLINE_DECL DurableImpl::_aCommitIsNeeded() { - if( !Lock::isLocked() ) { - DEV log() << "commitIfNeeded but we are unlocked that is ok but why do we get here" << endl; - Lock::GlobalRead r; - if( commitJob.bytes() < UncommittedBytesLimit ) { - // someone else beat us to it - return false; - } - commitNow(); - } - else if( Lock::isLocked() == 'w' ) { - if( Lock::atLeastReadLocked("local") ) { - error() << "can't commitNow from commitIfNeeded, as we are in local db lock" << endl; - printStackTrace(); - dassert(false); // this will make _DEBUG builds terminate. so we will notice in buildbot. - return false; - } - else if( Lock::atLeastReadLocked("admin") ) { - error() << "can't commitNow from commitIfNeeded, as we are in admin db lock" << endl; - printStackTrace(); - dassert(false); - return false; + switch (Lock::isLocked()) { + case '\0': { + DEV log() << "commitIfNeeded but we are unlocked that is ok but why do we get here" << endl; + Lock::GlobalRead r; + if( commitJob.bytes() < UncommittedBytesLimit ) { + // someone else beat us to it + return false; + } + commitNow(); + return true; } - else { + case 'w': { + if( Lock::atLeastReadLocked("local") ) { + error() << "can't commitNow from commitIfNeeded, as we are in local db lock" << endl; + printStackTrace(); + dassert(false); // this will make _DEBUG builds terminate. so we will notice in buildbot. + return false; + } + if( Lock::atLeastReadLocked("admin") ) { + error() << "can't commitNow from commitIfNeeded, as we are in admin db lock" << endl; + printStackTrace(); + dassert(false); + return false; + } + log(1) << "commitIfNeeded upgrading from shared write to exclusive write state" << endl; Lock::DBWrite::UpgradeToExclusive ex; if (ex.gotUpgrade()) { commitNow(); } + return true; } + + case 'W': + case 'R': + commitNow(); + return true; + + case 'r': + return false; + + default: + fassertFailed(16434); // unknown lock type } - else { - // 'W' - commitNow(); - } - return true; } /** we may need to commit earlier than normal if data are being written at diff --git a/src/mongo/db/dur_recover.cpp b/src/mongo/db/dur_recover.cpp index b3f4ce986a2..a7efd4ddb65 100644 --- a/src/mongo/db/dur_recover.cpp +++ b/src/mongo/db/dur_recover.cpp @@ -214,34 +214,9 @@ namespace mongo { _mmfs.clear(); } - void RecoveryJob::write(const ParsedJournalEntry& entry) { + void RecoveryJob::write(const ParsedJournalEntry& entry, MongoMMF* mmf) { //TODO(mathias): look into making some of these dasserts verify(entry.e); - verify(entry.dbName); - verify(strnlen(entry.dbName, MaxDatabaseNameLen) < MaxDatabaseNameLen); - - const string fn = fileName(entry.dbName, entry.e->getFileNo()); - MongoFile* file; - { - MongoFileFinder finder; // must release lock before creating new MongoMMF - file = finder.findByPath(fn); - } - - MongoMMF* mmf; - if (file) { - verify(file->isMongoMMF()); - mmf = (MongoMMF*)file; - } - else { - if( !_recovering ) { - log() << "journal error applying writes, file " << fn << " is not open" << endl; - verify(false); - } - boost::shared_ptr<MongoMMF> sp (new MongoMMF); - verify(sp->open(fn, false)); - _mmfs.push_back(sp); - mmf = sp.get(); - } if ((entry.e->ofs + entry.e->len) <= mmf->length()) { verify(mmf->view_write()); @@ -256,7 +231,7 @@ namespace mongo { } } - void RecoveryJob::applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump) { + void RecoveryJob::applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump, MongoMMF* mmf) { if( entry.e ) { if( dump ) { stringstream ss; @@ -270,7 +245,7 @@ namespace mongo { log() << ss.str() << endl; } if( apply ) { - write(entry); + write(entry, mmf); } } else if(entry.op) { @@ -287,14 +262,54 @@ namespace mongo { } } + MongoMMF* RecoveryJob::getMongoMMF(const ParsedJournalEntry& entry) { + verify(entry.dbName); + verify(strnlen(entry.dbName, MaxDatabaseNameLen) < MaxDatabaseNameLen); + + const string fn = fileName(entry.dbName, entry.e->getFileNo()); + MongoFile* file; + { + MongoFileFinder finder; // must release lock before creating new MongoMMF + file = finder.findByPath(fn); + } + + MongoMMF* mmf; + if (file) { + verify(file->isMongoMMF()); + mmf = (MongoMMF*)file; + } + else { + if( !_recovering ) { + log() << "journal error applying writes, file " << fn << " is not open" << endl; + verify(false); + } + boost::shared_ptr<MongoMMF> sp (new MongoMMF); + verify(sp->open(fn, false)); + _mmfs.push_back(sp); + mmf = sp.get(); + } + + return mmf; + } + void RecoveryJob::applyEntries(const vector<ParsedJournalEntry> &entries) { bool apply = (cmdLine.durOptions & CmdLine::DurScanOnly) == 0; bool dump = cmdLine.durOptions & CmdLine::DurDumpJournal; if( dump ) log() << "BEGIN section" << endl; + const char* lastDbName = NULL; + int lastFileNo = 0; + MongoMMF* mmf = NULL; for( vector<ParsedJournalEntry>::const_iterator i = entries.begin(); i != entries.end(); ++i ) { - applyEntry(*i, apply, dump); + if (i->e && (i->dbName != lastDbName || i->e->getFileNo() != lastFileNo)) { + mmf = getMongoMMF(*i); + lastDbName = i->dbName; + lastFileNo = i->e->getFileNo(); + } + fassert(16429, !i->e || mmf); + + applyEntry(*i, apply, dump, mmf); } if( dump ) diff --git a/src/mongo/db/dur_recover.h b/src/mongo/db/dur_recover.h index 47eadc8a4e2..e2c8b32ef3c 100644 --- a/src/mongo/db/dur_recover.h +++ b/src/mongo/db/dur_recover.h @@ -28,12 +28,13 @@ namespace mongo { static RecoveryJob & get() { return _instance; } private: - void write(const ParsedJournalEntry& entry); // actually writes to the file - void applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump); + void write(const ParsedJournalEntry& entry, MongoMMF* mmf); // actually writes to the file + void applyEntry(const ParsedJournalEntry& entry, bool apply, bool dump, MongoMMF* mmf); void applyEntries(const vector<ParsedJournalEntry> &entries); bool processFileBuffer(const void *, unsigned len); bool processFile(boost::filesystem::path journalfile); void _close(); // doesn't lock + MongoMMF* getMongoMMF(const ParsedJournalEntry& entry); list<boost::shared_ptr<MongoMMF> > _mmfs; diff --git a/src/mongo/db/dur_writetodatafiles.cpp b/src/mongo/db/dur_writetodatafiles.cpp index d77b0482c20..b50d1ae66d6 100644 --- a/src/mongo/db/dur_writetodatafiles.cpp +++ b/src/mongo/db/dur_writetodatafiles.cpp @@ -17,12 +17,17 @@ */ #include "pch.h" -#include "dur_commitjob.h" -#include "dur_stats.h" -#include "dur_recover.h" -#include "../util/timer.h" + +#include "mongo/db/dur_commitjob.h" +#include "mongo/db/dur_recover.h" +#include "mongo/db/dur_stats.h" +#include "mongo/util/concurrency/mutex.h" +#include "mongo/util/timer.h" namespace mongo { +#ifdef _WIN32 + extern SimpleMutex globalFlushMutex; // defined in mongo/util/mmap_win.cpp +#endif namespace dur { void debugValidateAllMapsMatch(); @@ -83,6 +88,9 @@ namespace mongo { */ void WRITETODATAFILES(const JSectHeader& h, AlignedBuilder& uncompressed) { +#ifdef _WIN32 + SimpleMutex::scoped_lock _globalFlushMutex(globalFlushMutex); +#endif Timer t; WRITETODATAFILES_Impl1(h, uncompressed); unsigned long long m = t.micros(); diff --git a/src/mongo/db/index.cpp b/src/mongo/db/index.cpp index 6787befd8fa..274c3aa37d9 100644 --- a/src/mongo/db/index.cpp +++ b/src/mongo/db/index.cpp @@ -434,4 +434,15 @@ namespace mongo { _init(); } + void IndexChanges::dupCheck(IndexDetails& idx, DiskLoc curObjLoc) { + if (added.empty() || + !idx.unique() || + ignoreUniqueIndex(idx)) { + return; + } + const Ordering ordering = Ordering::make(idx.keyPattern()); + + // "E11001 duplicate key on update" + idx.idxInterface().uassertIfDups(idx, added, idx.head, curObjLoc, ordering); + } } diff --git a/src/mongo/db/index.h b/src/mongo/db/index.h index 4942af17daf..b4e093da12b 100644 --- a/src/mongo/db/index.h +++ b/src/mongo/db/index.h @@ -249,12 +249,7 @@ namespace mongo { /** @curObjLoc - the object we want to add's location. if it is already in the index, that is allowed here (for bg indexing case). */ - void dupCheck(IndexDetails& idx, DiskLoc curObjLoc) { - if( added.empty() || !idx.unique() ) - return; - const Ordering ordering = Ordering::make(idx.keyPattern()); - idx.idxInterface().uassertIfDups(idx, added, idx.head, curObjLoc, ordering); // "E11001 duplicate key on update" - } + void dupCheck(IndexDetails& idx, DiskLoc curObjLoc); }; class NamespaceDetails; diff --git a/src/mongo/db/index_update.cpp b/src/mongo/db/index_update.cpp index fcb36f3ba46..e8633cf3734 100644 --- a/src/mongo/db/index_update.cpp +++ b/src/mongo/db/index_update.cpp @@ -28,6 +28,7 @@ #include "mongo/db/namespace_details.h" #include "mongo/db/pdfile_private.h" #include "mongo/db/replutil.h" +#include "mongo/db/repl/rs.h" #include "mongo/util/processinfo.h" #include "mongo/util/startup_test.h" @@ -126,7 +127,13 @@ namespace mongo { BSONObjSet keys; for ( int i = 0; i < n; i++ ) { // this call throws on unique constraint violation. we haven't done any writes yet so that is fine. - fetchIndexInserters(/*out*/keys, inserter, d, i, obj, loc); + fetchIndexInserters(/*out*/keys, + inserter, + d, + i, + obj, + loc, + ignoreUniqueIndex(d->idx(i))); if( keys.size() > 1 ) { multi.push_back(i); multiKeys.push_back(BSONObjSet()); @@ -143,12 +150,13 @@ namespace mongo { unsigned i = multi[j]; BSONObjSet& keys = multiKeys[j]; IndexDetails& idx = d->idx(i); + bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx); IndexInterface& ii = idx.idxInterface(); Ordering ordering = Ordering::make(idx.keyPattern()); d->setIndexIsMultikey(ns, i); for( BSONObjSet::iterator k = ++keys.begin()/*skip 1*/; k != keys.end(); k++ ) { try { - ii.bt_insert(idx.head, loc, *k, ordering, !idx.unique(), idx); + ii.bt_insert(idx.head, loc, *k, ordering, dupsAllowed, idx); } catch (AssertionException& e) { if( e.getCode() == 10287 && (int) i == d->nIndexes ) { DEV log() << "info: caught key already in index on bg indexing (ok)" << endl; @@ -269,7 +277,7 @@ namespace mongo { tlog(1) << "fastBuildIndex " << ns << " idxNo:" << idxNo << ' ' << idx.info.obj().toString() << endl; - bool dupsAllowed = !idx.unique(); + bool dupsAllowed = !idx.unique() || ignoreUniqueIndex(idx); bool dropDups = idx.dropDups() || inDBRepair; BSONObj order = idx.keyPattern(); diff --git a/src/mongo/db/instance.cpp b/src/mongo/db/instance.cpp index 218fb9eec2b..edbef6e882e 100644 --- a/src/mongo/db/instance.cpp +++ b/src/mongo/db/instance.cpp @@ -476,14 +476,7 @@ namespace mongo { mongo::log(1) << "note: not profiling because doing fsync+lock" << endl; } else { - Lock::DBWrite lk( currentOp.getNS() ); - if ( dbHolder()._isLoaded( nsToDatabase( currentOp.getNS() ) , dbpath ) ) { - Client::Context cx( currentOp.getNS(), dbpath, false ); - profile(c , currentOp ); - } - else { - mongo::log() << "note: not profiling because db went away - probably a close on: " << currentOp.getNS() << endl; - } + profile(c, op, currentOp); } } diff --git a/src/mongo/db/introspect.cpp b/src/mongo/db/introspect.cpp index 5680fa1481f..07ddba73233 100644 --- a/src/mongo/db/introspect.cpp +++ b/src/mongo/db/introspect.cpp @@ -16,28 +16,24 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include "pch.h" -#include "introspect.h" -#include "../bson/util/builder.h" -#include "../util/goodies.h" -#include "pdfile.h" -#include "jsobj.h" -#include "pdfile.h" -#include "curop.h" +#include "mongo/pch.h" -namespace mongo { - - BufBuilder profileBufBuilder; // reused, instead of allocated every time - avoids a malloc/free cycle +#include "mongo/bson/util/builder.h" +#include "mongo/db/curop.h" +#include "mongo/db/databaseholder.h" +#include "mongo/db/introspect.h" +#include "mongo/db/jsobj.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/goodies.h" - void profile( const Client& c , CurOp& currentOp ) { - verify( Lock::somethingWriteLocked() ); +namespace mongo { + static void _profile(const Client& c, CurOp& currentOp, BufBuilder& profileBufBuilder) { Database *db = c.database(); DEV verify( db ); const char *ns = db->profileName.c_str(); // build object - profileBufBuilder.reset(); BSONObjBuilder b(profileBufBuilder); b.appendDate("ts", jsTime()); currentOp.debug().append( currentOp , b ); @@ -79,6 +75,29 @@ namespace mongo { } } + void profile(const Client& c, int op, CurOp& currentOp) { + // initialize with 1kb to start, to avoid realloc later + // doing this outside the dblock to improve performance + BufBuilder profileBufBuilder(1024); + + try { + Lock::DBWrite lk( currentOp.getNS() ); + if ( dbHolder()._isLoaded( nsToDatabase( currentOp.getNS() ) , dbpath ) ) { + Client::Context cx( currentOp.getNS(), dbpath, false ); + _profile(c, currentOp, profileBufBuilder); + } + else { + mongo::log() << "note: not profiling because db went away - probably a close on: " + << currentOp.getNS() << endl; + } + } + catch (const AssertionException& assertionEx) { + warning() << "Caught Assertion while trying to profile " << opToString(op) + << " against " << currentOp.getNS() + << ": " << assertionEx.toString() << endl; + } + } + NamespaceDetails* getOrCreateProfileCollection(Database *db, bool force) { fassert(16372, db); const char* profileName = db->profileName.c_str(); diff --git a/src/mongo/db/introspect.h b/src/mongo/db/introspect.h index 0923fd0cd4f..a4a1eb8f694 100644 --- a/src/mongo/db/introspect.h +++ b/src/mongo/db/introspect.h @@ -29,7 +29,7 @@ namespace mongo { do when database->profile is set */ - void profile( const Client& c , CurOp& currentOp ); + void profile(const Client& c, int op, CurOp& currentOp); /** * Get (or create) the profile collection diff --git a/src/mongo/db/jsobj.cpp b/src/mongo/db/jsobj.cpp index 509bc530755..a7d4048be54 100644 --- a/src/mongo/db/jsobj.cpp +++ b/src/mongo/db/jsobj.cpp @@ -63,10 +63,7 @@ namespace mongo { // need to move to bson/, but has dependency on base64 so move that to bson/util/ first. inline string BSONElement::jsonString( JsonStringFormat format, bool includeFieldNames, int pretty ) const { - BSONType t = type(); int sign; - if ( t == Undefined ) - return "undefined"; stringstream s; if ( includeFieldNames ) @@ -105,6 +102,14 @@ namespace mongo { case jstNULL: s << "null"; break; + case Undefined: + if ( format == Strict ) { + s << "{ \"$undefined\" : true }"; + } + else { + s << "undefined"; + } + break; case Object: s << embeddedObject().jsonString( format, pretty ); break; diff --git a/src/mongo/db/namespace_details.cpp b/src/mongo/db/namespace_details.cpp index a830b1210e3..19f3eef4c49 100644 --- a/src/mongo/db/namespace_details.cpp +++ b/src/mongo/db/namespace_details.cpp @@ -753,9 +753,15 @@ namespace mongo { if ( isUserFlagSet( Flag_UsePowerOf2Sizes ) ) { - int x = bucket( minRecordSize ); - x = bucketSizes[x]; - return x; + int allocationSize = bucketSizes[ bucket( minRecordSize ) ]; + if ( allocationSize < minRecordSize ) { + // if we get here, it means we're allocating more than 8mb + // the highest bucket is 8mb, so the above code will never return more than 8mb for allocationSize + // if this happens, we are going to round up to the nearest megabyte + fassert( 16439, bucket( minRecordSize ) == MaxBucket ); + allocationSize = 1 + ( minRecordSize | ( ( 1 << 20 ) - 1 ) ); + } + return allocationSize; } return static_cast<int>(minRecordSize * _paddingFactor); diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp index d7f87074c5b..fc937657a78 100644 --- a/src/mongo/db/oplog.cpp +++ b/src/mongo/db/oplog.cpp @@ -124,10 +124,30 @@ namespace mongo { *b = EOO; } + /* we write to local.oplog.rs: + { ts : ..., h: ..., v: ..., op: ..., etc } + ts: an OpTime timestamp + h: hash + v: version + op: + "i" insert + "u" update + "d" delete + "c" db cmd + "db" declares presence of a database (ns is set to the db name + '.') + "n" no op + + bb param: + if not null, specifies a boolean to pass along to the other side as b: param. + used for "justOne" or "upsert" flags on 'd', 'u' + + */ + // global is safe as we are in write lock. we put the static outside the function to avoid the implicit mutex // the compiler would use if inside the function. the reason this is static is to avoid a malloc/free for this // on every logop call. static BufBuilder logopbufbuilder(8*1024); + const static int OPLOG_VERSION = 2; static void _logOpRS(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) { Lock::DBWrite lk1("local"); @@ -159,6 +179,7 @@ namespace mongo { BSONObjBuilder b(logopbufbuilder); b.appendTimestamp("ts", ts.asDate()); b.append("h", hashNew); + b.append("v", OPLOG_VERSION); b.append("op", opstr); b.append("ns", ns); if (fromMigrate) @@ -205,26 +226,6 @@ namespace mongo { } } - /* we write to local.oplog.$main: - { ts : ..., op: ..., ns: ..., o: ... } - ts: an OpTime timestamp - op: - "i" insert - "u" update - "d" delete - "c" db cmd - "db" declares presence of a database (ns is set to the db name + '.') - "n" no op - logNS: where to log it. 0/null means "local.oplog.$main". - bb: - if not null, specifies a boolean to pass along to the other side as b: param. - used for "justOne" or "upsert" flags on 'd', 'u' - first: true - when set, indicates this is the first thing we have logged for this database. - thus, the slave does not need to copy down all the data when it sees this. - - note this is used for single collection logging even when --replSet is enabled. - */ static void _logOpOld(const char *opstr, const char *ns, const char *logNS, const BSONObj& obj, BSONObj *o2, bool *bb, bool fromMigrate ) { Lock::DBWrite lk("local"); static BufBuilder bufbuilder(8*1024); // todo there is likely a mutex on this constructor @@ -759,8 +760,8 @@ namespace mongo { if( !o.getObjectID(_id) ) { /* No _id. This will be very slow. */ Timer t; - updateObjects(ns, o, o, true, false, false, debug, false, - QueryPlanSelectionPolicy::idElseNatural() ); + updateObjectsForReplication(ns, o, o, true, false, false, debug, false, + QueryPlanSelectionPolicy::idElseNatural() ); if( t.millis() >= 2 ) { RARELY OCCASIONALLY log() << "warning, repl doing slow updates (no _id field) for " << ns << endl; } @@ -775,8 +776,8 @@ namespace mongo { */ BSONObjBuilder b; b.append(_id); - updateObjects(ns, o, b.done(), true, false, false , debug, false, - QueryPlanSelectionPolicy::idElseNatural() ); + updateObjectsForReplication(ns, o, b.done(), true, false, false , debug, false, + QueryPlanSelectionPolicy::idElseNatural() ); } } } @@ -790,10 +791,18 @@ namespace mongo { OpDebug debug; BSONObj updateCriteria = op.getObjectField("o2"); bool upsert = fields[3].booleanSafe() || convertUpdateToUpsert; - UpdateResult ur = updateObjects(ns, o, updateCriteria, upsert, /*multi*/ false, - /*logop*/ false , debug, /*fromMigrate*/ false, + UpdateResult ur = + updateObjectsForReplication(ns, + o, + updateCriteria, + upsert, + /*multi*/ false, + /*logop*/ false, + debug, + /*fromMigrate*/ false, QueryPlanSelectionPolicy::idElseNatural() ); - if( ur.num == 0 ) { + + if( ur.num == 0 ) { if( ur.mod ) { if( updateCriteria.nFields() == 1 ) { // was a simple { _id : ... } update criteria diff --git a/src/mongo/db/ops/query.cpp b/src/mongo/db/ops/query.cpp index 9e8b1d37f07..2b12e0d577a 100644 --- a/src/mongo/db/ops/query.cpp +++ b/src/mongo/db/ops/query.cpp @@ -913,13 +913,13 @@ namespace mongo { // Run a command. if ( pq.couldBeCommand() ) { + curop.markCommand(); BufBuilder bb; bb.skip(sizeof(QueryResult)); BSONObjBuilder cmdResBuf; if ( runCommands(ns, jsobj, curop, bb, cmdResBuf, false, queryOptions) ) { curop.debug().iscommand = true; curop.debug().query = jsobj; - curop.markCommand(); auto_ptr< QueryResult > qr; qr.reset( (QueryResult *) bb.buf() ); diff --git a/src/mongo/db/ops/update.cpp b/src/mongo/db/ops/update.cpp index 8f9abf5ba7c..954d2e0b1f3 100644 --- a/src/mongo/db/ops/update.cpp +++ b/src/mongo/db/ops/update.cpp @@ -100,25 +100,20 @@ namespace mongo { if ( logop ) { DEV verify( mods->size() ); - BSONObj pattern = patternOrig; - if ( mss->haveArrayDepMod() ) { - BSONObjBuilder patternBuilder; - patternBuilder.appendElements( pattern ); - mss->appendSizeSpecForArrayDepMods( patternBuilder ); - pattern = patternBuilder.obj(); - } - - if( mss->needOpLogRewrite() ) { - DEBUGUPDATE( "\t rewrite update: " << mss->getOpLogRewrite() ); - logOp("u", ns, mss->getOpLogRewrite() , - &pattern, 0, fromMigrate ); - } - else { - logOp("u", ns, updateobj, &pattern, 0, fromMigrate ); + BSONObj logObj = mss->getOpLogRewrite(); + DEBUGUPDATE( "\t rewrite update: " << logObj ); + + // It is possible that the entire mod set was a no-op over this document. We + // would have an empty log record in that case. If we call logOp, with an empty + // record, that would be replicated as "clear this record", which is not what + // we want. Therefore, to get a no-op in the replica, we simply don't log. + if ( logObj.nFields() ) { + logOp("u", ns, logObj, &pattern, 0, fromMigrate ); } } return UpdateResult( 1 , 1 , 1 , BSONObj() ); + } // end $operator update // regular update @@ -142,7 +137,8 @@ namespace mongo { OpDebug& debug, RemoveSaver* rs, bool fromMigrate, - const QueryPlanSelectionPolicy& planPolicy ) { + const QueryPlanSelectionPolicy& planPolicy, + bool forReplication ) { DEBUGUPDATE( "update: " << ns << " update: " << updateobj @@ -168,10 +164,10 @@ namespace mongo { if( d && d->indexBuildInProgress ) { set<string> bgKeys; d->inProgIdx().keyPattern().getFieldNames(bgKeys); - mods.reset( new ModSet(updateobj, nsdt->indexKeys(), &bgKeys) ); + mods.reset( new ModSet(updateobj, nsdt->indexKeys(), &bgKeys, forReplication) ); } else { - mods.reset( new ModSet(updateobj, nsdt->indexKeys()) ); + mods.reset( new ModSet(updateobj, nsdt->indexKeys(), NULL, forReplication) ); } modsIsIndexed = mods->isIndexed(); } @@ -336,13 +332,11 @@ namespace mongo { const BSONObj& onDisk = loc.obj(); ModSet* useMods = mods.get(); - bool forceRewrite = false; auto_ptr<ModSet> mymodset; if ( details.hasElemMatchKey() && mods->hasDynamicArray() ) { useMods = mods->fixDynamicArray( details.elemMatchKey() ); mymodset.reset( useMods ); - forceRewrite = true; } auto_ptr<ModSetState> mss = useMods->prepare( onDisk ); @@ -394,21 +388,16 @@ namespace mongo { if ( logop ) { DEV verify( mods->size() ); - - if ( mss->haveArrayDepMod() ) { - BSONObjBuilder patternBuilder; - patternBuilder.appendElements( pattern ); - mss->appendSizeSpecForArrayDepMods( patternBuilder ); - pattern = patternBuilder.obj(); - } - - if ( forceRewrite || mss->needOpLogRewrite() ) { - DEBUGUPDATE( "\t rewrite update: " << mss->getOpLogRewrite() ); - logOp("u", ns, mss->getOpLogRewrite() , - &pattern, 0, fromMigrate ); - } - else { - logOp("u", ns, updateobj, &pattern, 0, fromMigrate ); + BSONObj logObj = mss->getOpLogRewrite(); + DEBUGUPDATE( "\t rewrite update: " << logObj ); + + // It is possible that the entire mod set was a no-op over this + // document. We would have an empty log record in that case. If we + // call logOp, with an empty record, that would be replicated as "clear + // this record", which is not what we want. Therefore, to get a no-op + // in the replica, we simply don't log. + if ( logObj.nFields() ) { + logOp("u", ns, logObj , &pattern, 0, fromMigrate ); } } numModded++; @@ -463,6 +452,18 @@ namespace mongo { return UpdateResult( 0 , isOperatorUpdate , 0 , BSONObj() ); } + void validateUpdate( const char* ns , const BSONObj& updateobj, const BSONObj& patternOrig ) { + uassert( 10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0 ); + if ( strstr(ns, ".system.") ) { + /* dm: it's very important that system.indexes is never updated as IndexDetails + has pointers into it */ + uassert( 10156, + str::stream() << "cannot update system collection: " + << ns << " q: " << patternOrig << " u: " << updateobj, + legalClientSystemNS( ns , true ) ); + } + } + UpdateResult updateObjects( const char* ns, const BSONObj& updateobj, const BSONObj& patternOrig, @@ -473,17 +474,39 @@ namespace mongo { bool fromMigrate, const QueryPlanSelectionPolicy& planPolicy ) { - uassert( 10155 , "cannot update reserved $ collection", strchr(ns, '$') == 0 ); - if ( strstr(ns, ".system.") ) { - /* dm: it's very important that system.indexes is never updated as IndexDetails has pointers into it */ - uassert( 10156, - str::stream() << "cannot update system collection: " << ns << " q: " << patternOrig << " u: " << updateobj, - legalClientSystemNS( ns , true ) ); - } + validateUpdate( ns , updateobj , patternOrig ); UpdateResult ur = _updateObjects(false, ns, updateobj, patternOrig, upsert, multi, logop, - debug, 0, fromMigrate, planPolicy ); + debug, NULL, fromMigrate, planPolicy ); + debug.nupdated = ur.num; + return ur; + } + + UpdateResult updateObjectsForReplication( const char* ns, + const BSONObj& updateobj, + const BSONObj& patternOrig, + bool upsert, + bool multi, + bool logop , + OpDebug& debug, + bool fromMigrate, + const QueryPlanSelectionPolicy& planPolicy ) { + + validateUpdate( ns , updateobj , patternOrig ); + + UpdateResult ur = _updateObjects(false, + ns, + updateobj, + patternOrig, + upsert, + multi, + logop, + debug, + NULL /* no remove saver */, + fromMigrate, + planPolicy, + true /* for replication */ ); debug.nupdated = ur.num; return ur; } diff --git a/src/mongo/db/ops/update.h b/src/mongo/db/ops/update.h index 76d864821c8..c24f0628091 100644 --- a/src/mongo/db/ops/update.h +++ b/src/mongo/db/ops/update.h @@ -58,6 +58,23 @@ namespace mongo { bool fromMigrate = false, const QueryPlanSelectionPolicy& planPolicy = QueryPlanSelectionPolicy::any()); + /* + * Similar to updateObjects but not strict about applying mods that can fail during initial + * replication. + * + * Reference ticket: SERVER-4781 + */ + UpdateResult updateObjectsForReplication(const char* ns, + const BSONObj& updateobj, + const BSONObj& pattern, + bool upsert, + bool multi, + bool logop, + OpDebug& debug, + bool fromMigrate = false, + const QueryPlanSelectionPolicy& planPolicy = + QueryPlanSelectionPolicy::any()); + UpdateResult _updateObjects(bool su, const char* ns, const BSONObj& updateobj, @@ -68,7 +85,8 @@ namespace mongo { OpDebug& debug, RemoveSaver* rs = 0, bool fromMigrate = false, - const QueryPlanSelectionPolicy& planPolicy = QueryPlanSelectionPolicy::any()); + const QueryPlanSelectionPolicy& planPolicy = QueryPlanSelectionPolicy::any(), + bool forReplication = false); /** diff --git a/src/mongo/db/ops/update_internal.cpp b/src/mongo/db/ops/update_internal.cpp index 69409304598..5569c6f10c3 100644 --- a/src/mongo/db/ops/update_internal.cpp +++ b/src/mongo/db/ops/update_internal.cpp @@ -21,6 +21,7 @@ #include "mongo/db/oplog.h" #include "mongo/db/jsobjmanipulator.h" #include "mongo/db/pdfile.h" +#include "mongo/util/mongoutils/str.h" #include "update_internal.h" @@ -98,6 +99,9 @@ namespace mongo { case INC: { appendIncremented( builder , in , ms ); + // We don't need to "fix" this operation into a $set, for oplog purposes, + // here. ModState::appendForOpLog will do that for us. It relies on the new value + // being in inc{int,long,double} inside the ModState that wraps around this Mod. break; } @@ -114,27 +118,36 @@ namespace mongo { case PUSH: { uassert( 10131 , "$push can only be applied to an array" , in.type() == Array ); - BSONObjBuilder bb( builder.subarrayStart( shortFieldName ) ); + BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); BSONObjIterator i( in.embeddedObject() ); - int n=0; while ( i.more() ) { bb.append( i.next() ); - n++; } - ms.pushStartSize = n; + bb.append( elt ); + + // We don't want to log a positional $set for which the '_checkForAppending' test + // won't pass. If we're in that case, fall back to non-optimized logging. + if ( (elt.type() == Object && elt.embeddedObject().okForStorage()) || + (elt.type() != Object) ) { + ms.fixedOpName = "$set"; + ms.forcePositional = true; + ms.position = bb.arrSize() - 1; + bb.done(); + } + else { + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray( bb.done().getOwned() ); + } - bb.appendAs( elt , bb.numStr( n ) ); - bb.done(); break; } case ADDTOSET: { uassert( 12592 , "$addToSet can only be applied to an array" , in.type() == Array ); - BSONObjBuilder bb( builder.subarrayStart( shortFieldName ) ); - + BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); BSONObjIterator i( in.embeddedObject() ); - int n=0; if ( isEach() ) { @@ -144,7 +157,6 @@ namespace mongo { while ( i.more() ) { BSONElement cur = i.next(); bb.append( cur ); - n++; toadd.erase( cur ); } @@ -153,64 +165,80 @@ namespace mongo { while ( i.more() ) { BSONElement e = i.next(); if ( toadd.count(e) ) { - bb.appendAs( e , BSONObjBuilder::numStr( n++ ) ); + bb.append( e ); toadd.erase( e ); } } } + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(bb.done().getOwned()); } else { bool found = false; - + int pos = 0; + int count = 0; while ( i.more() ) { BSONElement cur = i.next(); bb.append( cur ); - n++; - if ( elt.woCompare( cur , false ) == 0 ) + if ( elt.woCompare( cur , false ) == 0 ) { found = true; + pos = count; + } + count++; } - if ( ! found ) - bb.appendAs( elt , bb.numStr( n ) ); + if ( !found ) { + bb.append( elt ); + } + // We don't want to log a positional $set for which the '_checkForAppending' + // test won't pass. If we're in that case, fall back to non-optimized logging. + if ( (elt.type() == Object && elt.embeddedObject().okForStorage()) || + (elt.type() != Object) ) { + ms.fixedOpName = "$set"; + ms.forcePositional = true; + ms.position = found ? pos : bb.arrSize() - 1; + bb.done(); + } + else { + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(bb.done().getOwned()); + } } - bb.done(); break; } - - case PUSH_ALL: { uassert( 10132 , "$pushAll can only be applied to an array" , in.type() == Array ); uassert( 10133 , "$pushAll has to be passed an array" , elt.type() ); - BSONObjBuilder bb( builder.subarrayStart( shortFieldName ) ); + BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); BSONObjIterator i( in.embeddedObject() ); - int n=0; while ( i.more() ) { bb.append( i.next() ); - n++; } - ms.pushStartSize = n; - i = BSONObjIterator( elt.embeddedObject() ); while ( i.more() ) { - bb.appendAs( i.next() , bb.numStr( n++ ) ); + bb.append( i.next() ); } - bb.done(); + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(bb.done().getOwned()); break; } case PULL: case PULL_ALL: { uassert( 10134 , "$pull/$pullAll can only be applied to an array" , in.type() == Array ); - BSONObjBuilder bb( builder.subarrayStart( shortFieldName ) ); + BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); //temporarily record the things to pull. only use this set while 'elt' in scope. BSONElementSet toPull; @@ -221,8 +249,6 @@ namespace mongo { } } - int n = 0; - BSONObjIterator i( in.embeddedObject() ); while ( i.more() ) { BSONElement e = i.next(); @@ -236,36 +262,36 @@ namespace mongo { } if ( allowed ) - bb.appendAs( e , bb.numStr( n++ ) ); + bb.append( e ); } - bb.done(); + // If this is the last element of the array, then we want to write the empty array to the + // oplog. + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(bb.done().getOwned()); break; } case POP: { uassert( 10135 , "$pop can only be applied to an array" , in.type() == Array ); - BSONObjBuilder bb( builder.subarrayStart( shortFieldName ) ); + BSONArrayBuilder bb( builder.subarrayStart( shortFieldName ) ); - int n = 0; BSONObjIterator i( in.embeddedObject() ); if ( elt.isNumber() && elt.number() < 0 ) { // pop from front if ( i.more() ) { i.next(); - n++; } while( i.more() ) { - bb.appendAs( i.next() , bb.numStr( n - 1 ) ); - n++; + bb.append( i.next() ); } } else { // pop from back while( i.more() ) { - n++; BSONElement arrI = i.next(); if ( i.more() ) { bb.append( arrI ); @@ -273,9 +299,9 @@ namespace mongo { } } - ms.pushStartSize = n; - verify( ms.pushStartSize == in.embeddedObject().nFields() ); - bb.done(); + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(bb.done().getOwned()); break; } @@ -311,8 +337,25 @@ namespace mongo { } switch( in.type() ) { - case NumberInt: builder.append( shortFieldName , x ); break; - case NumberLong: builder.append( shortFieldName , y ); break; + + case NumberInt: + builder.append( shortFieldName , x ); + // By recording the result of the bit manipulation into the ModSet, we'll be + // set up so that this $bit operation be "fixed" as a $set of the final result + // in the oplog. This will happen in appendForOpLog and what triggers it is + // setting the incType in the ModSet that is around this Mod. + ms.incType = NumberInt; + ms.incint = x; + break; + + case NumberLong: + // Please see comment on fixing this $bit into a $set for logging purposes in + // the NumberInt case. + builder.append( shortFieldName , y ); + ms.incType = NumberLong; + ms.inclong = y; + break; + default: verify( 0 ); } @@ -320,10 +363,14 @@ namespace mongo { } case RENAME_FROM: { + // We don't need to "fix" this operation into a $set here. ModState::appendForOpLog + // will do that for us. It relies on the field name being stored on this Mod. break; } case RENAME_TO: { + // We don't need to "fix" this operation into a $set here, for the same reason we + // didn't either with RENAME_FROM. ms.handleRename( builder, shortFieldName ); break; } @@ -504,6 +551,7 @@ namespace mongo { } void ModState::appendForOpLog( BSONObjBuilder& b ) const { + // dontApply logic is deprecated for all but $rename. if ( dontApply ) { return; } @@ -537,10 +585,24 @@ namespace mongo { DEBUGUPDATE( "\t\t\t\t\t appendForOpLog name:" << name << " fixed: " << fixed << " fn: " << m->fieldName ); + if (strcmp(name, "$unset") == 0) { + BSONObjBuilder bb(b.subobjStart(name)); + bb.append(m->fieldName, 1); + bb.done(); + return; + } + BSONObjBuilder bb( b.subobjStart( name ) ); if ( fixed ) { bb.appendAs( *fixed , m->fieldName ); } + else if ( ! fixedArray.isEmpty() || forceEmptyArray ) { + bb.append( m->fieldName, fixedArray ); + } + else if ( forcePositional ) { + string positionalField = str::stream() << m->fieldName << "." << position; + bb.appendAs( m->elt, positionalField.c_str() ); + } else { bb.appendAs( m->elt , m->fieldName ); } @@ -579,19 +641,33 @@ namespace mongo { switch ( m.m->op ) { case Mod::UNSET: + m.fixedOpName = "$unset"; + break; + case Mod::ADDTOSET: + m.fixedOpName = "$set"; + m.fixed = &(m.old); + break; + case Mod::RENAME_FROM: case Mod::RENAME_TO: // this should have been handled by prepare break; + case Mod::PULL: case Mod::PULL_ALL: // this should have been handled by prepare + m.fixedOpName = "$set"; + m.fixed = &(m.old); break; + case Mod::POP: - verify( m.old.eoo() || ( m.old.isABSONObj() && m.old.Obj().isEmpty() ) ); + verify( m.old.isABSONObj() && m.old.Obj().isEmpty() ); + m.fixedOpName = "$set"; + m.fixed = &(m.old); break; // [dm] the BSONElementManipulator statements below are for replication (correct?) + case Mod::INC: if ( isOnDisk ) m.m->IncrementMe( m.old ); @@ -600,12 +676,14 @@ namespace mongo { m.fixedOpName = "$set"; m.fixed = &(m.old); break; + case Mod::SET: if ( isOnDisk ) BSONElementManipulator( m.old ).ReplaceTypeAndValue( m.m->elt ); else BSONElementManipulator( m.old ).replaceTypeAndValue( m.m->elt ); break; + default: uassert( 13478 , "can't apply mod in place - shouldn't have gotten here" , 0 ); } @@ -618,11 +696,25 @@ namespace mongo { set<string>& onedownseen ) { Mod& m = *((Mod*)(modState.m)); // HACK switch (m.op) { - // unset/pull/pullAll on nothing does nothing, so don't append anything - case Mod::UNSET: + // unset/pull/pullAll on nothing does nothing, so don't append anything. Still, + // explicitly log that the target array was reset. + case Mod::POP: case Mod::PULL: case Mod::PULL_ALL: + case Mod::UNSET: + modState.fixedOpName = "$unset"; return; + + // $rename may involve dotted path creation, so we want to make sure we're not + // creating a path here for a rename that's a no-op. In other words if we're + // issuing a {$rename: {a.b : c.d} } that's a no-op, we don't want to create + // the a and c paths here. See test NestedNoName in the 'repl' suite. + case Mod::RENAME_FROM: + case Mod::RENAME_TO: + if (modState.dontApply) { + return; + } + default: ;// fall through } @@ -714,9 +806,34 @@ namespace mongo { switch ( cmp ) { case LEFT_SUBFIELD: { // Mod is embedded under this element - uassert( 10145, - str::stream() << "LEFT_SUBFIELD only supports Object: " << field - << " not: " << e.type() , e.type() == Object || e.type() == Array ); + + // SERVER-4781 + bool isObjOrArr = e.type() == Object || e.type() == Array; + if ( ! isObjOrArr ) { + if (m->second->m->strictApply) { + uasserted( 10145, + str::stream() << "LEFT_SUBFIELD only supports Object: " << field + << " not: " << e.type() ); + } + else { + // Since we're not applying the mod, we keep what was there before + builder.append( e ); + + // Skip both as we're not applying this mod. Note that we'll advance + // the iterator on the mod side for all the mods that are under the + // root we are now. + e = es.next(); + m++; + while ( m != mend && + ( compareDottedFieldNames( m->second->m->fieldName, + field, + lexNumCmp ) == LEFT_SUBFIELD ) ) { + m++; + } + continue; + } + } + if ( onedownseen.count( e.fieldName() ) == 0 ) { onedownseen.insert( e.fieldName() ); if ( e.type() == Object ) { @@ -734,6 +851,11 @@ namespace mongo { // inc both as we handled both e = es.next(); m++; + while ( m != mend && + ( compareDottedFieldNames( m->second->m->fieldName , field , lexNumCmp ) == + LEFT_SUBFIELD ) ) { + m++; + } } else { massert( 16069 , "ModSet::createNewFromMods - " @@ -809,7 +931,7 @@ namespace mongo { // we have something like { x : { $gt : 5 } } // this can be a query piece // or can be a dbref or something - + int op = e.embeddedObject().firstElement().getGtLtOp( -1 ); if ( op >= 0 ) { // this means this is a $gt type filter, so don't make part of the new object @@ -845,7 +967,8 @@ namespace mongo { ModSet::ModSet( const BSONObj& from , const set<string>& idxKeys, - const set<string>* backgroundKeys) + const set<string>* backgroundKeys, + bool forReplication) : _isIndexed(0) , _hasDynamicArray( false ) { BSONObjIterator it(from); @@ -934,13 +1057,13 @@ namespace mongo { strstr( target , ".$" ) == 0 ); Mod from; - from.init( Mod::RENAME_FROM, f ); + from.init( Mod::RENAME_FROM, f , forReplication ); from.setFieldName( fieldName ); updateIsIndexed( from, idxKeys, backgroundKeys ); _mods[ from.fieldName ] = from; Mod to; - to.init( Mod::RENAME_TO, f ); + to.init( Mod::RENAME_TO, f , forReplication ); to.setFieldName( target ); updateIsIndexed( to, idxKeys, backgroundKeys ); _mods[ to.fieldName ] = to; @@ -952,7 +1075,7 @@ namespace mongo { _hasDynamicArray = _hasDynamicArray || strstr( fieldName , ".$" ) > 0; Mod m; - m.init( op , f ); + m.init( op , f , forReplication ); m.setFieldName( f.fieldName() ); updateIsIndexed( m, idxKeys, backgroundKeys ); _mods[m.fieldName] = m; diff --git a/src/mongo/db/ops/update_internal.h b/src/mongo/db/ops/update_internal.h index c69cd9fcf65..fee4484c950 100644 --- a/src/mongo/db/ops/update_internal.h +++ b/src/mongo/db/ops/update_internal.h @@ -45,13 +45,19 @@ namespace mongo { const char* fieldName; const char* shortFieldName; + // Determines if this mod must absoluetly be applied. In some replication scenarios, a + // failed apply of a mod does not constitute an error. In those cases, setting strict + // to off would not throw errors. + bool strictApply; + BSONElement elt; // x:5 note: this is the actual element from the updateobj boost::shared_ptr<Matcher> matcher; bool matcherOnPrimitive; - void init( Op o , BSONElement& e ) { + void init( Op o , BSONElement& e , bool forReplication ) { op = o; elt = e; + strictApply = !forReplication; if ( op == PULL && e.type() == Object ) { BSONObj t = e.embeddedObject(); if ( t.firstElement().getGtLtOp() == 0 ) { @@ -331,7 +337,8 @@ namespace mongo { ModSet( const BSONObj& from, const set<string>& idxKeys = set<string>(), - const set<string>* backgroundKeys = 0 ); + const set<string>* backgroundKeys = 0, + bool forReplication = false ); /** * re-check if this mod is impacted by indexes @@ -399,7 +406,11 @@ namespace mongo { const char* fixedOpName; BSONElement* fixed; - int pushStartSize; + BSONArray fixedArray; + bool forceEmptyArray; + bool forcePositional; + int position; + int DEPRECATED_pushStartSize; BSONType incType; int incint; @@ -411,7 +422,10 @@ namespace mongo { ModState() { fixedOpName = 0; fixed = 0; - pushStartSize = -1; + forceEmptyArray = false; + forcePositional = false; + position = 0; + DEPRECATED_pushStartSize = -1; incType = EOO; dontApply = false; } @@ -424,7 +438,7 @@ namespace mongo { return m->fieldName; } - bool needOpLogRewrite() const { + bool DEPRECATED_needOpLogRewrite() const { if ( dontApply ) return false; @@ -438,8 +452,7 @@ namespace mongo { case Mod::BIT: case Mod::BITAND: case Mod::BITOR: - // TODO: should we convert this to $set? - return false; + return true; default: return false; } @@ -518,49 +531,64 @@ namespace mongo { switch ( m.op ) { case Mod::PUSH: { + ms.fixedOpName = "$set"; if ( m.isEach() ) { - b.appendArray( m.shortFieldName, m.getEach() ); + BSONObj arr = m.getEach(); + b.appendArray( m.shortFieldName, arr ); + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(arr.getOwned()); } else { BSONObjBuilder arr( b.subarrayStart( m.shortFieldName ) ); arr.appendAs( m.elt, "0" ); - arr.done(); + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(arr.done().getOwned()); } break; } + case Mod::ADDTOSET: { + ms.fixedOpName = "$set"; if ( m.isEach() ) { // Remove any duplicates in given array - BSONObjBuilder arr( b.subarrayStart( m.shortFieldName ) ); + BSONArrayBuilder arr( b.subarrayStart( m.shortFieldName ) ); BSONElementSet toadd; m.parseEach( toadd ); BSONObjIterator i( m.getEach() ); - int n = 0; + // int n = 0; while ( i.more() ) { BSONElement e = i.next(); if ( toadd.count(e) ) { - arr.appendAs( e , BSONObjBuilder::numStr( n++ ) ); + arr.append( e ); toadd.erase( e ); } } - arr.done(); + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(arr.done().getOwned()); } else { - BSONObjBuilder arr( b.subarrayStart( m.shortFieldName ) ); - arr.appendAs( m.elt, "0" ); - arr.done(); + BSONArrayBuilder arr( b.subarrayStart( m.shortFieldName ) ); + arr.append( m.elt ); + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(arr.done().getOwned()); } break; } case Mod::PUSH_ALL: { b.appendAs( m.elt, m.shortFieldName ); + ms.fixedOpName = "$set"; + ms.forceEmptyArray = true; + ms.fixedArray = BSONArray(m.elt.Obj()); break; } - case Mod::UNSET: + case Mod::POP: case Mod::PULL: case Mod::PULL_ALL: - // no-op b/c unset/pull of nothing does nothing + case Mod::UNSET: + // No-op b/c unset/pull of nothing does nothing. Still, explicilty log that + // the target array was reset. + ms.fixedOpName = "$unset"; break; case Mod::INC: @@ -570,10 +598,12 @@ namespace mongo { b.appendAs( m.elt, m.shortFieldName ); break; } + // shouldn't see RENAME_FROM here case Mod::RENAME_TO: ms.handleRename( b, m.shortFieldName ); break; + default: stringstream ss; ss << "unknown mod in appendNewFromMod: " << m.op; @@ -601,9 +631,9 @@ namespace mongo { // re-writing for oplog - bool needOpLogRewrite() const { + bool DEPRECATED_needOpLogRewrite() const { for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) - if ( i->second->needOpLogRewrite() ) + if ( i->second->DEPRECATED_needOpLogRewrite() ) return true; return false; } @@ -615,21 +645,21 @@ namespace mongo { return b.obj(); } - bool haveArrayDepMod() const { + bool DEPRECATED_haveArrayDepMod() const { for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) if ( i->second->m->arrayDep() ) return true; return false; } - void appendSizeSpecForArrayDepMods( BSONObjBuilder& b ) const { + void DEPRECATED_appendSizeSpecForArrayDepMods( BSONObjBuilder& b ) const { for ( ModStateHolder::const_iterator i = _mods.begin(); i != _mods.end(); i++ ) { const ModState& m = *i->second; if ( m.m->arrayDep() ) { - if ( m.pushStartSize == -1 ) + if ( m.DEPRECATED_pushStartSize == -1 ) b.appendNull( m.fieldName() ); else - b << m.fieldName() << BSON( "$size" << m.pushStartSize ); + b << m.fieldName() << BSON( "$size" << m.DEPRECATED_pushStartSize ); } } } diff --git a/src/mongo/db/pdfile.cpp b/src/mongo/db/pdfile.cpp index 64e86613296..6719b73cc5e 100644 --- a/src/mongo/db/pdfile.cpp +++ b/src/mongo/db/pdfile.cpp @@ -659,8 +659,14 @@ namespace mongo { } DiskLoc Extent::_reuse(const char *nsname, bool capped) { - LOG(3) << "reset extent was:" << nsDiagnostic.toString() << " now:" << nsname << '\n'; - massert( 10360 , "Extent::reset bad magic value", magic == 0x41424344 ); + LOG(3) << "_reuse extent was:" << nsDiagnostic.toString() << " now:" << nsname << endl; + if (magic != extentSignature) { + StringBuilder sb; + sb << "bad extent signature " << toHex(&magic, 4) + << " for namespace '" << nsDiagnostic.toString() + << "' found in Extent::_reuse"; + msgasserted(10360, sb.str()); + } nsDiagnostic = nsname; markEmpty(); @@ -680,7 +686,7 @@ namespace mongo { /* assumes already zeroed -- insufficient for block 'reuse' perhaps */ DiskLoc Extent::init(const char *nsname, int _length, int _fileNo, int _offset, bool capped) { - magic = 0x41424344; + magic = extentSignature; myLoc.set(_fileNo, _offset); xnext.Null(); xprev.Null(); @@ -700,6 +706,56 @@ namespace mongo { return emptyLoc; } + bool Extent::validates(const DiskLoc diskLoc, BSONArrayBuilder* errors) { + bool extentOk = true; + if (magic != extentSignature) { + if (errors) { + StringBuilder sb; + sb << "bad extent signature " << toHex(&magic, 4) + << " in extent " << diskLoc.toString(); + *errors << sb.str(); + } + extentOk = false; + } + if (myLoc != diskLoc) { + if (errors) { + StringBuilder sb; + sb << "extent " << diskLoc.toString() + << " self-pointer is " << myLoc.toString(); + *errors << sb.str(); + } + extentOk = false; + } + if (firstRecord.isNull() != lastRecord.isNull()) { + if (errors) { + StringBuilder sb; + if (firstRecord.isNull()) { + sb << "in extent " << diskLoc.toString() + << ", firstRecord is null but lastRecord is " + << lastRecord.toString(); + } + else { + sb << "in extent " << diskLoc.toString() + << ", firstRecord is " << firstRecord.toString() + << " but lastRecord is null"; + } + *errors << sb.str(); + } + extentOk = false; + } + if (length < minSize()) { + if (errors) { + StringBuilder sb; + sb << "length of extent " << diskLoc.toString() + << " is " << length + << ", which is less than minimum length of " << minSize(); + *errors << sb.str(); + } + extentOk = false; + } + return extentOk; + } + /* Record* Extent::newRecord(int len) { if( firstEmptyRegion.isNull() )8 @@ -1229,6 +1285,8 @@ namespace mongo { for ( int idxNo = 0; idxNo < d->nIndexes; idxNo++ ) { if( d->idx(idxNo).unique() ) { IndexDetails& idx = d->idx(idxNo); + if (ignoreUniqueIndex(idx)) + continue; BSONObjSet keys; idx.getKeysFromObject(obj, keys); BSONObj order = idx.keyPattern(); @@ -1341,7 +1399,7 @@ namespace mongo { BSONObj info = loc.obj(); bool background = info["background"].trueValue(); - if( background && cc().isSyncThread() ) { + if (background && !isMasterNs(tabletoidxns.c_str())) { /* don't do background indexing on slaves. there are nuances. this could be added later but requires more code. */ @@ -1448,10 +1506,13 @@ namespace mongo { } int lenWHdr = d->getRecordAllocationSize( len + Record::HeaderSize ); - + fassert( 16440, lenWHdr >= ( len + Record::HeaderSize ) ); + // If the collection is capped, check if the new object will violate a unique index // constraint before allocating space. - if ( d->nIndexes && d->isCapped() && !god ) { + if (d->nIndexes && + d->isCapped() && + !god) { checkNoIndexConflicts( d, BSONObj( reinterpret_cast<const char *>( obuf ) ) ); } diff --git a/src/mongo/db/pdfile.h b/src/mongo/db/pdfile.h index d1904f928b9..20afa4353c4 100644 --- a/src/mongo/db/pdfile.h +++ b/src/mongo/db/pdfile.h @@ -314,6 +314,7 @@ namespace mongo { */ class Extent { public: + enum { extentSignature = 0x41424344 }; unsigned magic; DiskLoc myLoc; DiskLoc xnext, xprev; /* next/prev extent for this namespace */ @@ -330,10 +331,7 @@ namespace mongo { static int HeaderSize() { return sizeof(Extent)-4; } - bool validates() { - return !(firstRecord.isNull() ^ lastRecord.isNull()) && - length >= 0 && !myLoc.isNull(); - } + bool validates(const DiskLoc diskLoc, BSONArrayBuilder* errors = NULL); BSONObj dump() { return BSON( "loc" << myLoc.toString() << "xnext" << xnext.toString() << "xprev" << xprev.toString() @@ -356,7 +354,7 @@ namespace mongo { /* like init(), but for a reuse case */ DiskLoc reuse(const char *nsname, bool newUseIsAsCapped); - bool isOk() const { return magic == 0x41424344; } + bool isOk() const { return magic == extentSignature; } void assertOk() const { verify(isOk()); } Record* newRecord(int len); diff --git a/src/mongo/db/pipeline/document_source.cpp b/src/mongo/db/pipeline/document_source.cpp index 045ff4c3726..1d36eb30f37 100755 --- a/src/mongo/db/pipeline/document_source.cpp +++ b/src/mongo/db/pipeline/document_source.cpp @@ -87,11 +87,16 @@ namespace mongo { BSONObj DocumentSource::depsToProjection(const set<string>& deps) { BSONObjBuilder bb; - if (deps.count("_id") == 0) - bb.append("_id", 0); + + bool needId = false; string last; for (set<string>::const_iterator it(deps.begin()), end(deps.end()); it!=end; ++it) { + if (str::startsWith(*it, "_id") && (it->size() == 3 || (*it)[3] == '.')) { + // _id and subfields are handled specially due in part to SERVER-7502 + needId = true; + continue; + } if (!last.empty() && str::startsWith(*it, last)) { // we are including a parent of *it so we don't need to // include this field explicitly. In fact, due to @@ -102,6 +107,12 @@ namespace mongo { last = *it + '.'; bb.append(*it, 1); } + + if (needId) // we are explicit either way + bb.append("_id", 1); + else + bb.append("_id", 0); + return bb.obj(); } } diff --git a/src/mongo/db/pipeline/document_source.h b/src/mongo/db/pipeline/document_source.h index 329564ac490..e1e2cffcf96 100755 --- a/src/mongo/db/pipeline/document_source.h +++ b/src/mongo/db/pipeline/document_source.h @@ -965,7 +965,7 @@ namespace mongo { class DocumentSourceLimit : - public DocumentSource { + public SplittableDocumentSource { public: // virtuals from DocumentSource virtual ~DocumentSourceLimit(); @@ -988,6 +988,14 @@ namespace mongo { static intrusive_ptr<DocumentSourceLimit> create( const intrusive_ptr<ExpressionContext> &pExpCtx); + // Virtuals for SplittableDocumentSource + // Need to run on rounter. Running on shard as well is an optimization. + virtual intrusive_ptr<DocumentSource> getShardSource() { return this; } + virtual intrusive_ptr<DocumentSource> getRouterSource() { return this; } + + long long getLimit() const { return limit; } + void setLimit(long long newLimit) { limit = newLimit; } + /** Create a limiting DocumentSource from BSON. @@ -1019,7 +1027,7 @@ namespace mongo { }; class DocumentSourceSkip : - public DocumentSource { + public SplittableDocumentSource { public: // virtuals from DocumentSource virtual ~DocumentSourceSkip(); @@ -1042,6 +1050,14 @@ namespace mongo { static intrusive_ptr<DocumentSourceSkip> create( const intrusive_ptr<ExpressionContext> &pExpCtx); + // Virtuals for SplittableDocumentSource + // Need to run on rounter. Can't run on shards. + virtual intrusive_ptr<DocumentSource> getShardSource() { return NULL; } + virtual intrusive_ptr<DocumentSource> getRouterSource() { return this; } + + long long getSkip() const { return skip; } + void setSkip(long long newSkip) { skip = newSkip; } + /** Create a skipping DocumentSource from BSON. diff --git a/src/mongo/db/pipeline/document_source_limit.cpp b/src/mongo/db/pipeline/document_source_limit.cpp index 8bbcaff2113..48d12d5850f 100644 --- a/src/mongo/db/pipeline/document_source_limit.cpp +++ b/src/mongo/db/pipeline/document_source_limit.cpp @@ -27,9 +27,8 @@ namespace mongo { const char DocumentSourceLimit::limitName[] = "$limit"; - DocumentSourceLimit::DocumentSourceLimit( - const intrusive_ptr<ExpressionContext> &pExpCtx): - DocumentSource(pExpCtx), + DocumentSourceLimit::DocumentSourceLimit(const intrusive_ptr<ExpressionContext> &pExpCtx): + SplittableDocumentSource(pExpCtx), limit(0), count(0) { } diff --git a/src/mongo/db/pipeline/document_source_skip.cpp b/src/mongo/db/pipeline/document_source_skip.cpp index d4c1fc2caa6..5024b817a90 100644 --- a/src/mongo/db/pipeline/document_source_skip.cpp +++ b/src/mongo/db/pipeline/document_source_skip.cpp @@ -28,9 +28,8 @@ namespace mongo { const char DocumentSourceSkip::skipName[] = "$skip"; - DocumentSourceSkip::DocumentSourceSkip( - const intrusive_ptr<ExpressionContext> &pExpCtx): - DocumentSource(pExpCtx), + DocumentSourceSkip::DocumentSourceSkip(const intrusive_ptr<ExpressionContext> &pExpCtx): + SplittableDocumentSource(pExpCtx), skip(0), count(0) { } diff --git a/src/mongo/db/pipeline/pipeline.cpp b/src/mongo/db/pipeline/pipeline.cpp index 0bd4096fe78..52f725887fc 100644 --- a/src/mongo/db/pipeline/pipeline.cpp +++ b/src/mongo/db/pipeline/pipeline.cpp @@ -227,6 +227,35 @@ namespace mongo { } } + /* Move limits in front of skips. This is more optimal for sharding + * since currently, we can only split the pipeline at a single source + * and it is better to limit the results coming from each shard + */ + for(int i = pSourceVector->size() - 1; i >= 1 /* not looking at 0 */; i--) { + DocumentSourceLimit* limit = + dynamic_cast<DocumentSourceLimit*>((*pSourceVector)[i].get()); + DocumentSourceSkip* skip = + dynamic_cast<DocumentSourceSkip*>((*pSourceVector)[i-1].get()); + if (limit && skip) { + // Increase limit by skip since the skipped docs now pass through the $limit + limit->setLimit(limit->getLimit() + skip->getSkip()); + swap((*pSourceVector)[i], (*pSourceVector)[i-1]); + + // Start at back again. This is needed to handle cases with more than 1 $limit + // (S means skip, L means limit) + // + // These two would work without second pass (assuming back to front ordering) + // SL -> LS + // SSL -> LSS + // + // The following cases need a second pass to handle the second limit + // SLL -> LLS + // SSLL -> LLSS + // SLSL -> LLSS + i = pSourceVector->size(); // decremented before next pass + } + } + /* Coalesce adjacent filters where possible. Two adjacent filters are equivalent to one filter whose predicate is the conjunction of diff --git a/src/mongo/db/prefetch.cpp b/src/mongo/db/prefetch.cpp index 2af66b36cdf..776afc9c81d 100644 --- a/src/mongo/db/prefetch.cpp +++ b/src/mongo/db/prefetch.cpp @@ -135,7 +135,7 @@ namespace mongo { _dummy_char += *(result.objdata() + i); } // hit the last page, in case we missed it above - _dummy_char += *(result.objdata() + result.objsize()); + _dummy_char += *(result.objdata() + result.objsize() - 1); } } catch(const DBException& e) { diff --git a/src/mongo/db/repl/rs.cpp b/src/mongo/db/repl/rs.cpp index 7045aedf1c0..9bbbaf864de 100644 --- a/src/mongo/db/repl/rs.cpp +++ b/src/mongo/db/repl/rs.cpp @@ -84,10 +84,7 @@ namespace mongo { log() << "replSet See http://dochub.mongodb.org/core/resyncingaverystalereplicasetmember" << rsLog; // reset minvalid so that we can't become primary prematurely - { - Lock::DBWrite lk("local.replset.minvalid"); - Helpers::putSingleton("local.replset.minvalid", oldest); - } + setMinValid(oldest); sethbmsg("error RS102 too stale to catch up"); changeState(MemberState::RS_RECOVERING); @@ -426,6 +423,7 @@ namespace mongo { ghost(0), _writerPool(replWriterThreadCount), _prefetcherPool(replPrefetcherThreadCount), + oplogVersion(0), _indexPrefetchConfig(PREFETCH_ALL) { } @@ -551,6 +549,13 @@ namespace mongo { additive = false; } + // If we are changing chaining rules, we don't want this to be an additive reconfig so that + // the primary can step down and the sync targets change. + // TODO: This can be removed once SERVER-5208 is fixed. + if (reconf && config().chainingAllowed() != c.chainingAllowed()) { + additive = false; + } + _cfg = new ReplSetConfig(c); dassert( &config() == _cfg ); // config() is same thing but const, so we use that when we can for clarity below verify( config().ok() ); @@ -673,7 +678,16 @@ namespace mongo { try { vector<ReplSetConfig> configs; try { - configs.push_back( ReplSetConfig(HostAndPort::me()) ); + DBDirectClient cli; + BSONObj config = cli.findOne(rsConfigNs, Query()).getOwned(); + + // Add local config + if (config.isEmpty()) { + configs.push_back(ReplSetConfig()); + } + else { + configs.push_back(ReplSetConfig(config, false)); + } } catch(DBException& e) { log() << "replSet exception loading our local replset configuration object : " << e.toString() << rsLog; @@ -838,5 +852,13 @@ namespace mongo { cc().getAuthenticationInfo()->authorize("local","_repl"); } + void ReplSetImpl::setMinValid(BSONObj obj) { + BSONObjBuilder builder; + builder.appendTimestamp("ts", obj["ts"].date()); + builder.append("h", obj["h"]); + Lock::DBWrite cx( "local" ); + Helpers::putSingleton("local.replset.minvalid", builder.obj()); + } + } diff --git a/src/mongo/db/repl/rs.h b/src/mongo/db/repl/rs.h index e0425b42a6c..5bd41d11311 100644 --- a/src/mongo/db/repl/rs.h +++ b/src/mongo/db/repl/rs.h @@ -19,6 +19,7 @@ #pragma once #include "mongo/db/commands.h" +#include "mongo/db/index.h" #include "mongo/db/oplog.h" #include "mongo/db/oplogreader.h" #include "mongo/db/repl/rs_config.h" @@ -497,8 +498,8 @@ namespace mongo { private: bool _syncDoInitialSync_clone( const char *master, const list<string>& dbs , bool dataPass ); - bool _syncDoInitialSync_applyToHead( replset::InitialSync& init, OplogReader* r , - const Member* source, const BSONObj& lastOp, + bool _syncDoInitialSync_applyToHead( replset::SyncTail& syncer, OplogReader* r , + const Member* source, const BSONObj& lastOp, BSONObj& minValidOut); void _syncDoInitialSync(); void syncDoInitialSync(); @@ -538,7 +539,9 @@ namespace mongo { void syncRollback(OplogReader& r); void syncThread(); const OpTime lastOtherOpTime() const; - + static void setMinValid(BSONObj obj); + + int oplogVersion; private: IndexPrefetchConfig _indexPrefetchConfig; }; @@ -670,4 +673,31 @@ namespace mongo { _hbinfo.health = 1.0; } + inline bool ignoreUniqueIndex(IndexDetails& idx) { + if (!idx.unique()) { + return false; + } + if (!theReplSet) { + return false; + } + // see SERVER-6671 + MemberState ms = theReplSet->state(); + if (! ((ms == MemberState::RS_STARTUP2) || + (ms == MemberState::RS_RECOVERING) || + (ms == MemberState::RS_ROLLBACK))) { + return false; + } + // 2 is the oldest oplog version where operations + // are fully idempotent. + if (theReplSet->oplogVersion < 2) { + return false; + } + // Never ignore _id index + if (idx.isIdIndex()) { + return false; + } + + return true; + } + } diff --git a/src/mongo/db/repl/rs_config.cpp b/src/mongo/db/repl/rs_config.cpp index f0925ff51db..8d1f47c4f1e 100644 --- a/src/mongo/db/repl/rs_config.cpp +++ b/src/mongo/db/repl/rs_config.cpp @@ -130,6 +130,11 @@ namespace mongo { } if( !getLastErrorDefaults.isEmpty() ) settings << "getLastErrorDefaults" << getLastErrorDefaults; + + if (!_chainingAllowed) { + settings << "chainingAllowed" << _chainingAllowed; + } + b << "settings" << settings.obj(); } @@ -556,18 +561,32 @@ namespace mongo { ho.check(); try { getLastErrorDefaults = settings["getLastErrorDefaults"].Obj().copy(); } catch(...) { } + + // If the config explicitly sets chaining to false, turn it off. + if (settings.hasField("chainingAllowed") && + !settings["chainingAllowed"].trueValue()) { + _chainingAllowed = false; + } } // figure out the majority for this config setMajority(); } + bool ReplSetConfig::chainingAllowed() const { + return _chainingAllowed; + } + static inline void configAssert(bool expr) { uassert(13122, "bad repl set config?", expr); } + ReplSetConfig::ReplSetConfig() : + version(EMPTYCONFIG),_ok(false),_majority(-1) + {} + ReplSetConfig::ReplSetConfig(BSONObj cfg, bool force) : - _ok(false),_majority(-1) + _ok(false),_chainingAllowed(true),_majority(-1) { _constructed = false; clear(); @@ -583,7 +602,7 @@ namespace mongo { } ReplSetConfig::ReplSetConfig(const HostAndPort& h) : - _ok(false),_majority(-1) + _ok(false),_chainingAllowed(true),_majority(-1) { LOG(2) << "ReplSetConfig load " << h.toString() << rsLog; diff --git a/src/mongo/db/repl/rs_config.h b/src/mongo/db/repl/rs_config.h index 945b422bc0f..5702f3935bf 100644 --- a/src/mongo/db/repl/rs_config.h +++ b/src/mongo/db/repl/rs_config.h @@ -36,6 +36,7 @@ namespace mongo { // Protects _groups. static mongo::mutex groupMx; public: + ReplSetConfig(); /** * This contacts the given host and tries to get a config from them. * @@ -153,8 +154,20 @@ namespace mongo { int getMajority() const; bool _constructed; + + /** + * Returns if replication chaining is allowed. + */ + bool chainingAllowed() const; + private: bool _ok; + + /** + * If replication can be chained. If chaining is disallowed, it can still be explicitly + * enabled via the replSetSyncFrom command, but it will not happen automatically. + */ + bool _chainingAllowed; int _majority; void from(BSONObj); diff --git a/src/mongo/db/repl/rs_initialsync.cpp b/src/mongo/db/repl/rs_initialsync.cpp index eb56c634722..df37b96b08f 100644 --- a/src/mongo/db/repl/rs_initialsync.cpp +++ b/src/mongo/db/repl/rs_initialsync.cpp @@ -138,6 +138,8 @@ namespace mongo { return target; } + Member* primary = const_cast<Member*>(box.getPrimary()); + // wait for 2N pings before choosing a sync target if (_cfg) { int needMorePings = config().members.size()*2 - HeartbeatInfo::numPings; @@ -148,6 +150,12 @@ namespace mongo { } buildIndexes = myConfig().buildIndexes; + + // If we are only allowed to sync from the primary, return that + if (!_cfg->chainingAllowed()) { + // Returns NULL if we cannot reach the primary + return primary; + } } // find the member with the lowest ping time that has more data than me @@ -156,8 +164,7 @@ namespace mongo { // MAX_SLACK_TIME seconds behind. OpTime primaryOpTime; static const unsigned maxSlackDurationSeconds = 10 * 60; // 10 minutes - const Member* primary = box.getPrimary(); - if (primary) + if (primary) primaryOpTime = primary->hbinfo().opTime; else // choose a time that will exclude no candidates, since we don't see a primary @@ -204,8 +211,8 @@ namespace mongo { (m->hbinfo().ping > closest->hbinfo().ping)) continue; - if ( attempts == 0 && - myConfig().slaveDelay < m->config().slaveDelay ) { + if (attempts == 0 && + (myConfig().slaveDelay < m->config().slaveDelay || m->config().hidden)) { continue; // skip this one in the first attempt } @@ -248,8 +255,21 @@ namespace mongo { _veto[host] = time(0)+secs; } - bool ReplSetImpl::_syncDoInitialSync_applyToHead( replset::InitialSync& init, OplogReader* r, - const Member* source, const BSONObj& lastOp , + /** + * Replays the sync target's oplog from lastOp to the latest op on the sync target. + * + * @param syncer either initial sync (can reclone missing docs) or "normal" sync (no recloning) + * @param r the oplog reader + * @param source the sync target + * @param lastOp the op to start syncing at. replset::InitialSync writes this and then moves to + * the queue. replset::SyncTail does not write this, it moves directly to the + * queue. + * @param minValid populated by this function. The most recent op on the sync target's oplog, + * this function syncs to this value (inclusive) + * @return if applying the oplog succeeded + */ + bool ReplSetImpl::_syncDoInitialSync_applyToHead( replset::SyncTail& syncer, OplogReader* r, + const Member* source, const BSONObj& lastOp , BSONObj& minValid ) { /* our cloned copy will be strange until we apply oplog events that occurred through the process. we note that time point here. */ @@ -282,7 +302,7 @@ namespace mongo { // apply startingTS..mvoptime portion of the oplog { try { - init.oplogApplication(lastOp, minValid); + minValid = syncer.oplogApplication(lastOp, minValid); } catch (const DBException&) { log() << "replSet initial sync failed during oplog application phase" << rsLog; @@ -310,10 +330,26 @@ namespace mongo { } /** - * Do the initial sync for this member. + * Do the initial sync for this member. There are several steps to this process: + * + * 1. Record start time. + * 2. Clone. + * 3. Set minValid1 to sync target's latest op time. + * 4. Apply ops from start to minValid1, fetching missing docs as needed. + * 5. Set minValid2 to sync target's latest op time. + * 6. Apply ops from minValid1 to minValid2. + * 7. Build indexes. + * 8. Set minValid3 to sync target's latest op time. + * 9. Apply ops from minValid2 to minValid3. + * + * At that point, initial sync is finished. Note that the oplog from the sync target is applied + * three times: step 4, 6, and 8. 4 may involve refetching, 6 should not. By the end of 6, + * this member should have consistent data. 8 is "cosmetic," it is only to get this member + * closer to the latest op time before it can transition to secondary state. */ void ReplSetImpl::_syncDoInitialSync() { replset::InitialSync init(replset::BackgroundSync::get()); + replset::SyncTail tail(replset::BackgroundSync::get()); sethbmsg("initial sync pending",0); // if this is the first node, it may have already become primary @@ -345,6 +381,9 @@ namespace mongo { return; } + // written by applyToHead calls + BSONObj minValid; + if (replSettings.fastsync) { log() << "fastsync: skipping database clone" << rsLog; @@ -367,28 +406,24 @@ namespace mongo { } sethbmsg("initial sync data copy, starting syncup",0); - - BSONObj minValid; + + log() << "oplog sync 1 of 3" << endl; if ( ! _syncDoInitialSync_applyToHead( init, &r , source , lastOp , minValid ) ) { return; } lastOp = minValid; - // its currently important that lastOp is equal to the last op we actually pulled - // this is because the background thread only pulls each op once now - // so if its now, we'll be waiting forever - { - // this takes whatever the last op the we got is - // and stores it locally before we wipe it out below - Lock::DBRead lk(rsoplog); - Helpers::getLast(rsoplog, lastOp); - lastOp = lastOp.getOwned(); + + // Now we sync to the latest op on the sync target _again_, as we may have recloned ops + // that were "from the future" compared with minValid. During this second application, + // nothing should need to be recloned. + log() << "oplog sync 2 of 3" << endl; + if (!_syncDoInitialSync_applyToHead(tail, &r , source , lastOp , minValid)) { + return; } + // data should now be consistent - // reset state, as that "didn't count" - emptyOplog(); - lastOpTimeWritten = OpTime(); - lastH = 0; + lastOp = minValid; sethbmsg("initial sync building indexes",0); if ( ! _syncDoInitialSync_clone( sourceHostname.c_str(), dbs, false ) ) { @@ -398,10 +433,8 @@ namespace mongo { } } - sethbmsg("initial sync query minValid",0); - - BSONObj minValid; - if ( ! _syncDoInitialSync_applyToHead( init, &r, source, lastOp, minValid ) ) { + log() << "oplog sync 3 of 3" << endl; + if (!_syncDoInitialSync_applyToHead(tail, &r, source, lastOp, minValid)) { return; } @@ -419,7 +452,9 @@ namespace mongo { log() << "replSet set minValid=" << minValid["ts"]._opTime().toString() << rsLog; } catch(...) { } - Helpers::putSingleton("local.replset.minvalid", minValid); + + theReplSet->setMinValid(minValid); + cx.ctx().db()->flushFiles(true); } diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 9d7718445a9..c912116f9d5 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -304,18 +304,6 @@ namespace mongo { bson::bo goodVersionOfObject; }; - static void setMinValid(bo newMinValid) { - try { - log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog; - } - catch(...) { } - { - Helpers::putSingleton("local.replset.minvalid", newMinValid); - Client::Context cx( "local." ); - cx.db()->flushFiles(true); - } - } - void ReplSetImpl::syncFixUp(HowToFixUp& h, OplogReader& r) { DBClientConnection *them = r.conn(); @@ -378,6 +366,7 @@ namespace mongo { /* we have items we are writing that aren't from a point-in-time. thus best not to come online until we get to that point in freshness. */ + log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog; setMinValid(newMinValid); /** any full collection resyncs required? */ @@ -411,6 +400,7 @@ namespace mongo { err = "can't get minvalid from primary"; } else { + log() << "replSet minvalid=" << newMinValid["ts"]._opTime().toStringLong() << rsLog; setMinValid(newMinValid); } } @@ -583,6 +573,23 @@ namespace mongo { } void ReplSetImpl::syncRollback(OplogReader&r) { + // check that we are at minvalid, otherwise we cannot rollback as we may be in an + // inconsistent state + { + Lock::DBRead lk("local.replset.minvalid"); + BSONObj mv; + if( Helpers::getSingleton("local.replset.minvalid", mv) ) { + OpTime minvalid = mv["ts"]._opTime(); + if( minvalid > lastOpTimeWritten ) { + log() << "replSet need to rollback, but in inconsistent state" << endl; + log() << "minvalid: " << minvalid.toString() << " our last optime: " + << lastOpTimeWritten.toString() << endl; + changeState(MemberState::RS_FATAL); + return; + } + } + } + unsigned s = _syncRollback(r); if( s ) sleepsecs(s); diff --git a/src/mongo/db/repl/rs_sync.cpp b/src/mongo/db/repl/rs_sync.cpp index f7fdaf55e96..3c71b075bd3 100644 --- a/src/mongo/db/repl/rs_sync.cpp +++ b/src/mongo/db/repl/rs_sync.cpp @@ -39,7 +39,7 @@ namespace mongo { namespace replset { SyncTail::SyncTail(BackgroundSyncInterface *q) : - Sync(""), _networkQueue(q) + Sync(""), oplogVersion(0), _networkQueue(q) {} SyncTail::~SyncTail() {} @@ -109,11 +109,16 @@ namespace replset { // This free function is used by the writer threads to apply each op void multiSyncApply(const std::vector<BSONObj>& ops, SyncTail* st) { initializeWriterThread(); + + // convert update operations only for 2.2.1 or greater, because we need guaranteed + // idempotent operations for this to work. See SERVER-6825 + bool convertUpdatesToUpserts = theReplSet->oplogVersion > 1 ? true : false; + for (std::vector<BSONObj>::const_iterator it = ops.begin(); it != ops.end(); ++it) { try { - fassert(16359, st->syncApply(*it, true)); + fassert(16359, st->syncApply(*it, convertUpdatesToUpserts)); } catch (DBException& e) { error() << "writer worker caught exception: " << e.what() << " on: " << it->toString() << endl; @@ -145,13 +150,6 @@ namespace replset { } } catch (DBException& e) { - // Skip duplicate key exceptions. - // These are relatively common on initial sync: if a document is inserted - // early in the clone step, the insert will be replayed but the document - // will probably already have been cloned over. - if( e.getCode() == 11000 || e.getCode() == 11001 || e.getCode() == 12582) { - return; // ignore - } error() << "exception: " << e.what() << " on: " << it->toString() << endl; fassertFailed(16361); } @@ -247,49 +245,46 @@ namespace replset { InitialSync::~InitialSync() {} - - /* initial oplog application, during initial sync, after cloning. - */ - void InitialSync::oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj) { + BSONObj SyncTail::oplogApplySegment(const BSONObj& applyGTEObj, const BSONObj& minValidObj, + MultiSyncApplyFunc func) { OpTime applyGTE = applyGTEObj["ts"]._opTime(); OpTime minValid = minValidObj["ts"]._opTime(); - if (replSetForceInitialSyncFailure > 0) { - log() << "replSet test code invoked, forced InitialSync failure: " << replSetForceInitialSyncFailure << rsLog; - replSetForceInitialSyncFailure--; - throw DBException("forced error",0); - } - - syncApply(applyGTEObj); - _logOpObjRS(applyGTEObj); - - - // if there were no writes during the initial sync, there will be nothing in the queue so - // just go live - if (minValid == applyGTE) { - return; - } + // We have to keep track of the last op applied to the data, because there's no other easy + // way of getting this data synchronously. Batches may go past minValidObj, so we need to + // know to bump minValid past minValidObj. + BSONObj lastOp = applyGTEObj; + OpTime ts = applyGTE; - OpTime ts; time_t start = time(0); + time_t now = start; + unsigned long long n = 0, lastN = 0; - + while( ts < minValid ) { OpQueue ops; - while (ops.getSize() < replBatchSizeBytes) { + while (ops.getSize() < replBatchLimitBytes) { if (tryPopAndWaitForMore(&ops)) { break; } - } + // apply replication batch limits + now = time(0); + if (!ops.empty()) { + if (now > replBatchLimitSeconds) + break; + if (ops.getDeque().size() > replBatchLimitOperations) + break; + } + } + setOplogVersion(ops.getDeque().front()); - multiApply(ops.getDeque(), multiInitialSyncApply); + multiApply(ops.getDeque(), func); n += ops.getDeque().size(); if ( n > lastN + 1000 ) { - time_t now = time(0); if (now - start > 10) { // simple progress metering log() << "replSet initialSyncOplogApplication applied " << n << " operations, synced to " @@ -300,32 +295,77 @@ namespace replset { } // we want to keep a record of the last op applied, to compare with minvalid - const BSONObj& lastOp = ops.getDeque().back(); + lastOp = ops.getDeque().back(); OpTime tempTs = lastOp["ts"]._opTime(); applyOpsToOplog(&ops.getDeque()); ts = tempTs; } + + return lastOp; + } + + /* initial oplog application, during initial sync, after cloning. + */ + BSONObj InitialSync::oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj) { + if (replSetForceInitialSyncFailure > 0) { + log() << "replSet test code invoked, forced InitialSync failure: " << replSetForceInitialSyncFailure << rsLog; + replSetForceInitialSyncFailure--; + throw DBException("forced error",0); + } + + // create the initial oplog entry + syncApply(applyGTEObj); + _logOpObjRS(applyGTEObj); + + return oplogApplySegment(applyGTEObj, minValidObj, multiInitialSyncApply); + } + + BSONObj SyncTail::oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj) { + return oplogApplySegment(applyGTEObj, minValidObj, multiSyncApply); + } + + void SyncTail::setOplogVersion(const BSONObj& op) { + BSONElement version = op["v"]; + // old primaries do not get the unique index ignoring feature + // because some of their ops are not imdepotent, see + // SERVER-7186 + if (version.eoo()) { + theReplSet->oplogVersion = 1; + RARELY log() << "warning replset primary is an older version than we are; upgrade recommended" << endl; + } else { + theReplSet->oplogVersion = version.Int(); + } } /* tail an oplog. ok to return, will be re-called. */ void SyncTail::oplogApplication() { while( 1 ) { OpQueue ops; - time_t lastTimeChecked = time(0); verify( !Lock::isLocked() ); + Timer batchTimer; + int lastTimeChecked = 0; + // always fetch a few ops first - // tryPopAndWaitForMore returns true when we need to end a batch early while (!tryPopAndWaitForMore(&ops) && - (ops.getSize() < replBatchSizeBytes)) { + (ops.getSize() < replBatchLimitBytes)) { if (theReplSet->isPrimary()) { return; } - time_t now = time(0); + + int now = batchTimer.seconds(); + + // apply replication batch limits + if (!ops.empty()) { + if (now > replBatchLimitSeconds) + break; + if (ops.getDeque().size() > replBatchLimitOperations) + break; + } // occasionally check some things if (ops.empty() || now > lastTimeChecked) { lastTimeChecked = now; @@ -349,17 +389,31 @@ namespace replset { return; } } + + const int slaveDelaySecs = theReplSet->myConfig().slaveDelay; + if (!ops.empty() && slaveDelaySecs > 0) { + const BSONObj& lastOp = ops.getDeque().back(); + const unsigned int opTimestampSecs = lastOp["ts"]._opTime().getSecs(); + + // Stop the batch as the lastOp is too new to be applied. If we continue + // on, we can get ops that are way ahead of the delay and this will + // make this thread sleep longer when handleSlaveDelay is called + // and apply ops much sooner than we like. + if (opTimestampSecs > static_cast<unsigned int>(time(0) - slaveDelaySecs)) { + break; + } + } } + const BSONObj& lastOp = ops.getDeque().back(); + setOplogVersion(lastOp); handleSlaveDelay(lastOp); // Set minValid to the last op to be applied in this next batch. // This will cause this node to go into RECOVERING state // if we should crash and restart before updating the oplog - { - Client::WriteContext cx( "local" ); - Helpers::putSingleton("local.replset.minvalid", lastOp); - } + theReplSet->setMinValid(lastOp); + multiApply(ops.getDeque(), multiSyncApply); applyOpsToOplog(&ops.getDeque()); @@ -381,7 +435,7 @@ namespace replset { if (!peek_success) { // if we don't have anything in the queue, wait a bit for something to appear if (ops->empty()) { - // block 1 second + // block up to 1 second _networkQueue->waitForMore(); return false; } @@ -389,6 +443,7 @@ namespace replset { // otherwise, apply what we have return true; } + // check for commands if ((op["op"].valuestrsafe()[0] == 'c') || // Index builds are acheived through the use of an insert op, not a command op. @@ -404,6 +459,28 @@ namespace replset { return true; } + // check for oplog version change + BSONElement elemVersion = op["v"]; + int curVersion = 0; + if (elemVersion.eoo()) + // missing version means version 1 + curVersion = 1; + else + curVersion = elemVersion.Int(); + + if (curVersion != oplogVersion) { + // Version changes cause us to end a batch. + // If we are starting a new batch, reset version number + // and continue. + if (ops->empty()) { + oplogVersion = curVersion; + } + else { + // End batch early + return true; + } + } + // Copy the op to the deque and remove it from the bgsync queue. ops->push_back(op); _networkQueue->consume(); diff --git a/src/mongo/db/repl/rs_sync.h b/src/mongo/db/repl/rs_sync.h index f47538d9742..da0a66069d9 100644 --- a/src/mongo/db/repl/rs_sync.h +++ b/src/mongo/db/repl/rs_sync.h @@ -20,6 +20,7 @@ #include <vector> #include "mongo/db/client.h" +#include "mongo/db/dur.h" #include "mongo/db/jsobj.h" #include "mongo/db/oplog.h" #include "mongo/util/concurrency/thread_pool.h" @@ -38,6 +39,31 @@ namespace replset { SyncTail(BackgroundSyncInterface *q); virtual ~SyncTail(); virtual bool syncApply(const BSONObj &o, bool convertUpdateToUpsert = false); + + /** + * Apply ops from applyGTEObj's ts to at least minValidObj's ts. Note that, due to + * batching, this may end up applying ops beyond minValidObj's ts. + * + * @param applyGTEObj the op to start replicating at. This is actually not used except in + * comparision to minValidObj: the background sync thread keeps its own + * record of where we're synced to and starts providing ops from that + * point. + * @param minValidObj the op to finish syncing at. This function cannot return (other than + * fatally erroring out) without applying at least this op. + * @param func whether this should use initial sync logic (recloning docs) or + * "normal" logic. + * @return BSONObj the op that was synced to. This may be greater than minValidObj, as a + * single batch might blow right by minvalid. If applyGTEObj is the same + * op as minValidObj, this will be applyGTEObj. + */ + BSONObj oplogApplySegment(const BSONObj& applyGTEObj, const BSONObj& minValidObj, + MultiSyncApplyFunc func); + + /** + * Runs oplogApplySegment without allowing recloning documents. + */ + virtual BSONObj oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj); + void oplogApplication(); bool peek(BSONObj* obj); @@ -69,12 +95,19 @@ namespace replset { void applyOpsToOplog(std::deque<BSONObj>* ops); protected: - static const unsigned int replBatchSizeBytes = 1024 * 1024 * 256 ; + // Cap the batches using the limit on journal commits. + // This works out to be 100 MB (64 bit) or 50 MB (32 bit) + static const unsigned int replBatchLimitBytes = dur::UncommittedBytesLimit; + static const int replBatchLimitSeconds = 1; + static const unsigned int replBatchLimitOperations = 5000; // Prefetch and write a deque of operations, using the supplied function. // Initial Sync and Sync Tail each use a different function. void multiApply(std::deque<BSONObj>& ops, MultiSyncApplyFunc applyFunc); + // The version of the last op to be read + int oplogVersion; + private: BackgroundSyncInterface* _networkQueue; @@ -90,6 +123,7 @@ namespace replset { void fillWriterVectors(const std::deque<BSONObj>& ops, std::vector< std::vector<BSONObj> >* writerVectors); void handleSlaveDelay(const BSONObj& op); + void setOplogVersion(const BSONObj& op); }; /** @@ -99,7 +133,12 @@ namespace replset { public: virtual ~InitialSync(); InitialSync(BackgroundSyncInterface *q); - void oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj); + + /** + * Creates the initial oplog entry: applies applyGTEObj and writes it to the oplog. Then + * this runs oplogApplySegment allowing recloning documents. + */ + BSONObj oplogApplication(const BSONObj& applyGTEObj, const BSONObj& minValidObj); }; // TODO: move hbmsg into an error-keeping class (SERVER-4444) diff --git a/src/mongo/db/security_commands.cpp b/src/mongo/db/security_commands.cpp index 2023e5f656c..6dbbe3dabec 100644 --- a/src/mongo/db/security_commands.cpp +++ b/src/mongo/db/security_commands.cpp @@ -88,8 +88,8 @@ namespace mongo { { bool reject = false; - nonce64 *ln = lastNonce.release(); - if ( ln == 0 ) { + scoped_ptr<nonce64> ln(lastNonce.release()); + if ( !ln ) { reject = true; log(1) << "auth: no lastNonce" << endl; } diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index e2c145b9db2..76aa3165de3 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -38,7 +38,7 @@ namespace mongo { static string secondsExpireField; void doTTLForDB( const string& dbName ) { - + Client::GodScope god; vector<BSONObj> indexes; @@ -117,6 +117,10 @@ namespace mongo { continue; } + // if part of replSet but not in a readable state (e.g. during initial sync), skip. + if ( theReplSet && !theReplSet->state().readable() ) + continue; + set<string> dbs; { Lock::DBRead lk( "local" ); diff --git a/src/mongo/dbtests/basictests.cpp b/src/mongo/dbtests/basictests.cpp index ded62a13049..428cd7fa32a 100644 --- a/src/mongo/dbtests/basictests.cpp +++ b/src/mongo/dbtests/basictests.cpp @@ -267,7 +267,7 @@ namespace BasicTests { int maxSleepTimeMillis = 1000; int lastSleepTimeMillis = -1; - int epsMillis = 50; // Allowable inprecision for timing + int epsMillis = 100; // Allowable inprecision for timing Backoff backoff( maxSleepTimeMillis, maxSleepTimeMillis * 2 ); diff --git a/src/mongo/dbtests/documentsourcetests.cpp b/src/mongo/dbtests/documentsourcetests.cpp index d0734bc7066..f5336334f9a 100644 --- a/src/mongo/dbtests/documentsourcetests.cpp +++ b/src/mongo/dbtests/documentsourcetests.cpp @@ -44,6 +44,59 @@ namespace DocumentSourceTests { } }; + namespace DocumentSourceClass { + using mongo::DocumentSource; + + template<size_t ArrayLen> + set<string> arrayToSet(const char* (&array) [ArrayLen]) { + set<string> out; + for (size_t i = 0; i < ArrayLen; i++) + out.insert(array[i]); + return out; + } + + class Deps { + public: + void run() { + { + const char* array[] = {"a", "b"}; // basic + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "b" << 1 << "_id" << 0)); + } + { + const char* array[] = {"a", "ab"}; // prefixed but not subfield + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "ab" << 1 << "_id" << 0)); + } + { + const char* array[] = {"a", "b", "a.b"}; // a.b included by a + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "b" << 1 << "_id" << 0)); + } + { + const char* array[] = {"a", "_id"}; // _id now included + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "_id" << 1)); + } + { + const char* array[] = {"a", "_id.a"}; // still include whole _id (SERVER-7502) + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "_id" << 1)); + } + { + const char* array[] = {"a", "_id", "_id.a"}; // handle both _id and subfield + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("a" << 1 << "_id" << 1)); + } + { + const char* array[] = {"a", "_id", "_id_a"}; // _id prefixed but non-subfield + BSONObj proj = DocumentSource::depsToProjection(arrayToSet(array)); + ASSERT_EQUALS(proj, BSON("_id_a" << 1 << "a" << 1 << "_id" << 1)); + } + } + }; + } + namespace DocumentSourceCursor { using mongo::DocumentSourceCursor; @@ -1690,6 +1743,8 @@ namespace DocumentSourceTests { All() : Suite( "documentsource" ) { } void setupTests() { + add<DocumentSourceClass::Deps>(); + add<DocumentSourceCursor::Create>(); add<DocumentSourceCursor::Iterate>(); add<DocumentSourceCursor::Dispose>(); diff --git a/src/mongo/dbtests/jsontests.cpp b/src/mongo/dbtests/jsontests.cpp index 82b9c280390..82e6963767d 100644 --- a/src/mongo/dbtests/jsontests.cpp +++ b/src/mongo/dbtests/jsontests.cpp @@ -158,6 +158,17 @@ namespace JsonTests { } }; + class SingleUndefinedMember { + public: + void run() { + BSONObjBuilder b; + b.appendUndefined( "a" ); + ASSERT_EQUALS( "{ \"a\" : { \"$undefined\" : true } }", b.done().jsonString( Strict ) ); + ASSERT_EQUALS( "{ \"a\" : undefined }", b.done().jsonString( JS ) ); + ASSERT_EQUALS( "{ \"a\" : undefined }", b.done().jsonString( TenGen ) ); + } + }; + class SingleObjectMember { public: void run() { @@ -1108,6 +1119,7 @@ namespace JsonTests { add< JsonStringTests::NegativeNumber >(); add< JsonStringTests::SingleBoolMember >(); add< JsonStringTests::SingleNullMember >(); + add< JsonStringTests::SingleUndefinedMember >(); add< JsonStringTests::SingleObjectMember >(); add< JsonStringTests::TwoMembers >(); add< JsonStringTests::EmptyArray >(); diff --git a/src/mongo/dbtests/replica_set_monitor_test.cpp b/src/mongo/dbtests/replica_set_monitor_test.cpp index 358035cc82b..d3525574061 100644 --- a/src/mongo/dbtests/replica_set_monitor_test.cpp +++ b/src/mongo/dbtests/replica_set_monitor_test.cpp @@ -22,353 +22,286 @@ #include <vector> #include "mongo/client/dbclient_rs.h" -#include "mongo/dbtests/dbtests.h" +#include "mongo/unittest/unittest.h" namespace { using std::vector; using boost::scoped_ptr; using mongo::BSONObj; + using mongo::BSONArray; + using mongo::BSONArrayBuilder; using mongo::ReplicaSetMonitor; using mongo::HostAndPort; using mongo::ReadPreference; using mongo::TagSet; - const BSONObj SampleIsMasterDoc = BSON( "tags" - << BSON( "dc" << "NYC" - << "p" << "2" - << "region" << "NA" )); - const BSONObj NoTagIsMasterDoc = BSON( "isMaster" << true ); + const BSONObj SampleIsMasterDoc = BSON("tags" + << BSON("dc" << "NYC" + << "p" << "2" + << "region" << "NA")); + const BSONObj NoTagIsMasterDoc = BSON("isMaster" << true); - class SimpleGoodMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = BSON( "tags" << BSON( "dc" << "sf" )); - ASSERT( node.matchesTag( BSON( "dc" << "sf" ))); - } - }; + TEST(ReplSetMonitorNode, SimpleGoodMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = BSON("tags" << BSON("dc" << "sf")); + ASSERT(node.matchesTag(BSON("dc" << "sf"))); + } - class SimpleBadMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = BSON( "tags" << BSON( "dc" << "nyc" )); - ASSERT( !node.matchesTag( BSON( "dc" << "sf" ))); - } - }; - - class ExactMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( node.matchesTag( SampleIsMasterDoc["tags"].Obj() )); - } - }; + TEST(ReplSetMonitorNode, SimpleBadMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = BSON("tags" << BSON("dc" << "nyc")); + ASSERT(!node.matchesTag(BSON("dc" << "sf"))); + } - class EmptyTagTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( node.matchesTag( BSONObj() )); - } - }; + TEST(ReplSetMonitorNode, ExactMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.matchesTag(SampleIsMasterDoc["tags"].Obj())); + } - class MemberNoTagMatchesEmptyTagTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = NoTagIsMasterDoc; - ASSERT( node.matchesTag( BSONObj() )); - } - }; + TEST(ReplSetMonitorNode, EmptyTag) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.matchesTag(BSONObj())); + } - class MemberNoTagDoesNotMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = NoTagIsMasterDoc.copy(); - ASSERT( !node.matchesTag( BSON( "dc" << "NYC" ) )); - } - }; + TEST(ReplSetMonitorNode, MemberNoTagMatchesEmptyTag) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = NoTagIsMasterDoc; + ASSERT(node.matchesTag(BSONObj())); + } - class IncompleteMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( !node.matchesTag( BSON( "dc" << "NYC" - << "p" << "2" - << "hello" << "world" ) )); - } - }; + TEST(ReplSetMonitorNode, MemberNoTagDoesNotMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = NoTagIsMasterDoc.copy(); + ASSERT(!node.matchesTag(BSON("dc" << "NYC"))); + } - class PartialMatchTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( node.matchesTag( BSON( "dc" << "NYC" - << "p" << "2" ))); - } - }; + TEST(ReplSetMonitorNode, IncompleteMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.matchesTag(BSON("dc" << "NYC" + << "p" << "2" + << "hello" << "world"))); + } - class SingleTagCritTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( node.matchesTag( BSON( "p" << "2" ))); - } - }; + TEST(ReplSetMonitorNode, PartialMatch) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.matchesTag(BSON("dc" << "NYC" + << "p" << "2"))); + } - class BadSingleTagCritTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( !node.matchesTag( BSON( "dc" << "SF" ))); - } - }; + TEST(ReplSetMonitorNode, SingleTagCrit) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.matchesTag(BSON("p" << "2"))); + } - class NonExistingFieldTagTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( !node.matchesTag( BSON( "noSQL" << "Mongo" ))); - } - }; + TEST(ReplSetMonitorNode, BadSingleTagCrit) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.matchesTag(BSON("dc" << "SF"))); + } - class UnorederedMatchingTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( node.matchesTag( BSON( "p" << "2" << "dc" << "NYC" ))); - } - }; + TEST(ReplSetMonitorNode, NonExistingFieldTag) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.matchesTag(BSON("noSQL" << "Mongo"))); + } - class SameValueDiffKeyTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort(), NULL ); + TEST(ReplSetMonitorNode, UnorederedMatching) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); node.lastIsMaster = SampleIsMasterDoc.copy(); - ASSERT( !node.matchesTag( BSON( "datacenter" << "NYC" ))); - } - }; + ASSERT(node.matchesTag(BSON("p" << "2" << "dc" << "NYC"))); + } - class SimpleToStringTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + TEST(ReplSetMonitorNode, SameValueDiffKey) { + ReplicaSetMonitor::Node node(HostAndPort(), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.matchesTag(BSON("datacenter" << "NYC"))); + } - // Should not throw any exceptions - ASSERT( !node.toString().empty() ); - } - }; + TEST(ReplSetMonitorNode, SimpleToString) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - class SimpleToStringWithNoTagTest { - public: - void run() { - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = NoTagIsMasterDoc.copy(); + // Should not throw any exceptions + ASSERT(!node.toString().empty()); + } - // Should not throw any exceptions - ASSERT( !node.toString().empty() ); - } - }; + TEST(ReplSetMonitorNode, SimpleToStringWithNoTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = NoTagIsMasterDoc.copy(); - class PriNodeCompatibleTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + // Should not throw any exceptions + ASSERT(!node.toString().empty()); + } - node.ok = true; - node.ismaster = true; - node.secondary = false; + TEST(ReplSetMonitorNode, PriNodeCompatibleTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "NYC" )); + node.ok = true; + node.ismaster = true; + node.secondary = false; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "NYC")); - ASSERT( node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class SecNodeCompatibleTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = false; - node.secondary = true; + TEST(ReplSetMonitorNode, SecNodeCompatibleTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "NYC" )); + node.ok = true; + node.ismaster = false; + node.secondary = true; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "NYC")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class PriNodeNotCompatibleTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = true; - node.secondary = false; + TEST(ReplSetMonitorNode, PriNodeNotCompatibleTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "SF" )); + node.ok = true; + node.ismaster = true; + node.secondary = false; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "SF")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class SecNodeNotCompatibleTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = false; - node.secondary = true; + TEST(ReplSetMonitorNode, SecNodeNotCompatibleTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "SF" )); + node.ok = true; + node.ismaster = false; + node.secondary = true; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "SF")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class PriNodeCompatiblMultiTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = true; - node.secondary = false; + TEST(ReplSetMonitorNode, PriNodeCompatiblMultiTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "RP" )); - builder.append( BSON( "dc" << "NYC" << "p" << "2" )); + node.ok = true; + node.ismaster = true; + node.secondary = false; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "RP")); + builder.append(BSON("dc" << "NYC" << "p" << "2")); - ASSERT( node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class SecNodeCompatibleMultiTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = false; - node.secondary = true; + TEST(ReplSetMonitorNode, SecNodeCompatibleMultiTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "RP" )); - builder.append( BSON( "dc" << "NYC" << "p" << "2" )); + node.ok = true; + node.ismaster = false; + node.secondary = true; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "RP")); + builder.append(BSON("dc" << "NYC" << "p" << "2")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class PriNodeNotCompatibleMultiTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = true; - node.secondary = false; + TEST(ReplSetMonitorNode, PriNodeNotCompatibleMultiTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "sf" )); - builder.append( BSON( "dc" << "NYC" << "P" << "4" )); + node.ok = true; + node.ismaster = true; + node.secondary = false; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "sf")); + builder.append(BSON("dc" << "NYC" << "P" << "4")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); - class SecNodeNotCompatibleMultiTagTest { - public: - void run(){ - ReplicaSetMonitor::Node node( HostAndPort( "dummy", 3 ), NULL ); - node.lastIsMaster = SampleIsMasterDoc.copy(); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } - node.ok = true; - node.ismaster = false; - node.secondary = true; + TEST(ReplSetMonitorNode, SecNodeNotCompatibleMultiTag) { + ReplicaSetMonitor::Node node(HostAndPort("dummy", 3), NULL); + node.lastIsMaster = SampleIsMasterDoc.copy(); - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "sf" )); - builder.append( BSON( "dc" << "NYC" << "P" << "4" )); + node.ok = true; + node.ismaster = false; + node.secondary = true; - TagSet tags( BSONArray( builder.done() )); + BSONArrayBuilder builder; + builder.append(BSON("dc" << "sf")); + builder.append(BSON("dc" << "NYC" << "P" << "4")); - ASSERT( !node.isCompatible( ReadPreference_PrimaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_PrimaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryPreferred, &tags )); - ASSERT( !node.isCompatible( ReadPreference_SecondaryOnly, &tags )); - ASSERT( !node.isCompatible( ReadPreference_Nearest, &tags )); - } - }; + TagSet tags(BSONArray(builder.done())); + + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_PrimaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryPreferred, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_SecondaryOnly, &tags)); + ASSERT(!node.isCompatible(mongo::ReadPreference_Nearest, &tags)); + } class NodeSetFixtures { public: @@ -378,9 +311,9 @@ namespace { vector<ReplicaSetMonitor::Node> NodeSetFixtures::getThreeMemberWithTags() { vector<ReplicaSetMonitor::Node> nodes; - nodes.push_back( ReplicaSetMonitor::Node( HostAndPort( "a" ), NULL )); - nodes.push_back( ReplicaSetMonitor::Node( HostAndPort( "b" ), NULL )); - nodes.push_back( ReplicaSetMonitor::Node( HostAndPort( "c" ), NULL )); + nodes.push_back(ReplicaSetMonitor::Node(HostAndPort("a"), NULL)); + nodes.push_back(ReplicaSetMonitor::Node(HostAndPort("b"), NULL)); + nodes.push_back(ReplicaSetMonitor::Node(HostAndPort("c"), NULL)); nodes[0].ok = true; nodes[1].ok = true; @@ -390,9 +323,9 @@ namespace { nodes[1].ismaster = true; nodes[2].secondary = true; - nodes[0].lastIsMaster = BSON( "tags" << BSON( "dc" << "nyc" << "p" << "1" )); - nodes[1].lastIsMaster = BSON( "tags" << BSON( "dc" << "sf" )); - nodes[2].lastIsMaster = BSON( "tags" << BSON( "dc" << "nyc" << "p" << "2" )); + nodes[0].lastIsMaster = BSON("tags" << BSON("dc" << "nyc" << "p" << "1")); + nodes[1].lastIsMaster = BSON("tags" << BSON("dc" << "sf")); + nodes[2].lastIsMaster = BSON("tags" << BSON("dc" << "nyc" << "p" << "2")); return nodes; } @@ -407,1319 +340,1139 @@ namespace { BSONArray TagSetFixtures::getDefaultSet() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSONObj() ); + arrayBuilder.append(BSONObj()); return arrayBuilder.arr(); } BSONArray TagSetFixtures::getP2Tag() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "2" ) ); + arrayBuilder.append(BSON("p" << "2")); return arrayBuilder.arr(); } BSONArray TagSetFixtures::getSingleNoMatchTag() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "k" << "x" ) ); + arrayBuilder.append(BSON("k" << "x")); return arrayBuilder.arr(); } BSONArray TagSetFixtures::getMultiNoMatchTag() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "mongo" << "db" ) ); - arrayBuilder.append( BSON( "by" << "10gen" ) ); + arrayBuilder.append(BSON("mongo" << "db")); + arrayBuilder.append(BSON("by" << "10gen")); return arrayBuilder.arr(); } - class PrimaryOnlyTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TEST(ReplSetMonitorReadPref, PrimaryOnly) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class PrimaryOnlyPriNotOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TEST(ReplSetMonitorReadPref, PrimaryOnlyPriNotOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class PrimaryMissingTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TEST(ReplSetMonitorReadPref, PrimaryMissing) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - nodes[1].ismaster = false; + nodes[1].ismaster = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class PriPrefWithPriOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); + TEST(ReplSetMonitorReadPref, PriPrefWithPriOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class PriPrefWithPriNotOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, PriPrefWithPriNotOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecOnlyTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecOnly) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecOnlyOnlyPriOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecOnlyOnlyPriOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[0].ok = false; - nodes[2].ok = false; + nodes[0].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecPrefTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecPref) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefWithNoSecOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecPrefWithNoSecOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[0].ok = false; - nodes[2].ok = false; + nodes[0].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class SecPrefWithNoNodeOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecPrefWithNoNodeOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[1].addr; - nodes[0].ok = false; - nodes[1].ok = false; - nodes[2].ok = false; + nodes[0].ok = false; + nodes[1].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 1, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 1, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class NearestTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TEST(ReplSetMonitorReadPref, Nearest) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - nodes[0].pingTimeMillis = 1; - nodes[1].pingTimeMillis = 2; - nodes[2].pingTimeMillis = 3; + nodes[0].pingTimeMillis = 1; + nodes[1].pingTimeMillis = 2; + nodes[2].pingTimeMillis = 3; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = 0; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class NearestNoLocalTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getDefaultSet() ); - HostAndPort lastHost = nodes[0].addr; + TEST(ReplSetMonitorReadPref, NearestNoLocal) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getDefaultSet()); + HostAndPort lastHost = nodes[0].addr; - nodes[0].pingTimeMillis = 10; - nodes[1].pingTimeMillis = 20; - nodes[2].pingTimeMillis = 30; + nodes[0].pingTimeMillis = 10; + nodes[1].pingTimeMillis = 20; + nodes[2].pingTimeMillis = 30; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( !host.empty() ); - } - }; + ASSERT(!host.empty()); + } - class PriOnlyWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getP2Tag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, PriOnlyWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getP2Tag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - // Note: PrimaryOnly ignores tag - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + // Note: PrimaryOnly ignores tag + ASSERT_EQUALS("b", host.host()); + } - class PriPrefPriNotOkWithTagsTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getP2Tag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, PriPrefPriNotOkWithTags) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getP2Tag()); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class PriPrefPriOkWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, PriPrefPriOkWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class PriPrefPriNotOkWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, PriPrefPriNotOkWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecOnlyWithTagsTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getP2Tag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecOnlyWithTags) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getP2Tag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecOnlyWithTagsMatchOnlyPriTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecOnlyWithTagsMatchOnlyPri) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[2].addr; - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "dc" << "sf" )); - TagSet tags( arrayBuilder.arr() ); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("dc" << "sf")); + TagSet tags(arrayBuilder.arr()); - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecPrefWithTagsTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getP2Tag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecPrefWithTags) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getP2Tag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecPrefSecNotOkWithTagsTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, SecPrefSecNotOkWithTags) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[1].addr; - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "dc" << "nyc" )); - TagSet tags( arrayBuilder.arr() ); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("dc" << "nyc")); + TagSet tags(arrayBuilder.arr()); - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefPriOkWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecPrefPriOkWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class SecPrefPriNotOkWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecPrefPriNotOkWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecPrefPriOkWithSecNotMatchTagTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(ReplSetMonitorReadPref, SecPrefPriOkWithSecNotMatchTag) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class NearestWithTagsTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, NearestWithTags) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[1].addr; - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "1" )); - TagSet tags( arrayBuilder.arr() ); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("p" << "1")); + TagSet tags(arrayBuilder.arr()); - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class NearestWithTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getSingleNoMatchTag() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, NearestWithTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getSingleNoMatchTag()); + HostAndPort lastHost = nodes[1].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class MultiPriOnlyTagTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, MultiPriOnlyTag) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[1].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class MultiPriOnlyPriNotOkTagTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[1].addr; + TEST(ReplSetMonitorReadPref, MultiPriOnlyPriNotOkTag) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[1].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class PriPrefPriOkWithMultiTags { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); + TEST(ReplSetMonitorReadPref, PriPrefPriOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "1" )); - arrayBuilder.append( BSON( "p" << "2" )); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("p" << "1")); + arrayBuilder.append(BSON("p" << "2")); - TagSet tags( arrayBuilder.arr() ); - HostAndPort lastHost = nodes[2].addr; + TagSet tags(arrayBuilder.arr()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class MultiTagsMatchesFirstTest { + class MultiTags: public mongo::unittest::Test { public: - MultiTagsMatchesFirstTest() { - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "1" )); - arrayBuilder.append( BSON( "p" << "2" )); - - tags.reset( new TagSet( arrayBuilder.arr() )); - } - - virtual ~MultiTagsMatchesFirstTest() {} - vector<ReplicaSetMonitor::Node> getNodes() const { return NodeSetFixtures::getThreeMemberWithTags(); } - TagSet* getTagSet() { - return tags.get(); - } - - private: - scoped_ptr<TagSet> tags; - }; + TagSet* getMatchesFirstTagSet() { + if (matchFirstTags.get() != NULL) { + return matchFirstTags.get(); + } - class MultiTagsMatchesSecondTest { - public: - MultiTagsMatchesSecondTest() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "3" )); - arrayBuilder.append( BSON( "p" << "2" )); - arrayBuilder.append( BSON( "p" << "1" )); - - tags.reset( new TagSet( arrayBuilder.arr() )); - } - - virtual ~MultiTagsMatchesSecondTest() {}; + arrayBuilder.append(BSON("p" << "1")); + arrayBuilder.append(BSON("p" << "2")); + matchFirstTags.reset(new TagSet(arrayBuilder.arr())); - vector<ReplicaSetMonitor::Node> getNodes() const { - return NodeSetFixtures::getThreeMemberWithTags(); + return matchFirstTags.get(); } - TagSet* getTagSet() { - return tags.get(); - } - - private: - scoped_ptr<TagSet> tags; - }; + TagSet* getMatchesSecondTagSet() { + if (matchSecondTags.get() != NULL) { + return matchSecondTags.get(); + } - class MultiTagsMatchesLastTest { - public: - MultiTagsMatchesLastTest() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "12" )); - arrayBuilder.append( BSON( "p" << "23" )); - arrayBuilder.append( BSON( "p" << "19" )); - arrayBuilder.append( BSON( "p" << "34" )); - arrayBuilder.append( BSON( "p" << "1" )); + arrayBuilder.append(BSON("p" << "3")); + arrayBuilder.append(BSON("p" << "2")); + arrayBuilder.append(BSON("p" << "1")); + matchSecondTags.reset(new TagSet(arrayBuilder.arr())); - tags.reset( new TagSet( arrayBuilder.arr() )); + return matchSecondTags.get(); } - virtual ~MultiTagsMatchesLastTest() {} + TagSet* getMatchesLastTagSet() { + if (matchLastTags.get() != NULL) { + return matchLastTags.get(); + } - vector<ReplicaSetMonitor::Node> getNodes() const { - return NodeSetFixtures::getThreeMemberWithTags(); - } + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("p" << "12")); + arrayBuilder.append(BSON("p" << "23")); + arrayBuilder.append(BSON("p" << "19")); + arrayBuilder.append(BSON("p" << "34")); + arrayBuilder.append(BSON("p" << "1")); + matchLastTags.reset(new TagSet(arrayBuilder.arr())); - TagSet* getTagSet() { - return tags.get(); + return matchLastTags.get(); } - private: - scoped_ptr<TagSet> tags; - }; + TagSet* getMatchesPriTagSet() { + if (matchPriTags.get() != NULL) { + return matchPriTags.get(); + } - class MultiTagsMatchesPriTest { - public: - MultiTagsMatchesPriTest() { BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "dc" << "sf" )); - arrayBuilder.append( BSON( "p" << "1" )); - tags.reset( new TagSet( arrayBuilder.arr() )); - } - - virtual ~MultiTagsMatchesPriTest() {}; + arrayBuilder.append(BSON("dc" << "sf")); + arrayBuilder.append(BSON("p" << "1")); + matchPriTags.reset(new TagSet(arrayBuilder.arr())); - vector<ReplicaSetMonitor::Node> getNodes() const { - return NodeSetFixtures::getThreeMemberWithTags(); - } - - TagSet* getTagSet() { - return tags.get(); + return matchPriTags.get(); } private: - scoped_ptr<TagSet> tags; + scoped_ptr<TagSet> matchFirstTags; + scoped_ptr<TagSet> matchSecondTags; + scoped_ptr<TagSet> matchLastTags; + scoped_ptr<TagSet> matchPriTags; }; - class PriPrefPriNotOkWithMultiTagsMatchesFirstTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, MultiTagsMatchesFirst) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class PriPrefPriNotOkWithMultiTagsMatchesFirstNotOkTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, PriPrefPriNotOkMatchesFirstNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; - nodes[1].ok = false; + nodes[0].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class PriPrefPriNotOkWithMultiTagsMatchesSecondTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, PriPrefPriNotOkMatchesSecondTest) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class PriPrefPriNotOkWithMultiTagsMatchesSecondNotOkTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, PriPrefPriNotOkMatchesSecondNotOkTest) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; - nodes[2].ok = false; + nodes[1].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class PriPrefPriNotOkWithMultiTagsMatchesLastTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, PriPrefPriNotOkMatchesLastTest) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class PriPrefPriNotOkWithMultiTagsMatchesLastNotOkTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, PriPrefPriNotOkMatchesLastNotOkTest) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; - nodes[1].ok = false; + nodes[0].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class PriPrefPriOkWithMultiTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); + TEST(MultiTags, PriPrefPriOkNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class PriPrefPriNotOkWithMultiTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(MultiTags, PriPrefPriNotOkNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_PrimaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_PrimaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecOnlyWithMultiTagsMatchesFirstTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesFirstTest) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecOnlyWithMultiTagsMatchesFirstNotOkTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesFirstNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecOnlyWithMultiTagsMatchesSecondTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesSecond) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecOnlyWithMultiTagsMatchesSecondNotOkTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesSecondNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecOnlyWithMultiTagsMatchesLastTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesLast) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecOnlyWithMultiTagsMatchesLastNotOkTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMatchesLastNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecOnlyMultiTagsWithPriMatchTest : public MultiTagsMatchesPriTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMultiTagsWithPriMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, getMatchesPriTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecOnlyMultiTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecOnlyMultiTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryOnly, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryOnly, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SecPrefWithMultiTagsMatchesFirstTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesFirst) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefWithMultiTagsMatchesFirstNotOkTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesFirstNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecPrefWithMultiTagsMatchesSecondTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesSecond) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class SecPrefWithMultiTagsMatchesSecondNotOkTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesSecondNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesSecondTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefWithMultiTagsMatchesLastTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesLast) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefWithMultiTagsMatchesLastNotOkTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMatchesLastNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesLastTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class SecPrefMultiTagsWithPriMatchTest : public MultiTagsMatchesPriTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, SecPrefMultiTagsWithPriMatch) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, getMatchesPriTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class SecPrefMultiTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(MultiTags, SecPrefMultiTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + } - class SecPrefMultiTagsNoMatchPriNotOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(MultiTags, SecPrefMultiTagsNoMatchPriNotOk) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - nodes[1].ok = false; + nodes[1].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_SecondaryPreferred, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_SecondaryPreferred, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class NearestWithMultiTagsMatchesFirstTest : public MultiTagsMatchesFirstTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NearestMatchesFirst) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, getMatchesFirstTagSet(), + 3, &lastHost, &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class NearestWithMultiTagsMatchesFirstNotOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = NodeSetFixtures::getThreeMemberWithTags(); + TEST(MultiTags, NearestMatchesFirstNotOk) { + vector<ReplicaSetMonitor::Node> nodes = NodeSetFixtures::getThreeMemberWithTags(); - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "p" << "1" )); - arrayBuilder.append( BSON( "dc" << "sf" )); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("p" << "1")); + arrayBuilder.append(BSON("dc" << "sf")); - TagSet tags( arrayBuilder.arr() ); - HostAndPort lastHost = nodes[2].addr; + TagSet tags(arrayBuilder.arr()); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class NearestWithMultiTagsMatchesSecondTest : public MultiTagsMatchesSecondTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NearestMatchesSecond) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, getMatchesSecondTagSet(), 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "c", host.host() ); - ASSERT_EQUALS("c", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("c", host.host()); + ASSERT_EQUALS("c", lastHost.host()); + } - class NearestWithMultiTagsMatchesSecondNotOkTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NearestMatchesSecondNotOk) { + vector<ReplicaSetMonitor::Node> nodes = NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[2].addr; - BSONArrayBuilder arrayBuilder; - arrayBuilder.append( BSON( "z" << "2" )); - arrayBuilder.append( BSON( "p" << "2" )); - arrayBuilder.append( BSON( "dc" << "sf" )); + BSONArrayBuilder arrayBuilder; + arrayBuilder.append(BSON("z" << "2")); + arrayBuilder.append(BSON("p" << "2")); + arrayBuilder.append(BSON("dc" << "sf")); - TagSet tags( arrayBuilder.arr() ); + TagSet tags(arrayBuilder.arr()); - nodes[2].ok = false; + nodes[2].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class NearestWithMultiTagsMatchesLastTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NearestMatchesLast) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, getMatchesLastTagSet(), 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "a", host.host() ); - ASSERT_EQUALS("a", lastHost.host()); - } - }; + ASSERT(!isPrimarySelected); + ASSERT_EQUALS("a", host.host()); + ASSERT_EQUALS("a", lastHost.host()); + } - class NeatestWithMultiTagsMatchesLastNotOkTest : public MultiTagsMatchesLastTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = getNodes(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NeatestMatchesLastNotOk) { + vector<ReplicaSetMonitor::Node> nodes = getNodes(); + HostAndPort lastHost = nodes[2].addr; - nodes[0].ok = false; + nodes[0].ok = false; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, getMatchesLastTagSet(), 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class NearestMultiTagsWithPriMatchTest : public MultiTagsMatchesPriTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - HostAndPort lastHost = nodes[2].addr; + TEST_F(MultiTags, NearestMultiTagsWithPriMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, getTagSet(), 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, getMatchesPriTagSet(), 3, &lastHost, + &isPrimarySelected); - ASSERT_EQUALS( "b", host.host() ); - ASSERT_EQUALS("b", lastHost.host()); - } - }; + ASSERT(isPrimarySelected); + ASSERT_EQUALS("b", host.host()); + ASSERT_EQUALS("b", lastHost.host()); + } - class NearestMultiTagsNoMatchTest { - public: - void run() { - vector<ReplicaSetMonitor::Node> nodes = - NodeSetFixtures::getThreeMemberWithTags(); - TagSet tags( TagSetFixtures::getMultiNoMatchTag() ); - HostAndPort lastHost = nodes[2].addr; + TEST(TagSet, NearestMultiTagsNoMatch) { + vector<ReplicaSetMonitor::Node> nodes = + NodeSetFixtures::getThreeMemberWithTags(); + TagSet tags(TagSetFixtures::getMultiNoMatchTag()); + HostAndPort lastHost = nodes[2].addr; - HostAndPort host = ReplicaSetMonitor::selectNode( nodes, - ReadPreference_Nearest, &tags, 3, &lastHost ); + bool isPrimarySelected = false; + HostAndPort host = ReplicaSetMonitor::selectNode(nodes, + mongo::ReadPreference_Nearest, &tags, 3, &lastHost, + &isPrimarySelected); - ASSERT( host.empty() ); - } - }; + ASSERT(host.empty()); + } - class SingleTagSetTest { - public: - void run(){ - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "nyc" )); + TEST(TagSet, SingleTagSet) { + BSONArrayBuilder builder; + builder.append(BSON("dc" << "nyc")); - TagSet tags( BSONArray( builder.done() )); + TagSet tags(BSONArray(builder.done())); - ASSERT( !tags.isExhausted() ); - ASSERT( tags.getCurrentTag().equal( BSON( "dc" << "nyc" )) ); + ASSERT(!tags.isExhausted()); + ASSERT(tags.getCurrentTag().equal(BSON("dc" << "nyc"))); - ASSERT( !tags.isExhausted() ); - tags.next(); + ASSERT(!tags.isExhausted()); + tags.next(); - ASSERT( tags.isExhausted() ); + ASSERT(tags.isExhausted()); #if !(defined(_DEBUG) || defined(_DURABLEDEFAULTON) || defined(_DURABLEDEFAULTOFF)) - // TODO: remove this guard once SERVER-6317 is fixed - ASSERT_THROWS( tags.getCurrentTag(), AssertionException ); + // TODO: remove this guard once SERVER-6317 is fixed + ASSERT_THROWS(tags.getCurrentTag(), mongo::AssertionException); #endif - } - }; + } - class MultiTagSetTest { - public: - void run(){ - BSONArrayBuilder builder; - builder.append( BSON( "dc" << "nyc" )); - builder.append( BSON( "dc" << "sf" )); - builder.append( BSON( "dc" << "ma" )); + TEST(TagSet, MultiTagSet) { + BSONArrayBuilder builder; + builder.append(BSON("dc" << "nyc")); + builder.append(BSON("dc" << "sf")); + builder.append(BSON("dc" << "ma")); - TagSet tags( BSONArray( builder.done() )); + TagSet tags(BSONArray(builder.done())); - ASSERT( !tags.isExhausted() ); - ASSERT( tags.getCurrentTag().equal( BSON( "dc" << "nyc" )) ); + ASSERT(!tags.isExhausted()); + ASSERT(tags.getCurrentTag().equal(BSON("dc" << "nyc"))); - ASSERT( !tags.isExhausted() ); - tags.next(); - ASSERT( tags.getCurrentTag().equal( BSON( "dc" << "sf" )) ); + ASSERT(!tags.isExhausted()); + tags.next(); + ASSERT(tags.getCurrentTag().equal(BSON("dc" << "sf"))); - ASSERT( !tags.isExhausted() ); - tags.next(); - ASSERT( tags.getCurrentTag().equal( BSON( "dc" << "ma" )) ); + ASSERT(!tags.isExhausted()); + tags.next(); + ASSERT(tags.getCurrentTag().equal(BSON("dc" << "ma"))); - ASSERT( !tags.isExhausted() ); - tags.next(); + ASSERT(!tags.isExhausted()); + tags.next(); - ASSERT( tags.isExhausted() ); + ASSERT(tags.isExhausted()); #if !(defined(_DEBUG) || defined(_DURABLEDEFAULTON) || defined(_DURABLEDEFAULTOFF)) - // TODO: remove this guard once SERVER-6317 is fixed - ASSERT_THROWS( tags.getCurrentTag(), AssertionException ); + // TODO: remove this guard once SERVER-6317 is fixed + ASSERT_THROWS(tags.getCurrentTag(), mongo::AssertionException); #endif - } - }; + } - class EmptyArrayTagsTest { - public: - void run() { - BSONArray emptyArray; - TagSet tags( emptyArray ); + TEST(TagSet, EmptyArrayTags) { + BSONArray emptyArray; + TagSet tags(emptyArray); - ASSERT( tags.isExhausted() ); + ASSERT(tags.isExhausted()); #if !(defined(_DEBUG) || defined(_DURABLEDEFAULTON) || defined(_DURABLEDEFAULTOFF)) - // TODO: remove this guard once SERVER-6317 is fixed - ASSERT_THROWS( tags.getCurrentTag(), AssertionException ); + // TODO: remove this guard once SERVER-6317 is fixed + ASSERT_THROWS(tags.getCurrentTag(), mongo::AssertionException); #endif - } - }; - - class AllNodeSuite : public Suite { - public: - AllNodeSuite() : Suite( "replicaSetMonitor_node" ){ - } - - void setupTests(){ - add< SimpleGoodMatchTest >(); - add< SimpleBadMatchTest >(); - add< ExactMatchTest >(); - add< EmptyTagTest >(); - add< MemberNoTagMatchesEmptyTagTest >(); - add< MemberNoTagDoesNotMatchTest >(); - add< IncompleteMatchTest >(); - add< PartialMatchTest >(); - add< SingleTagCritTest >(); - add< BadSingleTagCritTest >(); - add< NonExistingFieldTagTest >(); - add< UnorederedMatchingTest >(); - add< SameValueDiffKeyTest >(); - add< SimpleToStringTest >(); - add< SimpleToStringWithNoTagTest >(); - - add< PriNodeCompatibleTagTest >(); - add< SecNodeCompatibleTagTest >(); - add< PriNodeNotCompatibleTagTest >(); - add< SecNodeNotCompatibleTagTest >(); - add< PriNodeCompatiblMultiTagTest >(); - add< SecNodeCompatibleMultiTagTest >(); - add< PriNodeNotCompatibleMultiTagTest >(); - add< SecNodeNotCompatibleMultiTagTest >(); - } - } allNode; - - class AllNodeSelectorSuite : public Suite { - public: - AllNodeSelectorSuite() : Suite( "replicaSetMonitor_select_node" ){ - } - - void setupTests() { - add< PrimaryOnlyTest >(); - add< PrimaryOnlyPriNotOkTest >(); - add< PriOnlyWithTagsNoMatchTest >(); - add< PrimaryMissingTest >(); - add< MultiPriOnlyTagTest >(); - add< MultiPriOnlyPriNotOkTagTest >(); - - add< PriPrefWithPriOkTest >(); - add< PriPrefWithPriNotOkTest >(); - add< PriPrefPriNotOkWithTagsTest >(); - add< PriPrefPriOkWithTagsNoMatchTest >(); - add< PriPrefPriNotOkWithTagsNoMatchTest >(); - add< PriPrefPriOkWithMultiTags >(); - - add< PriPrefPriNotOkWithMultiTagsMatchesFirstTest >(); - add< PriPrefPriNotOkWithMultiTagsMatchesFirstNotOkTest >(); - add< PriPrefPriNotOkWithMultiTagsMatchesSecondTest >(); - add< PriPrefPriNotOkWithMultiTagsMatchesSecondNotOkTest >(); - add< PriPrefPriNotOkWithMultiTagsMatchesLastTest >(); - add< PriPrefPriNotOkWithMultiTagsMatchesLastNotOkTest >(); - add< PriPrefPriOkWithMultiTagsNoMatchTest >(); - add< PriPrefPriNotOkWithMultiTagsNoMatchTest >(); - - add< SecOnlyTest >(); - add< SecOnlyOnlyPriOkTest >(); - add< SecOnlyWithTagsTest >(); - add< SecOnlyWithTagsMatchOnlyPriTest >(); - - add< SecOnlyWithMultiTagsMatchesFirstTest >(); - add< SecOnlyWithMultiTagsMatchesFirstNotOkTest >(); - add< SecOnlyWithMultiTagsMatchesSecondTest >(); - add< SecOnlyWithMultiTagsMatchesSecondNotOkTest >(); - add< SecOnlyWithMultiTagsMatchesLastTest >(); - add< SecOnlyWithMultiTagsMatchesLastNotOkTest >(); - add< SecOnlyMultiTagsWithPriMatchTest >(); - add< SecOnlyMultiTagsNoMatchTest >(); - - add< SecPrefTest >(); - add< SecPrefWithNoSecOkTest >(); - add< SecPrefWithNoNodeOkTest >(); - add< SecPrefWithTagsTest >(); - add< SecPrefSecNotOkWithTagsTest >(); - add< SecPrefPriOkWithTagsNoMatchTest >(); - add< SecPrefPriNotOkWithTagsNoMatchTest >(); - add< SecPrefPriOkWithSecNotMatchTagTest >(); - - add< SecPrefWithMultiTagsMatchesFirstTest >(); - add< SecPrefWithMultiTagsMatchesFirstNotOkTest >(); - add< SecPrefWithMultiTagsMatchesSecondTest >(); - add< SecPrefWithMultiTagsMatchesSecondNotOkTest >(); - add< SecPrefWithMultiTagsMatchesLastTest >(); - add< SecPrefWithMultiTagsMatchesLastNotOkTest >(); - add< SecPrefMultiTagsWithPriMatchTest >(); - add< SecPrefMultiTagsNoMatchTest >(); - add< SecPrefMultiTagsNoMatchPriNotOkTest >(); - - add< NearestTest >(); - add< NearestNoLocalTest >(); - add< NearestWithTagsTest >(); - add< NearestWithTagsNoMatchTest >(); - - add< NearestWithMultiTagsMatchesFirstTest >(); - add< NearestWithMultiTagsMatchesFirstNotOkTest >(); - add< NearestWithMultiTagsMatchesSecondTest >(); - add< NearestWithMultiTagsMatchesSecondNotOkTest >(); - add< NearestWithMultiTagsMatchesLastTest >(); - add< NearestMultiTagsWithPriMatchTest >(); - add< NearestMultiTagsNoMatchTest >(); - } - } allNodeSelectorSuite; - - class TagSetSuite : public Suite { - public: - TagSetSuite() : Suite( "tagSet" ) { - } - - void setupTests() { - add< SingleTagSetTest >(); - add< MultiTagSetTest >(); - add< EmptyArrayTagsTest >(); - } - } tagSetSuite; + } } - diff --git a/src/mongo/dbtests/replsettests.cpp b/src/mongo/dbtests/replsettests.cpp index 74343af2b07..7b4acfead21 100644 --- a/src/mongo/dbtests/replsettests.cpp +++ b/src/mongo/dbtests/replsettests.cpp @@ -407,7 +407,8 @@ namespace ReplSetTests { class TestRSSync : public Base { - void addOp(const string& op, BSONObj o, BSONObj* o2 = 0, const char* coll = 0) { + void addOp(const string& op, BSONObj o, BSONObj* o2 = NULL, const char* coll = NULL, + int version = 0) { OpTime ts; { Lock::GlobalWrite lk; @@ -416,6 +417,9 @@ namespace ReplSetTests { BSONObjBuilder b; b.appendTimestamp("ts", ts.asLL()); + if (version != 0) { + b.append("v", version); + } b.append("op", op); b.append("o", o); @@ -439,6 +443,12 @@ namespace ReplSetTests { } } + void addVersionedInserts(int expected) { + for (int i=0; i < expected; i++) { + addOp("i", BSON("_id" << i << "x" << 789), NULL, NULL, i); + } + } + void addUpdates() { BSONObj id = BSON("_id" << "123456something"); addOp("i", id); @@ -456,6 +466,18 @@ namespace ReplSetTests { "timestamp" << 1334810820))), &id); } + void addConflictingUpdates() { + BSONObj first = BSON("_id" << "asdfasdfasdf"); + addOp("i", first); + + BSONObj filter = BSON("_id" << "asdfasdfasdf" << "sp" << BSON("$size" << 2)); + // Test an op with no version, op is ignored and replication continues (code assumes + // version 1) + addOp("u", BSON("$push" << BSON("sp" << 42)), &filter, NULL, 0); + // The following line generates an fassert because it's version 2 + //addOp("u", BSON("$push" << BSON("sp" << 42)), &filter, NULL, 2); + } + void addUniqueIndex() { addOp("i", BSON("ns" << ns() << "key" << BSON("x" << 1) << "name" << "x1" << "unique" << true), 0, "unittests.system.indexes"); addInserts(2); @@ -475,6 +497,12 @@ namespace ReplSetTests { ASSERT_EQUALS(expected, static_cast<int>(client()->count(ns()))); drop(); + addVersionedInserts(100); + applyOplog(); + + ASSERT_EQUALS(expected, static_cast<int>(client()->count(ns()))); + + drop(); addUpdates(); applyOplog(); @@ -485,6 +513,14 @@ namespace ReplSetTests { ASSERT_EQUALS(1334810820, obj["requests"]["100002_1"]["timestamp"].number()); drop(); + + // test converting updates to upserts but only for version 2.2.1 and greater, + // which means oplog version 2 and greater. + addConflictingUpdates(); + applyOplog(); + + drop(); + } }; diff --git a/src/mongo/dbtests/repltests.cpp b/src/mongo/dbtests/repltests.cpp index 23ae7680092..754556ae540 100644 --- a/src/mongo/dbtests/repltests.cpp +++ b/src/mongo/dbtests/repltests.cpp @@ -815,6 +815,23 @@ namespace ReplTests { } }; + class PushWithDollarSigns : public Base { + void doIt() const { + client()->update( ns(), + BSON( "_id" << 0), + BSON( "$push" << BSON( "a" << BSON( "$foo" << 1 ) ) ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{'_id':0, a:[0, {'$foo':1}]}"), one( fromjson( "{'_id':0}" ) ) ); + } + void reset() const { + deleteAll( ns() ); + insert( BSON( "_id" << 0 << "a" << BSON_ARRAY( 0 ) ) ); + } + }; + class PushAllUpsert : public Base { public: void doIt() const { @@ -1009,6 +1026,138 @@ namespace ReplTests { } }; + class NestedNoRename : public Base { + public: + void doIt() const { + client()->update( ns(), BSON( "_id" << 0 ), + fromjson( "{$rename:{'a.b':'c.d'},$set:{z:1}}" + ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( BSON( "_id" << 0 << "z" << 1 ) , one( fromjson("{'_id':0}" ) ) ); + } + void reset() const { + deleteAll( ns() ); + insert( fromjson( "{'_id':0}" ) ); + } + }; + + class SingletonNoRename : public Base { + public: + void doIt() const { + client()->update( ns(), BSONObj(), fromjson("{$rename:{a:'b'}}" ) ); + + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{_id:0,z:1}" ), one(fromjson("{'_id':0}" ) ) ); + } + void reset() const { + deleteAll( ns() ); + insert( fromjson( "{'_id':0,z:1}" ) ); + } + }; + + class IndexedSingletonNoRename : public Base { + public: + void doIt() const { + client()->update( ns(), BSONObj(), fromjson("{$rename:{a:'b'}}" ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{_id:0,z:1}" ), one(fromjson("{'_id':0}" ) ) ); + } + void reset() const { + deleteAll( ns() ); + // Add an index on 'a'. This prevents the update from running 'in place'. + client()->ensureIndex( ns(), BSON( "a" << 1 ) ); + insert( fromjson( "{'_id':0,z:1}" ) ); + } + }; + + class AddToSetEmptyMissing : public Base { + public: + void doIt() const { + client()->update( ns(), BSON( "_id" << 0 ), fromjson( + "{$addToSet:{a:{$each:[]}}}" ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{_id:0,a:[]}" ), one( fromjson("{'_id':0}" ) ) + ); + } + void reset() const { + deleteAll( ns() ); + insert( fromjson( "{'_id':0}" ) ); + } + }; + + class AddToSetWithDollarSigns : public Base { + void doIt() const { + client()->update( ns(), + BSON( "_id" << 0), + BSON( "$addToSet" << BSON( "a" << BSON( "$foo" << 1 ) ) ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{'_id':0, a:[0, {'$foo':1}]}"), one( fromjson( "{'_id':0}" ) ) ); + } + void reset() const { + deleteAll( ns() ); + insert( BSON( "_id" << 0 << "a" << BSON_ARRAY( 0 ) ) ); + } + }; + + // + // replay cases + // + + class ReplaySetPreexistingNoOpPull : public Base { + public: + void doIt() const { + client()->update( ns(), BSONObj(), fromjson( "{$unset:{z:1}}" )); + + // This is logged as {$set:{'a.b':[]},$set:{z:1}}, which might not be + // replayable against future versions of a document (here {_id:0,a:1,z:1}) due + // to SERVER-4781. As a result the $set:{z:1} will not be replayed in such + // cases (and also an exception may abort replication). If this were instead + // logged as {$set:{z:1}}, SERVER-4781 would not be triggered. + client()->update( ns(), BSONObj(), fromjson( "{$pull:{'a.b':1}, $set:{z:1}}" ) ); + client()->update( ns(), BSONObj(), fromjson( "{$set:{a:1}}" ) ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{_id:0,a:1,z:1}" ), one( fromjson("{'_id':0}") ) ); + } + void reset() const { + deleteAll( ns() ); + insert( fromjson( "{'_id':0,a:{b:[]},z:1}" ) ); + } + }; + + class ReplayArrayFieldNotAppended : public Base { + public: + void doIt() const { + client()->update( ns(), BSONObj(), fromjson( "{$push:{'a.0.b':2}}" ) ); + client()->update( ns(), BSONObj(), fromjson( "{$set:{'a.0':1}}") ); + } + using ReplTests::Base::check; + void check() const { + ASSERT_EQUALS( 1, count() ); + check( fromjson( "{_id:0,a:[1,{b:[1]}]}" ), one(fromjson("{'_id':0}") ) ); + } + void reset() const { + deleteAll( ns() ); + insert( fromjson( "{'_id':0,a:[{b:[0]},{b:[1]}]}" ) ); + } + }; } // namespace Idempotence @@ -1218,6 +1367,7 @@ namespace ReplTests { add< Idempotence::EmptyPush >(); add< Idempotence::EmptyPushSparseIndex >(); add< Idempotence::PushAll >(); + add< Idempotence::PushWithDollarSigns >(); add< Idempotence::PushAllUpsert >(); add< Idempotence::EmptyPushAll >(); add< Idempotence::Pull >(); @@ -1230,6 +1380,13 @@ namespace ReplTests { add< Idempotence::RenameReplace >(); add< Idempotence::RenameOverwrite >(); add< Idempotence::NoRename >(); + add< Idempotence::NestedNoRename >(); + add< Idempotence::SingletonNoRename >(); + add< Idempotence::IndexedSingletonNoRename >(); + add< Idempotence::AddToSetEmptyMissing >(); + add< Idempotence::AddToSetWithDollarSigns >(); + add< Idempotence::ReplaySetPreexistingNoOpPull >(); + add< Idempotence::ReplayArrayFieldNotAppended >(); add< DeleteOpIsIdBased >(); add< DatabaseIgnorerBasic >(); add< DatabaseIgnorerUpdate >(); diff --git a/src/mongo/dbtests/updatetests.cpp b/src/mongo/dbtests/updatetests.cpp index e916299924a..89fe9a57f0a 100644 --- a/src/mongo/dbtests/updatetests.cpp +++ b/src/mongo/dbtests/updatetests.cpp @@ -640,8 +640,23 @@ namespace UpdateTests { test( BSON( "$push" << BSON( "a" << 5 ) ) , fromjson( "{a:[1]}" ) , fromjson( "{a:[1,5]}" ) ); } }; - - class IncRewrite { + + + class IncRewriteInPlace { + public: + void run() { + BSONObj obj = BSON( "a" << 2 ); + BSONObj mod = BSON( "$inc" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << 3 ) ), modSetState->getOpLogRewrite() ); + } + }; + + // Check if not applying in place changes anything. + class InRewriteForceNotInPlace { public: void run() { BSONObj obj = BSON( "a" << 2 ); @@ -649,11 +664,10 @@ namespace UpdateTests { ModSet modSet( mod ); auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); modSetState->createNewFromMods(); - ASSERT( modSetState->needOpLogRewrite() ); ASSERT_EQUALS( BSON( "$set" << BSON( "a" << 3 ) ), modSetState->getOpLogRewrite() ); } }; - + class IncRewriteNestedArray { public: void run() { @@ -661,13 +675,423 @@ namespace UpdateTests { BSONObj mod = BSON( "$inc" << BSON( "a.0" << 1 ) ); ModSet modSet( mod ); auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); - modSetState->createNewFromMods(); - ASSERT( modSetState->needOpLogRewrite() ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); ASSERT_EQUALS( BSON( "$set" << BSON( "a.0" << 3 ) ), - modSetState->getOpLogRewrite() ); + modSetState->getOpLogRewrite() ); + } + }; + + class IncRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << 2 ); + BSONObj mod = BSON( "$inc" << BSON( "a" << 1 ) << "$set" << BSON( "b" << 2) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << 3 ) << "$set" << BSON("b" << 2)), + modSetState->getOpLogRewrite() ); + } + }; + + class IncRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "c" << 1 ); + BSONObj mod = BSON( "$inc" << BSON( "a" << 1 ) << "$set" << BSON( "b" << 2) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << 1 ) << "$set" << BSON("b" << 2)), + modSetState->getOpLogRewrite() ); + } + }; + + // Push is never applied in place + class PushRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj mod = BSON( "$push" << BSON( "a" << 3 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a.2" << 3 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class PushRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "b" << 1 ); + BSONObj mod = BSON( "$push" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 ) ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class PushAllRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj modAll = BSON( "$pushAll" << BSON( "a" << BSON_ARRAY( 3 << 4 << 5 ) ) ); + ModSet modSetAll( modAll ); + auto_ptr<ModSetState> modSetStateAll = modSetAll.prepare( obj ); + ASSERT_FALSE( modSetStateAll->canApplyInPlace() ); + modSetStateAll->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 << 2 << 3 << 4 << 5) ) ), + modSetStateAll->getOpLogRewrite() ); + } + }; + + class PushAllRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "b" << 1 ); + BSONObj modAll = BSON( "$pushAll" << BSON( "a" << BSON_ARRAY( 1 << 2 << 3) ) ); + ModSet modSetAll( modAll ); + auto_ptr<ModSetState> modSetStateAll = modSetAll.prepare( obj ); + ASSERT_FALSE( modSetStateAll->canApplyInPlace() ); + modSetStateAll->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 << 2 << 3 ) ) ), + modSetStateAll->getOpLogRewrite() ); + } + }; + + // Pull is only in place if it's a no-op. + class PullRewriteInPlace { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj modMatcher = BSON( "$pull" << BSON( "a" << BSON( "$gt" << 3 ) ) ); + ModSet modSetMatcher( modMatcher ); + auto_ptr<ModSetState> modSetStateMatcher = modSetMatcher.prepare( obj ); + ASSERT_TRUE( modSetStateMatcher->canApplyInPlace() ); + modSetStateMatcher->applyModsInPlace(false); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 << 2) ) ), + modSetStateMatcher->getOpLogRewrite() ); + } + }; + + class PullRewriteForceNotInPlace { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj modMatcher = BSON( "$pull" << BSON( "a" << BSON( "$gt" << 3 ) ) ); + ModSet modSetMatcher( modMatcher ); + auto_ptr<ModSetState> modSetStateMatcher = modSetMatcher.prepare( obj ); + modSetStateMatcher->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 << 2) ) ), + modSetStateMatcher->getOpLogRewrite() ); + } + }; + + class PullRewriteNonExistingUnsets { + public: + void run() { + BSONObj obj; + BSONObj modMatcher = BSON( "$pull" << BSON( "a" << BSON( "$gt" << 3 ) ) ); + ModSet modSetMatcher( modMatcher ); + auto_ptr<ModSetState> modSetStateMatcher = modSetMatcher.prepare( obj ); + ASSERT_FALSE( modSetStateMatcher->canApplyInPlace() ); + modSetStateMatcher->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "a" << 1 ) ), + modSetStateMatcher->getOpLogRewrite() ); + } + }; + + class PullRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj mod = BSON( "$pull" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 2 ) ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class PullRewriteLastExistingField { + public: + void run() { + // check last pull corner case + BSONObj obj = BSON( "a" << BSON_ARRAY( 2 ) ); + BSONObj mod = BSON( "$pull" << BSON( "a" << 2 ) ); + ModSet modSetLast( mod ); + auto_ptr<ModSetState> modSetStateLast = modSetLast.prepare( obj ); + ASSERT_FALSE( modSetStateLast->canApplyInPlace() ); + modSetStateLast->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSONArray() ) ), + modSetStateLast->getOpLogRewrite() ); + } + }; + + class PullRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "b" << 1 ); + BSONObj mod = BSON( "$pull" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "a" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class TwoNestedPulls { + public: + void run() { + BSONObj obj = fromjson( "{ a:{ b:[ 1, 2 ], c:[ 1, 2 ] } }" ); + BSONObj mod = fromjson( "{ $pull:{ 'a.b':2, 'a.c':2 } }" ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( fromjson( "{ $set:{ 'a.b':[ 1 ] }, $set:{ 'a.c':[ 1 ] } }" ), + modSetState->getOpLogRewrite() ); + } + }; + + + // Pop is only applied in place if the target array remains the same size (i.e. if + // it is empty already. + class PopRewriteEmptyArray { + public: + void run() { + BSONObj obj = BSON( "a" << BSONArray() ); + BSONObj mod = BSON( "$pop" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSONArray() ) ), + modSetState->getOpLogRewrite() ); } }; + class PopRewriteLastElement { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 ) ); + BSONObj mod = BSON( "$pop" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSONArray() ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class PopRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2) ); + BSONObj mod = BSON( "$pop" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 ) ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class PopRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 ) ); + BSONObj mod = BSON( "$pop" << BSON( "b" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "b" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + // AddToSet is in place if it is a no-op. + class AddToSetRewriteInPlace { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj mod = BSON( "$addToSet" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << BSON_ARRAY( 1 << 2 ) ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class AddToSetRewriteForceNotInPlace { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 << 2 ) ); + BSONObj mod = BSON( "$addToSet" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a.0" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class AddToSetRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 ) ); + BSONObj mod = BSON( "$addToSet" << BSON( "a" << 2 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a.1" << 2 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class AddToSetRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << BSON_ARRAY( 1 ) ); + BSONObj mod = BSON( "$addToSet" << BSON( "b" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "b" << BSON_ARRAY( 1 ) ) ), + modSetState->getOpLogRewrite() ); + } + }; + + // Rename doesn't log if both fields are not present. + class RenameRewriteBothNonExistent { + public: + void run() { + BSONObj obj = BSON( "a" << 1 ); + BSONObj mod = BSON( "$rename" << BSON( "b" << "c" ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); + ASSERT_EQUALS( BSONObj(), modSetState->getOpLogRewrite() ); + } + }; + + class RenameRewriteExistingToField { + public: + void run() { + BSONObj obj = BSON( "b" << 100 ); + BSONObj mod = BSON( "$rename" << BSON( "a" << "b" ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_TRUE( modSetState->canApplyInPlace() ); + modSetState->applyModsInPlace(false); + ASSERT_EQUALS( BSONObj(), modSetState->getOpLogRewrite() ); + } + }; + + class RenameRewriteExistingFromField { + public: + void run() { + BSONObj obj = BSON( "a" << 100 ); + BSONObj mod = BSON( "$rename" << BSON( "a" << "b" ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "a" << 1 ) << "$set" << BSON ( "b" << 100 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class RenameRewriteBothExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << 100 << "b" << 200); + BSONObj mod = BSON( "$rename" << BSON( "a" << "b" ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "a" << 1 ) << "$set" << BSON ( "b" << 100 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + // $bit is never applied in place currently + class BitRewriteExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << 0 ); + BSONObj mod = BSON( "$bit" << BSON( "a" << BSON( "or" << 1 ) ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "a" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class BitRewriteNonExistingField { + public: + void run() { + BSONObj obj = BSON( "a" << 0 ); + BSONObj mod = BSON( "$bit" << BSON( "b" << BSON( "or" << 1 ) ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "b" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class SetIsNotRewritten { + public: + void run() { + BSONObj obj = BSON( "a" << 0 ); + BSONObj mod = BSON( "$set" << BSON( "b" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$set" << BSON( "b" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; + + class UnsetIsNotRewritten { + public: + void run() { + BSONObj obj = BSON( "a" << 0 ); + BSONObj mod = BSON( "$unset" << BSON( "a" << 1 ) ); + ModSet modSet( mod ); + auto_ptr<ModSetState> modSetState = modSet.prepare( obj ); + ASSERT_FALSE( modSetState->canApplyInPlace() ); + modSetState->createNewFromMods(); + ASSERT_EQUALS( BSON( "$unset" << BSON( "a" << 1 ) ), + modSetState->getOpLogRewrite() ); + } + }; }; namespace basic { @@ -933,8 +1357,40 @@ namespace UpdateTests { add< ModSetTests::inc2 >(); add< ModSetTests::set1 >(); add< ModSetTests::push1 >(); - add< ModSetTests::IncRewrite >(); + + add< ModSetTests::IncRewriteInPlace >(); + add< ModSetTests::InRewriteForceNotInPlace >(); add< ModSetTests::IncRewriteNestedArray >(); + add< ModSetTests::IncRewriteExistingField >(); + add< ModSetTests::IncRewriteNonExistingField >(); + add< ModSetTests::PushRewriteExistingField >(); + add< ModSetTests::PushRewriteNonExistingField >(); + add< ModSetTests::PushAllRewriteExistingField >(); + add< ModSetTests::PushAllRewriteNonExistingField >(); + add< ModSetTests::PullRewriteInPlace >(); + add< ModSetTests::PullRewriteForceNotInPlace >(); + add< ModSetTests::PullRewriteNonExistingUnsets >(); + add< ModSetTests::PullRewriteExistingField >(); + add< ModSetTests::PullRewriteLastExistingField >(); + add< ModSetTests::PullRewriteNonExistingField >(); + add< ModSetTests::TwoNestedPulls >(); + add< ModSetTests::PopRewriteEmptyArray >(); + add< ModSetTests::PopRewriteLastElement >(); + add< ModSetTests::PopRewriteExistingField >(); + add< ModSetTests::PopRewriteNonExistingField >(); + add< ModSetTests::AddToSetRewriteInPlace >(); + add< ModSetTests::AddToSetRewriteForceNotInPlace >(); + add< ModSetTests::AddToSetRewriteExistingField >(); + add< ModSetTests::AddToSetRewriteNonExistingField >(); + add< ModSetTests::RenameRewriteBothNonExistent >(); + add< ModSetTests::RenameRewriteExistingToField >(); + add< ModSetTests::RenameRewriteExistingFromField >(); + add< ModSetTests::RenameRewriteBothExistingField >(); + add< ModSetTests::BitRewriteExistingField >(); + // XXX $bit over non-existing field is missing. Probably out of scope to fix it here. + // add< ModSetTests::BitRewriteNonExistingField >(); + add< ModSetTests::SetIsNotRewritten >(); + add< ModSetTests::UnsetIsNotRewritten >(); add< basic::inc1 >(); add< basic::inc2 >(); diff --git a/src/mongo/s/balancer_policy.cpp b/src/mongo/s/balancer_policy.cpp index 1cbb1796462..f5179250893 100644 --- a/src/mongo/s/balancer_policy.cpp +++ b/src/mongo/s/balancer_policy.cpp @@ -82,12 +82,12 @@ namespace mongo { for ( ShardInfoMap::const_iterator i = _shardInfo.begin(); i != _shardInfo.end(); ++i ) { if ( i->second.isSizeMaxed() || i->second.isDraining() || i->second.hasOpsQueued() ) { - log() << i->first << " is unavailable" << endl; + LOG(1) << i->first << " is unavailable" << endl; continue; } if ( ! i->second.hasTag( tag ) ) { - log() << i->first << " doesn't have right tag" << endl; + LOG(1) << i->first << " doesn't have right tag" << endl; continue; } @@ -212,8 +212,8 @@ namespace mongo { int balancedLastTime ) { - // 1) check for shards that policy require to us to move off of - // draining, maxSize + // 1) check for shards that policy require to us to move off of: + // draining only // 2) check tag policy violations // 3) then we make sure chunks are balanced for each tag @@ -226,7 +226,7 @@ namespace mongo { string shard = *z; const ShardInfo& info = distribution.shardInfo( shard ); - if ( ! info.isSizeMaxed() && ! info.isDraining() ) + if ( ! info.isDraining() ) continue; if ( distribution.numberOfChunksInShard( shard ) == 0 ) diff --git a/src/mongo/s/balancer_policy.h b/src/mongo/s/balancer_policy.h index d4ee5a718e8..d1c92aa94c9 100644 --- a/src/mongo/s/balancer_policy.h +++ b/src/mongo/s/balancer_policy.h @@ -80,6 +80,10 @@ namespace mongo { */ bool hasOpsQueued() const { return _hasOpsQueued; } + long long getMaxSize() const { return _maxSize; } + + long long getCurrSize() const { return _currSize; } + string toString() const; private: diff --git a/src/mongo/s/balancer_policy_tests.cpp b/src/mongo/s/balancer_policy_tests.cpp index 2db4761cf50..8cd9e56ba1f 100644 --- a/src/mongo/s/balancer_policy_tests.cpp +++ b/src/mongo/s/balancer_policy_tests.cpp @@ -291,5 +291,221 @@ namespace mongo { } + /** + * Idea for this test is to set up three shards, one of which is overloaded (too much data). + * + * Even though the overloaded shard has less chunks, we shouldn't move chunks to that shard. + */ + TEST( BalancerPolicyTests, MaxSizeRespect ) { + + ShardToChunksMap chunks; + addShard( chunks, 3 , false ); + addShard( chunks, 4 , false ); + addShard( chunks, 6 , true ); + + // Note that maxSize of shard0 is 1, and it is therefore overloaded with currSize = 3. + // Other shards have maxSize = 0 = unset. + + ShardInfoMap shards; + // ShardInfo(maxSize, currSize, draining, opsQueued) + shards["shard0"] = ShardInfo( 1, 3, false, false ); + shards["shard1"] = ShardInfo( 0, 4, false, false ); + shards["shard2"] = ShardInfo( 0, 6, false, false ); + + DistributionStatus d( shards, chunks ); + MigrateInfo* m = BalancerPolicy::balance( "ns", d, 0 ); + ASSERT( m ); + ASSERT_EQUALS( "shard2" , m->from ); + ASSERT_EQUALS( "shard1" , m->to ); + + } + + /** + * Here we check that being over the maxSize is *not* equivalent to draining, we don't want + * to empty shards for no other reason than they are over this limit. + */ + TEST( BalancerPolicyTests, MaxSizeNoDrain ) { + + ShardToChunksMap chunks; + // Shard0 will be overloaded + addShard( chunks, 4 , false ); + addShard( chunks, 4 , false ); + addShard( chunks, 4 , true ); + + // Note that maxSize of shard0 is 1, and it is therefore overloaded with currSize = 4. + // Other shards have maxSize = 0 = unset. + + ShardInfoMap shards; + // ShardInfo(maxSize, currSize, draining, opsQueued) + shards["shard0"] = ShardInfo( 1, 4, false, false ); + shards["shard1"] = ShardInfo( 0, 4, false, false ); + shards["shard2"] = ShardInfo( 0, 4, false, false ); + + DistributionStatus d( shards, chunks ); + MigrateInfo* m = BalancerPolicy::balance( "ns", d, 0 ); + ASSERT( !m ); + } + + // Note: Only in 2.2, 2.4 has utility class + class PseudoRandom { + public: + + PseudoRandom(unsigned int seed) { + _seed = seed; + } + + int nextInt32( int max = -1 ){ + +#if !defined(_WIN32) + int r = rand_r( &_seed ) ; +#else + int r = ::rand(); // seed not used in this case +#endif + return max > 0 ? r % max : r; + } + + private: + unsigned int _seed; + }; + + /** + * Idea behind this test is that we set up several shards, the first two of which are + * draining and the second two of which have a data size limit. We also simulate a random + * number of chunks on each shard. + * + * Once the shards are setup, we virtually migrate numChunks times, or until there are no + * more migrations to run. Each chunk is assumed to have a size of 1 unit, and we increment + * our currSize for each shard as the chunks move. + * + * Finally, we ensure that the drained shards are drained, the data-limited shards aren't + * overloaded, and that all shards (including the data limited shard if the baseline isn't + * over the limit are balanced to within 1 unit of some baseline. + * + */ + TEST( BalancerPolicyTests, Simulation ) { + + // Hardcode seed here, make test deterministic. + int64_t seed = 1337; + PseudoRandom rng(seed); + + // Run test 10 times + for (int test = 0; test < 10; test++) { + + // + // Setup our shards as draining, with maxSize, and normal + // + + int numShards = 7; + int numChunks = 0; + + ShardToChunksMap chunks; + ShardInfoMap shards; + + map<string,int> expected; + + for (int i = 0; i < numShards; i++) { + + int numShardChunks = rng.nextInt32(100); + bool draining = i < 2; + bool maxed = i >= 2 && i < 4; + + if (draining) expected[str::stream() << "shard" << i] = 0; + if (maxed) expected[str::stream() << "shard" << i] = numShardChunks + 1; + + addShard(chunks, numShardChunks, false); + numChunks += numShardChunks; + + shards[str::stream() << "shard" << i] = + ShardInfo(maxed ? numShardChunks + 1 : 0, + numShardChunks, draining, false); + } + + for (ShardInfoMap::iterator it = shards.begin(); it != shards.end(); ++it) { + log() << it->first << " : " << it->second.toString() << endl; + } + + // + // Perform migrations and increment data size as chunks move + // + + for (int i = 0; i < numChunks; i++) { + + DistributionStatus d( shards, chunks ); + MigrateInfo* m = BalancerPolicy::balance( "ns", d, i != 0 ); + + if (!m) { + log() << "Finished with test moves." << endl; + break; + } + + moveChunk(chunks, m); + + { + ShardInfo& info = shards[m->from]; + shards[m->from] = ShardInfo(info.getMaxSize(), + info.getCurrSize() - 1, + info.isDraining(), + info.hasOpsQueued()); + } + + { + ShardInfo& info = shards[m->to]; + shards[m->to] = ShardInfo(info.getMaxSize(), + info.getCurrSize() + 1, + info.isDraining(), + info.hasOpsQueued()); + } + } + + // + // Make sure our balance is correct and our data size is low. + // + + // The balanced value is the count on the last shard, since it's not draining or + // limited + int balancedSize = (--shards.end())->second.getCurrSize(); + + for (ShardInfoMap::iterator it = shards.begin(); it != shards.end(); ++it) { + log() << it->first << " : " << it->second.toString() << endl; + } + + for (ShardInfoMap::iterator it = shards.begin(); it != shards.end(); ++it) { + + log() << it->first << " : " << it->second.toString() << endl; + + map<string,int>::iterator expectedIt = expected.find(it->first); + + if (expectedIt == expected.end()) { + bool isInRange = it->second.getCurrSize() >= balancedSize - 1 && + it->second.getCurrSize() <= balancedSize + 1; + + if (!isInRange) { + warning() << "non-limited and non-draining shard had " + << it->second.getCurrSize() << " chunks, expected near " + << balancedSize << endl; + } + + ASSERT(isInRange); + } + else { + int expectedSize = expectedIt->second; + bool isInRange = it->second.getCurrSize() <= expectedSize; + if (isInRange && expectedSize >= balancedSize) { + isInRange = it->second.getCurrSize() >= balancedSize - 1 && + it->second.getCurrSize() <= balancedSize + 1; + } + + if (!isInRange) { + warning() << "limited or draining shard had " + << it->second.getCurrSize() << " chunks, expected less than " + << expectedSize << " and (if less than expected) near " + << balancedSize << endl; + } + + ASSERT(isInRange); + } + } + } + } } } diff --git a/src/mongo/s/config.cpp b/src/mongo/s/config.cpp index fa3d43defce..8c8a2d2fbbf 100644 --- a/src/mongo/s/config.cpp +++ b/src/mongo/s/config.cpp @@ -1033,15 +1033,16 @@ namespace mongo { log(1) << "replicaSetChange: shard not found for set: " << monitor->getServerAddress() << endl; return; } - scoped_ptr<ScopedDbConnection> conn( ScopedDbConnection::getScopedDbConnection( + scoped_ptr<ScopedDbConnection> conn( ScopedDbConnection::getInternalScopedDbConnection( configServer.getConnectionString().toString(), 30.0 ) ); conn->get()->update( ShardNS::shard, BSON( "_id" << s.getName() ), BSON( "$set" << BSON( "host" << monitor->getServerAddress() ) ) ); conn->done(); } - catch ( DBException & ) { - error() << "RSChangeWatcher: could not update config db for set: " << monitor->getName() << " to: " << monitor->getServerAddress() << endl; + catch (DBException& e) { + error() << "RSChangeWatcher: could not update config db for set: " << monitor->getName() + << " to: " << monitor->getServerAddress() << causedBy(e) << endl; } } diff --git a/src/mongo/s/d_logic.cpp b/src/mongo/s/d_logic.cpp index 4d04eea2be9..c58ed173968 100644 --- a/src/mongo/s/d_logic.cpp +++ b/src/mongo/s/d_logic.cpp @@ -100,21 +100,24 @@ namespace mongo { dbresponse->responseTo = m.header()->id; return true; } - - uassert( 9517 , "writeback" , ( d.reservedField() & Reserved_FromWriteback ) == 0 ); - OID writebackID; - writebackID.initSequential(); + uassert( 9517 , "writeback" , ( d.reservedField() & Reserved_FromWriteback ) == 0 ); const OID& clientID = ShardedConnectionInfo::get(false)->getID(); massert( 10422 , "write with bad shard config and no server id!" , clientID.isSet() ); + // We need to check this here, since otherwise we'll get errors wrapping the writeback - + // not just here, but also when returning as a command result. + // We choose 1/2 the overhead of the internal maximum so that we can still handle ops of + // 16MB exactly. + massert( 16437, "data size of operation is too large to queue for writeback", + m.dataSize() < BSONObjMaxInternalSize - (8 * 1024)); + LOG(1) << "writeback queued for " << m.toString() << endl; BSONObjBuilder b; b.appendBool( "writeBack" , true ); b.append( "ns" , ns ); - b.append( "id" , writebackID ); b.append( "connectionId" , cc().getConnectionId() ); b.append( "instanceIdent" , prettyHostName() ); wanted.addToBSON( b ); @@ -123,11 +126,9 @@ namespace mongo { b.appendBinData( "msg" , m.header()->len , bdtCustom , (char*)(m.singleData()) ); LOG(2) << "writing back msg with len: " << m.header()->len << " op: " << m.operation() << endl; - // Don't register the writeback until immediately before we queue it - - // after this line, mongos will wait for an hour if we don't queue correctly - lastError.getSafe()->writeback( writebackID ); + OID writebackID = writeBackManager.queueWriteBack( clientID.str() , b ); - writeBackManager.queueWriteBack( clientID.str() , b.obj() ); + lastError.getSafe()->writeback( writebackID ); return true; } diff --git a/src/mongo/s/d_migrate.cpp b/src/mongo/s/d_migrate.cpp index 138081fe376..479e9ad1085 100644 --- a/src/mongo/s/d_migrate.cpp +++ b/src/mongo/s/d_migrate.cpp @@ -193,6 +193,9 @@ namespace mongo { ShardForceVersionOkModeBlock sf; { RemoveSaver rs("moveChunk",ns,"post-cleanup"); + + log() << "moveChunk starting delete for: " << this->toString() << migrateLog; + long long numDeleted = Helpers::removeRange( ns , min , @@ -202,7 +205,9 @@ namespace mongo { secondaryThrottle , cmdLine.moveParanoia ? &rs : 0 , /*callback*/ true ); /*fromMigrate*/ - log() << "moveChunk deleted: " << numDeleted << migrateLog; + + log() << "moveChunk deleted " << numDeleted << " documents for " + << this->toString() << migrateLog; } @@ -260,7 +265,13 @@ namespace mongo { const BSONObj& min , const BSONObj& max , const BSONObj& shardKeyPattern ) { - scoped_lock ll(_workLock); + + // + // Do not hold _workLock + // + + //scoped_lock ll(_workLock); + scoped_lock l(_m); // reads and writes _active verify( ! _active ); @@ -283,7 +294,10 @@ namespace mongo { } void done() { - Lock::DBRead lk( _ns ); + log() << "MigrateFromStatus::done About to acquire global write lock to exit critical " + "section" << endl; + Lock::GlobalWrite lk; + log() << "MigrateFromStatus::done Global lock acquired" << endl; { scoped_spinlock lk( _trackerLocks ); @@ -329,7 +343,7 @@ namespace mongo { case 'd': { - if ( getThreadName() == cleanUpThreadName ) { + if (getThreadName().find(cleanUpThreadName) == 0) { // we don't want to xfer things we're cleaning // as then they'll be deleted on TO // which is bad @@ -662,11 +676,16 @@ namespace mongo { }; void _cleanupOldData( OldDataCleanup cleanup ) { - Client::initThread( cleanUpThreadName ); + + Client::initThread((string(cleanUpThreadName) + string("-") + + OID::gen().toString()).c_str()); + if (!noauth) { cc().getAuthenticationInfo()->authorize("local", internalSecurity.user); } - log() << " (start) waiting to cleanup " << cleanup << " # cursors:" << cleanup.initial.size() << migrateLog; + + log() << " (start) waiting to cleanup " << cleanup + << ", # cursors remaining: " << cleanup.initial.size() << migrateLog; int loops = 0; Timer t; @@ -897,6 +916,7 @@ namespace mongo { configServer.logChange( "moveChunk.start" , ns , chunkInfo ); ShardChunkVersion maxVersion; + ShardChunkVersion startingVersion; string myOldShard; { scoped_ptr<ScopedDbConnection> conn( @@ -965,10 +985,10 @@ namespace mongo { // it's possible this shard will be *at* zero version from a previous migrate and // no refresh will be done // TODO: Make this less fragile - ShardChunkVersion shardVersion = maxVersion; - shardingState.trySetVersion( ns , shardVersion /* will return updated */ ); + startingVersion = maxVersion; + shardingState.trySetVersion( ns , startingVersion /* will return updated */ ); - log() << "moveChunk request accepted at version " << shardVersion << migrateLog; + log() << "moveChunk request accepted at version " << startingVersion << migrateLog; } timing.done(2); @@ -1006,7 +1026,8 @@ namespace mongo { res ); } catch( DBException& e ){ - errmsg = str::stream() << "moveChunk could not contact to: shard " << to << " to start transfer" << causedBy( e ); + errmsg = str::stream() << "moveChunk could not contact to: shard " + << to << " to start transfer" << causedBy( e ); warning() << errmsg << endl; return false; } @@ -1018,6 +1039,7 @@ namespace mongo { verify( res["errmsg"].type() ); errmsg += res["errmsg"].String(); result.append( "cause" , res ); + warning() << errmsg << endl; return false; } @@ -1081,8 +1103,7 @@ namespace mongo { // 5.a // we're under the collection lock here, so no other migrate can change maxVersion or ShardChunkManager state migrateFromStatus.setInCriticalSection( true ); - ShardChunkVersion currVersion = maxVersion; - ShardChunkVersion myVersion = currVersion; + ShardChunkVersion myVersion = maxVersion; myVersion.incMajor(); { @@ -1102,7 +1123,8 @@ namespace mongo { { BSONObj res; scoped_ptr<ScopedDbConnection> connTo( - ScopedDbConnection::getScopedDbConnection( toShard.getConnString() ) ); + ScopedDbConnection::getScopedDbConnection( toShard.getConnString(), + 35.0 ) ); bool ok; @@ -1114,21 +1136,24 @@ namespace mongo { catch( DBException& e ){ errmsg = str::stream() << "moveChunk could not contact to: shard " << toShard.getConnString() << " to commit transfer" << causedBy( e ); warning() << errmsg << endl; - return false; + ok = false; } connTo->done(); if ( ! ok ) { + log() << "moveChunk migrate commit not accepted by TO-shard: " << res + << " resetting shard version to: " << startingVersion << migrateLog; { - Lock::DBWrite lk( ns ); + Lock::GlobalWrite lk; + log() << "moveChunk global lock acquired to reset shard version from " + "failed migration" << endl; // revert the chunk manager back to the state before "forgetting" about the chunk - shardingState.undoDonateChunk( ns , min , max , currVersion ); + shardingState.undoDonateChunk( ns , min , max , startingVersion ); } - - log() << "moveChunk migrate commit not accepted by TO-shard: " << res - << " resetting shard version to: " << currVersion << migrateLog; + log() << "Shard version successfully reset to clean up failed migration" + << endl; errmsg = "_recvChunkCommit failed!"; result.append( "cause" , res ); @@ -1579,6 +1604,8 @@ namespace mongo { // this will prevent us from going into critical section until we're ready Timer t; while ( t.minutes() < 600 ) { + log() << "Waiting for replication to catch up before entering critical section" + << endl; if ( flushPendingWrites( lastOpApplied ) ) break; sleepsecs(1); @@ -1678,10 +1705,16 @@ namespace mongo { } } + // id object most likely has form { _id : ObjectId(...) } + // infer from that correct index to use, e.g. { _id : 1 } + BSONObj idIndexPattern; + Helpers::toKeyFormat( id , idIndexPattern ); + + // TODO: create a better interface to remove objects directly Helpers::removeRange( ns , id , id, - findShardKeyIndexPattern_locked( ns , shardKeyPattern ), + idIndexPattern , true , /*maxInclusive*/ false , /* secondaryThrottle */ cmdLine.moveParanoia ? &rs : 0 , /*callback*/ @@ -1762,7 +1795,8 @@ namespace mongo { Timer t; // we wait for the commit to succeed before giving up - while ( t.minutes() <= 5 ) { + while ( t.seconds() <= 30 ) { + log() << "Waiting for commit to finish" << endl; sleepmillis(1); if ( state == DONE ) return true; @@ -1843,9 +1877,26 @@ namespace mongo { migrateStatus.from = cmdObj["from"].String(); migrateStatus.min = cmdObj["min"].Obj().getOwned(); migrateStatus.max = cmdObj["max"].Obj().getOwned(); - migrateStatus.shardKeyPattern = cmdObj["shardKeyPattern"].Obj().getOwned(); migrateStatus.secondaryThrottle = cmdObj["secondaryThrottle"].trueValue(); - + if (cmdObj.hasField("shardKeyPattern")) { + migrateStatus.shardKeyPattern = cmdObj["shardKeyPattern"].Obj().getOwned(); + } else { + // shardKeyPattern may not be provided if another shard is from pre 2.2 + // In that case, assume the shard key pattern is the same as the range + // specifiers provided. + BSONObj keya , keyb; + Helpers::toKeyFormat( migrateStatus.min , keya ); + Helpers::toKeyFormat( migrateStatus.max , keyb ); + verify( keya == keyb ); + + warning() << "No shard key pattern provided by source shard for migration." + " This is likely because the source shard is running a version prior to 2.2." + " Falling back to assuming the shard key matches the pattern of the min and max" + " chunk range specifiers. Inferred shard key: " << keya << endl; + + migrateStatus.shardKeyPattern = keya.getOwned(); + } + if ( migrateStatus.secondaryThrottle && ! anyReplEnabled() ) { warning() << "secondaryThrottle asked for, but not replication" << endl; migrateStatus.secondaryThrottle = false; diff --git a/src/mongo/s/d_split.cpp b/src/mongo/s/d_split.cpp index 048c8150c6b..da7e063d6f9 100644 --- a/src/mongo/s/d_split.cpp +++ b/src/mongo/s/d_split.cpp @@ -749,16 +749,38 @@ namespace mongo { if (newChunks.size() == 2){ // If one of the chunks has only one object in it we should move it - static const BSONObj fields = BSON("_id" << 1 ); - DBDirectClient conn; for (int i=1; i >= 0 ; i--){ // high chunk more likely to have only one obj - ChunkInfo chunk = newChunks[i]; - Query q = Query().minKey(chunk.min).maxKey(chunk.max); - scoped_ptr<DBClientCursor> c (conn.query(ns, q, /*limit*/-2, 0, &fields)); - if (c && c->itcount() == 1) { - result.append("shouldMigrate", BSON("min" << chunk.min << "max" << chunk.max)); + + Client::ReadContext ctx( ns ); + NamespaceDetails *d = nsdetails( ns.c_str() ); + + const IndexDetails *idx = d->findIndexByPrefix( keyPattern , + true ); /* exclude multikeys */ + if ( idx == NULL ) { break; } + + ChunkInfo chunk = newChunks[i]; + BSONObj newmin = Helpers::modifiedRangeBound(chunk.min, idx->keyPattern(), -1); + BSONObj newmax = Helpers::modifiedRangeBound(chunk.max , idx->keyPattern(), -1); + + scoped_ptr<BtreeCursor> bc( BtreeCursor::make( d , + d->idxNo(*idx) , + *idx , + newmin , /* lower */ + newmax , /* upper */ + false , /* upper noninclusive */ + 1 ) ); /* direction */ + + // check if exactly one document found + if ( bc->ok() ) { + bc->advance(); + if ( bc->eof() ) { + result.append( "shouldMigrate", + BSON("min" << chunk.min << "max" << chunk.max) ); + break; + } + } } } diff --git a/src/mongo/s/d_state.cpp b/src/mongo/s/d_state.cpp index 7cf31358b19..37adfe8131b 100644 --- a/src/mongo/s/d_state.cpp +++ b/src/mongo/s/d_state.cpp @@ -162,6 +162,7 @@ namespace mongo { void ShardingState::undoDonateChunk( const string& ns , const BSONObj& min , const BSONObj& max , ShardChunkVersion version ) { scoped_lock lk( _mutex ); + log() << "ShardingState::undoDonateChunk acquired _mutex" << endl; ChunkManagersMap::const_iterator it = _chunks.find( ns ); verify( it != _chunks.end() ) ; @@ -634,7 +635,7 @@ namespace mongo { if ( version < globalVersion && version.hasCompatibleEpoch( globalVersion ) ) { while ( shardingState.inCriticalMigrateSection() ) { dbtemprelease r; - sleepmillis(2); + sleepmillis(20); OCCASIONALLY log() << "waiting till out of critical section" << endl; } errmsg = "shard global version for collection is higher than trying to set to '" + ns + "'"; diff --git a/src/mongo/s/d_writeback.cpp b/src/mongo/s/d_writeback.cpp index 0278e40f712..441de1a0d65 100644 --- a/src/mongo/s/d_writeback.cpp +++ b/src/mongo/s/d_writeback.cpp @@ -42,21 +42,18 @@ namespace mongo { WriteBackManager::~WriteBackManager() { } - void WriteBackManager::queueWriteBack( const string& remote , const BSONObj& o ) { - static mongo::mutex xxx( "WriteBackManager::queueWriteBack tmp" ); - static OID lastOID; - - scoped_lock lk( xxx ); - const BSONElement& e = o["id"]; - - if ( lastOID.isSet() ) { - if ( e.OID() < lastOID ) { - log() << "this could fail" << endl; - printStackTrace(); - } - } - lastOID = e.OID(); - getWritebackQueue( remote )->queue.push( o ); + OID WriteBackManager::queueWriteBack( const string& remote , BSONObjBuilder& b ) { + static mongo::mutex writebackIDOrdering( "WriteBackManager::queueWriteBack id ordering" ); + + scoped_lock lk( writebackIDOrdering ); + + OID writebackID; + writebackID.initSequential(); + b.append( "id", writebackID ); + + getWritebackQueue( remote )->queue.push( b.obj() ); + + return writebackID; } shared_ptr<WriteBackManager::QueueInfo> WriteBackManager::getWritebackQueue( const string& remote ) { diff --git a/src/mongo/s/d_writeback.h b/src/mongo/s/d_writeback.h index fc5d9c19f29..38ce386515b 100644 --- a/src/mongo/s/d_writeback.h +++ b/src/mongo/s/d_writeback.h @@ -56,10 +56,12 @@ namespace mongo { * @param remote server ID this operation came from * @param op the operation itself * - * Enqueues opeartion 'op' in server 'remote's queue. The operation will be written back to - * remote at a later stager. + * Enqueues operation 'op' in server 'remote's queue. The operation will be written back to + * remote at a later stage. + * + * @return the writebackId generated */ - void queueWriteBack( const string& remote , const BSONObj& op ); + OID queueWriteBack( const string& remote , BSONObjBuilder& opBuilder ); /* * @param remote server ID diff --git a/src/mongo/s/shard_version.cpp b/src/mongo/s/shard_version.cpp index 7d0ef93da8f..2931ac24ab4 100644 --- a/src/mongo/s/shard_version.cpp +++ b/src/mongo/s/shard_version.cpp @@ -43,24 +43,24 @@ namespace mongo { S getSequence( DBClientBase * conn , const string& ns ) { scoped_lock lk( _mutex ); - return _map[conn][ns]; + return _map[conn->getConnectionId()][ns]; } void setSequence( DBClientBase * conn , const string& ns , const S& s ) { scoped_lock lk( _mutex ); - _map[conn][ns] = s; + _map[conn->getConnectionId()][ns] = s; } void reset( DBClientBase * conn ) { scoped_lock lk( _mutex ); - _map.erase( conn ); + _map.erase( conn->getConnectionId() ); } // protects _map mongo::mutex _mutex; // a map from a connection into ChunkManager's sequence number for each namespace - map<DBClientBase*, map<string,unsigned long long> > _map; + map<unsigned long long, map<string,unsigned long long> > _map; } connectionShardStatus; @@ -232,6 +232,8 @@ namespace mongo { << " version: " << version << " manager: " << manager.get() << endl; + const string versionableServerAddress(conn->getServerAddress()); + BSONObj result; if ( setShardVersion( *conn , ns , version , authoritative , result ) ) { // success! @@ -246,7 +248,9 @@ namespace mongo { massert( 10428 , "need_authoritative set but in authoritative mode already" , ! authoritative ); if ( ! authoritative ) { - checkShardVersion( conn , ns , refManager, 1 , tryNumber + 1 ); + // use the original connection and get a fresh versionable connection + // since conn can be invalidated (or worse, freed) after the failure + checkShardVersion(conn_in, ns, refManager, 1, tryNumber + 1); return true; } @@ -268,13 +272,15 @@ namespace mongo { const int maxNumTries = 7; if ( tryNumber < maxNumTries ) { LOG( tryNumber < ( maxNumTries / 2 ) ? 1 : 0 ) - << "going to retry checkShardVersion host: " << conn->getServerAddress() << " " << result << endl; + << "going to retry checkShardVersion host: " << versionableServerAddress << " " << result << endl; sleepmillis( 10 * tryNumber ); - checkShardVersion( conn , ns , refManager, true , tryNumber + 1 ); + // use the original connection and get a fresh versionable connection + // since conn can be invalidated (or worse, freed) after the failure + checkShardVersion(conn_in, ns, refManager, true, tryNumber + 1); return true; } - string errmsg = str::stream() << "setShardVersion failed host: " << conn->getServerAddress() << " " << result; + string errmsg = str::stream() << "setShardVersion failed host: " << versionableServerAddress << " " << result; log() << " " << errmsg << endl; massert( 10429 , errmsg , 0 ); return true; diff --git a/src/mongo/s/writeback_listener.cpp b/src/mongo/s/writeback_listener.cpp index 75b474fa162..aece7e29ccb 100644 --- a/src/mongo/s/writeback_listener.cpp +++ b/src/mongo/s/writeback_listener.cpp @@ -332,8 +332,6 @@ namespace mongo { gle = b.obj(); } - log() << "GLE is " << gle << endl; - if ( gle["code"].numberInt() == 9517 ) { log() << "new version change detected, " diff --git a/src/mongo/shell/collection.js b/src/mongo/shell/collection.js index 342f07b936f..b3b2dace6ae 100644 --- a/src/mongo/shell/collection.js +++ b/src/mongo/shell/collection.js @@ -35,9 +35,9 @@ DBCollection.prototype.help = function () { print("\tdb." + shortName + ".copyTo(newColl) - duplicates collection by copying all documents to newColl; no indexes are copied."); print("\tdb." + shortName + ".convertToCapped(maxBytes) - calls {convertToCapped:'" + shortName + "', size:maxBytes}} command"); print("\tdb." + shortName + ".dataSize()"); - print("\tdb." + shortName + ".distinct( key ) - eg. db." + shortName + ".distinct( 'x' )"); + print("\tdb." + shortName + ".distinct( key ) - e.g. db." + shortName + ".distinct( 'x' )"); print("\tdb." + shortName + ".drop() drop the collection"); - print("\tdb." + shortName + ".dropIndex(name)"); + print("\tdb." + shortName + ".dropIndex(index) - e.g. db." + shortName + ".dropIndex( \"indexName\" ) or db." + shortName + ".dropIndex( { \"indexKey\" : 1 } )"); print("\tdb." + shortName + ".dropIndexes()"); print("\tdb." + shortName + ".ensureIndex(keypattern[,options]) - options is an object with these possible fields: name, unique, dropDups"); print("\tdb." + shortName + ".reIndex()"); @@ -464,21 +464,19 @@ DBCollection.prototype.clean = function() { * <p>Drop a specified index.</p> * * <p> - * Name is the name of the index in the system.indexes name field. (Run db.system.indexes.find() to - * see example data.) + * "index" is the name of the index in the system.indexes name field (run db.system.indexes.find() to + * see example data), or an object holding the key(s) used to create the index. + * For example: + * db.collectionName.dropIndex( "myIndexName" ); + * db.collectionName.dropIndex( { "indexKey" : 1 } ); * </p> * - * <p>Note : alpha: space is not reclaimed </p> - * @param {String} name of index to delete. + * @param {String} name or key object of index to delete. * @return A result object. result.ok will be true if successful. */ DBCollection.prototype.dropIndex = function(index) { - assert(index , "need to specify index to dropIndex" ); - - if ( ! isString( index ) && isObject( index ) ) - index = this._genIndexName( index ); - - var res = this._dbCommand( "deleteIndexes" ,{ index: index } ); + assert(index, "need to specify index to dropIndex" ); + var res = this._dbCommand( "deleteIndexes", { index: index } ); this.resetIndexCache(); return res; } diff --git a/src/mongo/shell/servers_misc.js b/src/mongo/shell/servers_misc.js index f66db5709fe..5018e19dbaf 100644 --- a/src/mongo/shell/servers_misc.js +++ b/src/mongo/shell/servers_misc.js @@ -55,8 +55,9 @@ MongodRunner.prototype.port = function() { return this.port_; } MongodRunner.prototype.toString = function() { return [ this.port_, this.dbpath_, this.peer_, this.arbiter_ ].toString(); } -ToolTest = function( name ){ +ToolTest = function( name, extraOptions ){ this.name = name; + this.options = extraOptions; this.port = allocatePorts(1)[0]; this.baseName = "jstests_tool_" + name; this.root = "/data/db/" + this.baseName; @@ -69,8 +70,17 @@ ToolTest = function( name ){ ToolTest.prototype.startDB = function( coll ){ assert( ! this.m , "db already running" ); - - this.m = startMongoProgram( "mongod" , "--port", this.port , "--dbpath" , this.dbpath , "--nohttpinterface", "--noprealloc" , "--smallfiles" , "--bind_ip", "127.0.0.1" ); + + var options = {port : this.port, + dbpath : this.dbpath, + nohttpinterface : "", + noprealloc : "", + smallfiles : "", + bind_ip : "127.0.0.1"}; + + Object.extend(options, this.options); + + this.m = startMongoProgram.apply(null, MongoRunner.arrOptions("mongod", options)); this.db = this.m.getDB( this.baseName ); if ( coll ) return this.db.getCollection( coll ); diff --git a/src/mongo/shell/utils_sh.js b/src/mongo/shell/utils_sh.js index 3aa2102b2f4..9d1b3a47822 100644 --- a/src/mongo/shell/utils_sh.js +++ b/src/mongo/shell/utils_sh.js @@ -341,8 +341,8 @@ sh.removeShardTag = function( shard, tag ) { sh.addTagRange = function( ns, min, max, tag ) { var config = db.getSisterDB( "config" ); - config.tags.update( { ns : ns , min : min } , - { ns : ns , min : min , max : max , tag : tag } , - true ); + config.tags.update( {_id: { ns : ns , min : min } } , + {_id: { ns : ns , min : min }, ns : ns , min : min , max : max , tag : tag } , + true ); sh._checkLastError( config ); } diff --git a/src/mongo/tools/dump.cpp b/src/mongo/tools/dump.cpp index 11780ca0226..d92238b60ea 100644 --- a/src/mongo/tools/dump.cpp +++ b/src/mongo/tools/dump.cpp @@ -98,7 +98,6 @@ public: queryOptions |= QueryOption_OplogReplay; else if ( _query.isEmpty() && !hasParam("dbpath") && !hasParam("forceTableScan") ) { q.snapshot(); - log() << "doing snapshot query" << endl; } DBClientBase& connBase = conn(true); @@ -137,33 +136,32 @@ public: map<string, BSONObj> options, multimap<string, BSONObj> indexes ) { log() << "\tMetadata for " << coll << " to " << outputFile.string() << endl; - ofstream file (outputFile.string().c_str()); - uassert(15933, "Couldn't open file: " + outputFile.string(), file.is_open()); - bool hasOptions = options.count(coll) > 0; bool hasIndexes = indexes.count(coll) > 0; - if (hasOptions) { - file << "{options : " << options.find(coll)->second.jsonString(); + BSONObjBuilder metadata; - if (hasIndexes) { - file << ", "; - } - } else { - file << "{"; + if (hasOptions) { + metadata << "options" << options.find(coll)->second; } if (hasIndexes) { - file << "indexes:["; - for (multimap<string, BSONObj>::iterator it=indexes.equal_range(coll).first; it!=indexes.equal_range(coll).second; ++it) { - if (it != indexes.equal_range(coll).first) { - file << ", "; - } - file << (*it).second.jsonString(); + BSONArrayBuilder indexesOutput (metadata.subarrayStart("indexes")); + + // I'd kill for C++11 auto here... + const pair<multimap<string, BSONObj>::iterator, multimap<string, BSONObj>::iterator> + range = indexes.equal_range(coll); + + for (multimap<string, BSONObj>::iterator it=range.first; it!=range.second; ++it) { + indexesOutput << it->second; } - file << "]"; + + indexesOutput.done(); } - file << "}"; + + ofstream file (outputFile.string().c_str()); + uassert(15933, "Couldn't open file: " + outputFile.string(), file.is_open()); + file << metadata.done().jsonString(); } @@ -196,7 +194,7 @@ public: BSONObj obj = cursor->nextSafe(); const string name = obj.getField( "name" ).valuestr(); if (obj.hasField("options")) { - collectionOptions.insert( pair<string,BSONObj> (name, obj.getField("options").embeddedObject()) ); + collectionOptions[name] = obj.getField("options").embeddedObject().getOwned(); } // skip namespaces with $ in them only if we don't specify a collection to dump diff --git a/src/mongo/tools/restore.cpp b/src/mongo/tools/restore.cpp index d7779e6a3a7..4ec0ef1c538 100644 --- a/src/mongo/tools/restore.cpp +++ b/src/mongo/tools/restore.cpp @@ -159,7 +159,7 @@ public: drillDown(root, _db != "", _coll != "", true); // should this happen for oplog replay as well? - conn().getLastError(); + conn().getLastError(_db == "" ? "admin" : _db); if (doOplog) { log() << "\t Replaying oplog" << endl; @@ -353,7 +353,7 @@ public: // wait for ops to propagate to "w" nodes (doesn't warn if w used without replset) if ( _w > 1 ) { - conn().getLastError(false, false, _w); + conn().getLastError(db, false, false, _w); } } else if ( endsWith( _curns.c_str() , ".system.indexes" )) { @@ -370,7 +370,7 @@ public: // wait for insert to propagate to "w" nodes (doesn't warn if w used without replset) if ( _w > 1 ) { - conn().getLastErrorDetailed(false, false, _w); + conn().getLastErrorDetailed(_curdb, false, false, _w); } } } @@ -478,20 +478,28 @@ private: conn().insert( _curdb + ".system.indexes" , o ); // We're stricter about errors for indexes than for regular data - BSONObj err = conn().getLastErrorDetailed(false, false, _w); + BSONObj err = conn().getLastErrorDetailed(_curdb, false, false, _w); - if ( ! ( err["err"].isNull() ) ) { - if (err["err"].String() == "norepl" && _w > 1) { + if (err.hasField("err") && !err["err"].isNull()) { + if (err["err"].str() == "norepl" && _w > 1) { error() << "Cannot specify write concern for non-replicas" << endl; } else { - error() << "Error creating index " << o["ns"].String(); - error() << ": " << err["code"].Int() << " " << err["err"].String() << endl; - error() << "To resume index restoration, run " << _name << " on file" << _fileName << " manually." << endl; + string errCode; + + if (err.hasField("code")) { + errCode = str::stream() << err["code"].numberInt(); + } + + error() << "Error creating index " << o["ns"].String() << ": " + << errCode << " " << err["err"] << endl; } ::abort(); } + + massert(16441, str::stream() << "Error calling getLastError: " << err["errmsg"], + err["ok"].trueValue()); } }; diff --git a/src/mongo/tools/tool.cpp b/src/mongo/tools/tool.cpp index c092cdbd51a..5dc3e90b67c 100644 --- a/src/mongo/tools/tool.cpp +++ b/src/mongo/tools/tool.cpp @@ -308,6 +308,9 @@ namespace mongo { if ( useDirectClient ) dbexit( EXIT_CLEAN ); + + fflush(stdout); + fflush(stderr); return ret; } @@ -423,8 +426,10 @@ namespace mongo { } string errmsg; - if ( _conn->auth( dbname , _username , _password , errmsg, true, level ) ) { - return; + if (dbname.size()) { + if ( _conn->auth( dbname , _username , _password , errmsg, true, level ) ) { + return; + } } // try against the admin db diff --git a/src/mongo/util/assert_util.cpp b/src/mongo/util/assert_util.cpp index b1c1d2321ca..f8487d6361e 100644 --- a/src/mongo/util/assert_util.cpp +++ b/src/mongo/util/assert_util.cpp @@ -147,7 +147,7 @@ namespace mongo { NOINLINE_DECL void msgasserted(int msgid, const char *msg) { assertionCount.condrollover( ++assertionCount.warning ); - tlog() << "Assertion: " << msgid << ":" << msg << endl; + log() << "Assertion: " << msgid << ":" << msg << endl; setLastError(msgid,msg && *msg ? msg : "massert failure"); //breakpoint(); logContext(); diff --git a/src/mongo/util/log.cpp b/src/mongo/util/log.cpp index 20f5e9fba51..980bb6a5490 100644 --- a/src/mongo/util/log.cpp +++ b/src/mongo/util/log.cpp @@ -336,10 +336,10 @@ namespace mongo { stringstream sss; sss << "warning: log line attempted (" << msg.size() / 1024 << "k) over max size(" << MAX_LOG_LINE / 1024 << "k)"; sss << ", printing beginning and end ... "; - b.appendStr( sss.str() ); + b.appendStr( sss.str(), false ); const char * xx = msg.c_str(); b.appendBuf( xx , MAX_LOG_LINE / 3 ); - b.appendStr( " .......... " ); + b.appendStr( " .......... ", false ); b.appendStr( xx + msg.size() - ( MAX_LOG_LINE / 3 ) ); } else { diff --git a/src/mongo/util/mmap_win.cpp b/src/mongo/util/mmap_win.cpp index 55cdc1ddeb8..599882f3c16 100644 --- a/src/mongo/util/mmap_win.cpp +++ b/src/mongo/util/mmap_win.cpp @@ -326,6 +326,9 @@ namespace mongo { return newPrivateView; } + // prevent WRITETODATAFILES() from running at the same time as FlushViewOfFile() + SimpleMutex globalFlushMutex("globalFlushMutex"); + class WindowsFlushable : public MemoryMappedFile::Flushable { public: WindowsFlushable( void * view , HANDLE fd , string filename , boost::shared_ptr<mutex> flushMutex ) @@ -336,6 +339,7 @@ namespace mongo { if (!_view || !_fd) return; + SimpleMutex::scoped_lock _globalFlushMutex(globalFlushMutex); scoped_lock lk(*_flushMutex); int loopCount = 0; diff --git a/src/mongo/util/processinfo_darwin.cpp b/src/mongo/util/processinfo_darwin.cpp index a28d2ad88d4..e8408f3c6d7 100644 --- a/src/mongo/util/processinfo_darwin.cpp +++ b/src/mongo/util/processinfo_darwin.cpp @@ -114,12 +114,25 @@ namespace mongo { typedef long long NumberVal; template <typename Variant> Variant getSysctlByName( const char * sysctlName ) { - char value[256]; - size_t len = sizeof(value); - if ( sysctlbyname(sysctlName, &value, &len, NULL, 0) < 0 ) { - log() << "Unable to resolve sysctl " << sysctlName << " (string) " << endl; + string value; + size_t len; + int status; + // NB: sysctlbyname is called once to determine the buffer length, and once to copy + // the sysctl value. Retry if the buffer length grows between calls. + do { + status = sysctlbyname(sysctlName, NULL, &len, NULL, 0); + if (status == -1) + break; + value.resize(len); + status = sysctlbyname(sysctlName, &*value.begin(), &len, NULL, 0); + } while (status == -1 && errno == ENOMEM); + if (status == -1) { + // unrecoverable error from sysctlbyname + log() << sysctlName << " unavailable" << endl; + return ""; } - return string(value, len - 1); + value.resize(len); + return value; } /** diff --git a/src/mongo/util/queue.h b/src/mongo/util/queue.h index 3bf43125ea5..1f43a0e6dee 100644 --- a/src/mongo/util/queue.h +++ b/src/mongo/util/queue.h @@ -59,7 +59,7 @@ namespace mongo { void push(T const& t) { scoped_lock l( _lock ); size_t tSize = _getSize(t); - while (_queue.size()+tSize >= _maxSize) { + while (_currentSize + tSize >= _maxSize) { _cvNoLongerFull.wait( l.boost() ); } _queue.push( t ); diff --git a/src/mongo/util/stacktrace.cpp b/src/mongo/util/stacktrace.cpp index 68528a187d4..032b5070b0e 100644 --- a/src/mongo/util/stacktrace.cpp +++ b/src/mongo/util/stacktrace.cpp @@ -161,6 +161,8 @@ namespace mongo { printWindowsStackTrace( context, os ); } + static SimpleMutex _stackTraceMutex( "stackTraceMutex" ); + /** * Print stack trace (using a specified stack context) to "os" * @@ -168,6 +170,7 @@ namespace mongo { * @param os ostream& to receive printed stack backtrace */ void printWindowsStackTrace( CONTEXT& context, std::ostream& os ) { + SimpleMutex::scoped_lock lk(_stackTraceMutex); HANDLE process = GetCurrentProcess(); BOOL ret = SymInitialize( process, NULL, TRUE ); if ( ret == FALSE ) { diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp index a5f3dd9cc4f..2d6ba3ae86b 100644 --- a/src/mongo/util/version.cpp +++ b/src/mongo/util/version.cpp @@ -43,7 +43,7 @@ namespace mongo { * 1.2.3-rc4-pre- * If you really need to do something else you'll need to fix _versionArray() */ - const char versionString[] = "2.2.0"; + const char versionString[] = "2.2.2"; // See unit test for example outputs static BSONArray _versionArray(const char* version){ |
