diff options
Diffstat (limited to 'src')
52 files changed, 1125 insertions, 445 deletions
diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 816a6bfacf8..265cd4800e0 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -433,6 +433,7 @@ serverOnlyFiles = [ "db/curop.cpp", "db/dbcommands_admin.cpp", # most commands are only for mongod + "db/commands/dbhash.cpp", "db/commands/fsync.cpp", "db/commands/distinct.cpp", "db/commands/find_and_modify.cpp", diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 1dfa1702b64..5fa8cde17e2 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -27,4 +27,7 @@ error_code("AlreadyInitialized", 23) error_code("LockTimeout", 24) error_code("RemoteValidationError", 25) +# Non-sequential error codes (for compatibility only) +error_code("NodeNotFound", 74) + error_class("NetworkError", ["HostUnreachable", "HostNotFound"]) diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 21a5b0810f4..cc82c2aa948 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -126,54 +126,6 @@ namespace mongo { } _populateReadPrefSecOkCmdList; /** - * @param ns the namespace of the query. - * @param queryOptionFlags the flags for the query. - * @param queryObj the query object to check. - * - * @return true if the given query can be sent to a secondary node without taking the - * slaveOk flag into account. - */ - bool _isQueryOkToSecondary(const string& ns, int queryOptionFlags, const BSONObj& queryObj) { - if (queryOptionFlags & QueryOption_SlaveOk) { - return true; - } - - if (!Query::hasReadPreference(queryObj)) { - return false; - } - - if (ns.find(".$cmd") == string::npos) { - return true; - } - - BSONObj actualQueryObj; - if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { - actualQueryObj = queryObj["query"].embeddedObject(); - } - else { - actualQueryObj = queryObj; - } - - const string cmdName = actualQueryObj.firstElementFieldName(); - if (_secOkCmdList.count(cmdName) == 1) { - return true; - } - - if (cmdName == "mapReduce" || cmdName == "mapreduce") { - if (!actualQueryObj.hasField("out")) { - return false; - } - - BSONElement outElem(actualQueryObj["out"]); - if (outElem.isABSONObj() && outElem["inline"].trueValue()) { - return true; - } - } - - return false; - } - - /** * Selects the right node given the nodes to pick from and the preference. * This method does strict tag matching, and will not implicitly fallback * to matching anything. @@ -275,15 +227,18 @@ namespace mongo { * * @param query the raw query document * - * @return the read preference setting. If the tags field was not present, it will contain one - * empty tag document {} which matches any tag. + * @return the read preference setting if a read preference exists, otherwise the default read + * preference of Primary_Only. If the tags field was not present, it will contain one + * empty tag document {} which matches any tag. * * @throws AssertionException if the read preference object is malformed */ - ReadPreferenceSetting* _extractReadPref(const BSONObj& query) { - ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; + ReadPreferenceSetting* _extractReadPref(const BSONObj& query, int queryOptions) { if (Query::hasReadPreference(query)) { + + ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; + BSONElement readPrefElement; if (query.hasField(Query::ReadPrefField.name())) { @@ -334,9 +289,17 @@ namespace mongo { return new ReadPreferenceSetting(pref, tags); } + else { + TagSet tags(BSON_ARRAY(BSONObj())); + return new ReadPreferenceSetting(pref, tags); + } } + // Default read pref is primary only or secondary preferred with slaveOK TagSet tags(BSON_ARRAY(BSONObj())); + ReadPreference pref = + queryOptions & QueryOption_SlaveOk ? + mongo::ReadPreference_SecondaryPreferred : mongo::ReadPreference_PrimaryOnly; return new ReadPreferenceSetting(pref, tags); } @@ -405,7 +368,6 @@ namespace mongo { // delete ReplicaSetMonitors from ReplicaSetMonitor::remove. ReplicaSetMonitor::~ReplicaSetMonitor() { scoped_lock lk ( _lock ); - log() << "deleting replica set monitor for: " << _getServerAddress_inlock() << endl; _cacheServerAddresses_inlock(); pool.removeHost( _getServerAddress_inlock() ); _nodes.clear(); @@ -1501,6 +1463,55 @@ namespace mongo { return rsm->getServerAddress(); } + // Internal implementation of isSecondaryQuery, takes previously-parsed read preference + static bool _isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + const ReadPreferenceSetting& readPref ) { + + // If the read pref is primary only, this is not a secondary query + if (readPref.pref == ReadPreference_PrimaryOnly) return false; + + if (ns.find(".$cmd") == string::npos) { + return true; + } + + // This is a command with secondary-possible read pref + // Only certain commands are supported for secondary operation. + + BSONObj actualQueryObj; + if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { + actualQueryObj = queryObj["query"].embeddedObject(); + } + else { + actualQueryObj = queryObj; + } + + const string cmdName = actualQueryObj.firstElementFieldName(); + if (_secOkCmdList.count(cmdName) == 1) { + return true; + } + + if (cmdName == "mapReduce" || cmdName == "mapreduce") { + if (!actualQueryObj.hasField("out")) { + return false; + } + + BSONElement outElem(actualQueryObj["out"]); + if (outElem.isABSONObj() && outElem["inline"].trueValue()) { + return true; + } + } + + return false; + } + + bool DBClientReplicaSet::isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ) { + auto_ptr<ReadPreferenceSetting> readPref( _extractReadPref( queryObj, queryOptions ) ); + return _isSecondaryQuery( ns, queryObj, *readPref ); + } + DBClientConnection * DBClientReplicaSet::checkMaster() { ReplicaSetMonitorPtr monitor = _getMonitor(); HostAndPort h = monitor->getMaster(); @@ -1593,6 +1604,18 @@ namespace mongo { return _getMonitor()->isAnyNodeOk(); } + void DBClientReplicaSet::authPrimary(const BSONObj& params) { + _auth(params); + } + + bool DBClientReplicaSet::authPrimary( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + return auth( dbname, username, password_text, errmsg, digestPassword ); + } + void DBClientReplicaSet::_auth(const BSONObj& params) { DBClientConnection * m = checkMaster(); @@ -1620,6 +1643,98 @@ namespace mongo { _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); } + bool DBClientReplicaSet::authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + try { + authAny(BSON(saslCommandMechanismFieldName << "MONGODB-CR" << + saslCommandPrincipalSourceFieldName << dbname << + saslCommandPrincipalFieldName << username << + saslCommandPasswordFieldName << password_text << + saslCommandDigestPasswordFieldName << digestPassword)); + return true; + } catch(const UserException& ex) { + if (ex.getCode() != ErrorCodes::AuthenticationFailed) + throw; + errmsg = ex.what(); + return false; + } + } + + static bool isAuthenticationException( const DBException& ex ) { + return ex.getCode() == ErrorCodes::AuthenticationFailed; + } + + void DBClientReplicaSet::authAny( const BSONObj& params ) { + + // We prefer to authenticate against a primary, but otherwise a secondary is ok too + // Empty tag matches every secondary + TagSet tags(BSON_ARRAY(BSONObj())); + shared_ptr<ReadPreferenceSetting> readPref( + new ReadPreferenceSetting( ReadPreference_PrimaryPreferred, tags ) ); + + LOG(3) << "dbclient_rs authentication of " << _getMonitor()->getName() << endl; + + // NOTE that we retry MAX_RETRY + 1 times, since we're always primary preferred we don't + // fallback to the primary. + Status lastNodeStatus = Status::OK(); + for ( size_t retry = 0; retry < MAX_RETRY + 1; retry++ ) { + try { + DBClientConnection* conn = selectNodeUsingTags( readPref ); + + if ( conn == NULL ) { + break; + } + + conn->auth( params ); + + // Cache the new auth information since we've now validated it's good + _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); + + // Ensure the only child connection open is the one we authenticated against - other + // child connections may not have full authentication information. + // NOTE: _lastSlaveOkConn may or may not be the same as _master + dassert(_lastSlaveOkConn.get() == conn || _master.get() == conn); + if ( conn != _lastSlaveOkConn.get() ) { + _lastSlaveOkHost = HostAndPort(); + _lastSlaveOkConn.reset(); + } + if ( conn != _master.get() ) { + _masterHost = HostAndPort(); + _master.reset(); + } + + return; + } + catch ( const DBException &ex ) { + + // We care if we can't authenticate (i.e. bad password) in credential params. + if ( isAuthenticationException( ex ) ) { + throw; + } + + StringBuilder errMsgB; + errMsgB << "can't authenticate against replica set node " + << _lastSlaveOkHost.toString(); + lastNodeStatus = ex.toStatus( errMsgB.str() ); + + LOG(1) << lastNodeStatus.reason() << endl; + invalidateLastSlaveOkCache(); + } + } + + if ( lastNodeStatus.isOK() ) { + StringBuilder assertMsgB; + assertMsgB << "Failed to authenticate, no good nodes in " << _getMonitor()->getName(); + uasserted( ErrorCodes::NodeNotFound, assertMsgB.str() ); + } + else { + uasserted( lastNodeStatus.code(), lastNodeStatus.reason() ); + } + } + void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) { DBClientConnection* priConn = checkMaster(); @@ -1668,9 +1783,8 @@ namespace mongo { int queryOptions, int batchSize) { - if ( _isQueryOkToSecondary( ns, queryOptions, query.obj ) ) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs query using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1718,9 +1832,9 @@ namespace mongo { const Query& query, const BSONObj *fieldsToReturn, int queryOptions) { - if (_isQueryOkToSecondary(ns, queryOptions, query.obj)) { - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs findOne using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1872,17 +1986,15 @@ namespace mongo { _lazyState = LazyState(); const int lastOp = toSend.operation(); - bool slaveOk = false; if (lastOp == dbQuery) { // TODO: might be possible to do this faster by changing api DbMessage dm(toSend); QueryMessage qm(dm); - const bool slaveOk = qm.queryOptions & QueryOption_SlaveOk; - if (_isQueryOkToSecondary(qm.ns, qm.queryOptions, qm.query)) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( qm.ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs say using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1910,7 +2022,7 @@ namespace mongo { conn->say(toSend); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._isSecondaryQuery = true; _lazyState._lastClient = conn; } catch ( const DBException& DBExcep ) { @@ -1936,7 +2048,7 @@ namespace mongo { *actualServer = master->getServerAddress(); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._isSecondaryQuery = false; // Don't retry requests to primary since there is only one host to try _lazyState._retries = MAX_RETRY; _lazyState._lastClient = master; @@ -1981,7 +2093,7 @@ namespace mongo { if( nReturned == 1 ) dataObj = BSONObj( data ); // Check if we should retry here - if( _lazyState._lastOp == dbQuery && _lazyState._slaveOk ){ + if( _lazyState._lastOp == dbQuery && _lazyState._isSecondaryQuery ){ // Check the error code for a slave not secondary error if( nReturned == -1 || @@ -2036,9 +2148,9 @@ namespace mongo { QueryMessage qm(dm); ns = qm.ns; - if (_isQueryOkToSecondary(ns, qm.queryOptions, qm.query)) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs call using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index 05f7faa9b2b..d4c9512576f 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -518,10 +518,65 @@ namespace mongo { virtual bool call( Message &toSend, Message &response, bool assertOk=true , string * actualServer = 0 ); virtual bool callRead( Message& toSend , Message& response ) { return checkMaster()->callRead( toSend , response ); } + /** + * Authenticate using supplied credentials. Authenticates against the primary node, fails + * if node is down. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + * + * This is the default authentication mode for DBClientReplicaSet connections. + */ + void authPrimary(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authPrimary(const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword); + + /** + * Authenticate using supplied credentials. Prefers authentication against the primary + * node, will fall back to a secondary and retry if the primary node is down but + * secondaries are still available. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + */ + void authAny(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ); + + /* + * Returns whether a query or command can be sent to secondaries based on the query object + * and options. + * + * @param ns the namespace of the query. + * @param queryObj the query object to check. + * @param queryOptions the query options + * + * @return true if the query/cmd could potentially be sent to a secondary, false otherwise + */ + static bool isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ); protected: - /** Authorize. Authorizes all nodes as needed - */ + virtual void _auth(const BSONObj& params); virtual void sayPiggyBack( Message &toSend ) { checkMaster()->say( toSend ); } @@ -606,10 +661,12 @@ namespace mongo { */ class LazyState { public: - LazyState() : _lastClient( NULL ), _lastOp( -1 ), _slaveOk( false ), _retries( 0 ) {} + LazyState() : + _lastClient( NULL ), _lastOp( -1 ), _isSecondaryQuery( false ), _retries( 0 ) { + } DBClientConnection* _lastClient; int _lastOp; - bool _slaveOk; + bool _isSecondaryQuery; int _retries; } _lazyState; diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index 18649d9c6b9..7dcc7573df0 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -715,26 +715,28 @@ namespace mongo { } const DBClientBase* rawConn = state->conn->getRawConn(); - if (( _options & QueryOption_SlaveOk ) && - rawConn->type() == ConnectionString::SET && - rawConn->isFailed() ) { - /* A side effect of this short circuiting is this will not be - * able figure out that the primary is now up on it's own and - * has to rely on other threads to refresh the node states. - */ + bool allowShardVersionFailure = + rawConn->type() == ConnectionString::SET && + DBClientReplicaSet::isSecondaryQuery( _qSpec.ns(), _qSpec.query(), _qSpec.options() ); + + if ( allowShardVersionFailure && rawConn->isFailed() ) { + + state->conn->donotCheckVersion(); + + // A side effect of this short circuiting is the mongos will not be able figure out that + // the primary is now up on it's own and has to rely on other threads to refresh node + // states. OCCASIONALLY { - const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( rawConn ); + const DBClientReplicaSet* repl = dynamic_cast<const DBClientReplicaSet*>( rawConn ); + dassert(repl); warning() << "Primary for " << repl->getServerAddress() << " was down before, bypassing setShardVersion." - << " Local config view can be stale." << endl; + << " The local replica set view and targeting may be stale." << endl; } - } else { + } + else { try { - /* TODO: Undo SERVER-5797. This try-catch is a temporary hack until - * secondaries can properly handle shard versioning - */ if ( state->conn->setVersion() ) { // It's actually okay if we set the version here, since either the // manager will be verified as compatible, or if the manager doesn't @@ -742,19 +744,20 @@ namespace mongo { LOG( pc ) << "needed to set remote version on connection to value " << "compatible with " << vinfo << endl; } - } catch ( const DBException& dbEx ) { - if ( (dbEx.getCode() == 10009 /* no master */ && - ( _options & QueryOption_SlaveOk )) ) { + } + catch ( const DBException& dbEx ) { + if ( allowShardVersionFailure ) { + + // It's okay if we don't set the version when talking to a secondary, we can + // be stale in any case. OCCASIONALLY { const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( - state->conn->getRawConn() ); - - warning() << "Cannot contact primary for " - << repl->getServerAddress() - << " to check shard version. " - << "SlaveOk query can be sent to the wrong shard." + dynamic_cast<const DBClientReplicaSet*>( state->conn->getRawConn() ); + dassert(repl); + warning() << "Cannot contact primary for " << repl->getServerAddress() + << " to check shard version." + << " The local replica set view and targeting may be stale." << endl; } } diff --git a/src/mongo/db/auth/auth_external_state_s.cpp b/src/mongo/db/auth/auth_external_state_s.cpp index b7167cef509..7e890636cdf 100644 --- a/src/mongo/db/auth/auth_external_state_s.cpp +++ b/src/mongo/db/auth/auth_external_state_s.cpp @@ -49,9 +49,11 @@ namespace mongo { } bool AuthExternalStateMongos::_findUser(const string& usersNamespace, - const BSONObj& query, + const BSONObj& queryDoc, BSONObj* result) const { scoped_ptr<ScopedDbConnection> conn(getConnectionForUsersCollection(usersNamespace)); + Query query(queryDoc); + query.readPref(ReadPreference_PrimaryPreferred, BSONArray()); *result = conn->get()->findOne(usersNamespace, query).getOwned(); conn->done(); return !result->isEmpty(); diff --git a/src/mongo/db/auth/authorization_manager.cpp b/src/mongo/db/auth/authorization_manager.cpp index 1cf8efede39..f30443eda1b 100644 --- a/src/mongo/db/auth/authorization_manager.cpp +++ b/src/mongo/db/auth/authorization_manager.cpp @@ -443,6 +443,18 @@ namespace { return _authenticatedPrincipals.getNames(); } + std::string AuthorizationManager::getAuthenticatedPrincipalNamesToken() { + std::string ret; + for (PrincipalSet::NameIterator nameIter = getAuthenticatedPrincipalNames(); + nameIter.more(); + nameIter.next()) { + ret += '\0'; // Using a NUL byte which isn't valid in usernames to separate them. + ret += nameIter->getFullName(); + } + + return ret; + } + Status AuthorizationManager::acquirePrivilege(const Privilege& privilege, const PrincipalName& authorizingPrincipal) { if (!_authenticatedPrincipals.lookup(authorizingPrincipal)) { diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h index a32710557dd..7131d7624a6 100644 --- a/src/mongo/db/auth/authorization_manager.h +++ b/src/mongo/db/auth/authorization_manager.h @@ -91,6 +91,10 @@ namespace mongo { // Gets an iterator over the names of all authenticated principals stored in this manager. PrincipalSet::NameIterator getAuthenticatedPrincipalNames(); + // Returns a string representing all logged-in principals on the current session. + // WARNING: this string will contain NUL bytes so don't call c_str()! + std::string getAuthenticatedPrincipalNamesToken(); + // Removes any authenticated principals whose authorization credentials came from the given // database, and revokes any privileges that were granted via that principal. void logoutDatabase(const std::string& dbname); diff --git a/src/mongo/db/cmdline.cpp b/src/mongo/db/cmdline.cpp index 0dea50faeef..c14cc603b56 100644 --- a/src/mongo/db/cmdline.cpp +++ b/src/mongo/db/cmdline.cpp @@ -151,9 +151,14 @@ namespace { if ( s.find( "FASTSYNC" ) != string::npos ) cout << "warning \"fastsync\" should not be put in your configuration file" << endl; - if ( s.c_str()[0] == '#' ) { - // skipping commented line - } else if ( s.find( "=FALSE" ) == string::npos ) { + // skip commented lines + if ( s.c_str()[0] == '#' ) { + // In this block, we copy the actual line into our intermediate buffer to actually be + // parsed later only if the string does not contain the substring "=FALSE" OR the option + // is a setParameter option. Note that this is done after we call boost::to_upper + // above. + } else if ( s.find( "=FALSE" ) == string::npos || + s.find( "SETPARAMETER" ) == 0 ) { ss << line << endl; } else { cout << "warning: remove or comment out this line by starting it with \'#\', skipping now : " << line << endl; diff --git a/src/mongo/db/commands/dbhash.cpp b/src/mongo/db/commands/dbhash.cpp new file mode 100644 index 00000000000..38a07cace56 --- /dev/null +++ b/src/mongo/db/commands/dbhash.cpp @@ -0,0 +1,216 @@ +// dbhash.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/commands/dbhash.h" + +#include "mongo/db/btreecursor.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/database.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/timer.h" + +namespace mongo { + + DBHashCmd dbhashCmd; + + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt ) { + dbhashCmd.wipeCacheForCollection( ns ); + } + + // ---- + + DBHashCmd::DBHashCmd() + : Command( "dbHash", false, "dbhash" ), + _cachedHashedMutex( "_cachedHashedMutex" ){ + } + + void DBHashCmd::addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector<Privilege>* out) { + ActionSet actions; + actions.addAction(ActionType::dbHash); + out->push_back(Privilege(dbname, actions)); + } + + string DBHashCmd::hashCollection( const string& fullCollectionName, bool* fromCache ) { + + scoped_ptr<scoped_lock> cachedHashedLock; + + if ( isCachable( fullCollectionName ) ) { + cachedHashedLock.reset( new scoped_lock( _cachedHashedMutex ) ); + string hash = _cachedHashed[fullCollectionName]; + if ( hash.size() > 0 ) { + *fromCache = true; + return hash; + } + } + + *fromCache = false; + NamespaceDetails * nsd = nsdetails( fullCollectionName ); + verify( nsd ); + + // debug SERVER-761 + NamespaceDetails::IndexIterator ii = nsd->ii(); + while( ii.more() ) { + const IndexDetails &idx = ii.next(); + if ( !idx.head.isValid() || !idx.info.isValid() ) { + log() << "invalid index for ns: " << fullCollectionName << " " << idx.head << " " << idx.info; + if ( idx.info.isValid() ) + log() << " " << idx.info.obj(); + log() << endl; + } + } + + int idNum = nsd->findIdIndex(); + + shared_ptr<Cursor> cursor; + + if ( idNum >= 0 ) { + cursor.reset( BtreeCursor::make( nsd, + nsd->idx( idNum ), + BSONObj(), + BSONObj(), + false, + 1 ) ); + } + else if ( nsd->isCapped() ) { + cursor = findTableScan( fullCollectionName.c_str() , BSONObj() ); + } + else { + log() << "can't find _id index for: " << fullCollectionName << endl; + return "no _id _index"; + } + + md5_state_t st; + md5_init(&st); + + long long n = 0; + + while ( cursor->ok() ) { + BSONObj c = cursor->current(); + md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); + n++; + cursor->advance(); + } + + md5digest d; + md5_finish(&st, d); + string hash = digestToString( d ); + + if ( cachedHashedLock.get() ) { + _cachedHashed[fullCollectionName] = hash; + } + + return hash; + } + + bool DBHashCmd::run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + Timer timer; + + set<string> desiredCollections; + if ( cmdObj["collections"].type() == Array ) { + BSONObjIterator i( cmdObj["collections"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( e.type() != String ) { + errmsg = "collections entries have to be strings"; + return false; + } + desiredCollections.insert( e.String() ); + } + } + + list<string> colls; + Database* db = cc().database(); + if ( db ) + db->namespaceIndex.getNamespaces( colls ); + colls.sort(); + + result.appendNumber( "numCollections" , (long long)colls.size() ); + result.append( "host" , prettyHostName() ); + + md5_state_t globalState; + md5_init(&globalState); + + vector<string> cached; + + BSONObjBuilder bb( result.subobjStart( "collections" ) ); + for ( list<string>::iterator i=colls.begin(); i != colls.end(); i++ ) { + string fullCollectionName = *i; + string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); + + if ( shortCollectionName.find( "system." ) == 0 ) + continue; + + if ( desiredCollections.size() > 0 && + desiredCollections.count( shortCollectionName ) == 0 ) + continue; + + bool fromCache = false; + string hash = hashCollection( fullCollectionName, &fromCache ); + + bb.append( shortCollectionName, hash ); + + md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); + if ( fromCache ) + cached.push_back( fullCollectionName ); + } + bb.done(); + + md5digest d; + md5_finish(&globalState, d); + string hash = digestToString( d ); + + result.append( "md5" , hash ); + result.appendNumber( "timeMillis", timer.millis() ); + + result.append( "fromCache", cached ); + + return 1; + } + + void DBHashCmd::wipeCacheForCollection( const StringData& ns ) { + if ( !isCachable( ns ) ) + return; + scoped_lock lk( _cachedHashedMutex ); + _cachedHashed.erase( ns.toString() ); + } + + bool DBHashCmd::isCachable( const StringData& ns ) const { + return ns.startsWith( "config." ); + } + +} diff --git a/src/mongo/db/commands/dbhash.h b/src/mongo/db/commands/dbhash.h new file mode 100644 index 00000000000..262c6609868 --- /dev/null +++ b/src/mongo/db/commands/dbhash.h @@ -0,0 +1,67 @@ +// dbhash.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/commands.h" + +namespace mongo { + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt ); + + class DBHashCmd : public Command { + public: + DBHashCmd(); + + virtual bool slaveOk() const { return true; } + virtual LockType locktype() const { return READ; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector<Privilege>* out); + + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool); + + void wipeCacheForCollection( const StringData& ns ); + + private: + + bool isCachable( const StringData& ns ) const; + + string hashCollection( const string& fullCollectionName, bool* fromCache ); + + map<string,string> _cachedHashed; + mutex _cachedHashedMutex; + + }; + +} diff --git a/src/mongo/db/commands/group.cpp b/src/mongo/db/commands/group.cpp index 441a1192905..20481521e91 100644 --- a/src/mongo/db/commands/group.cpp +++ b/src/mongo/db/commands/group.cpp @@ -20,9 +20,11 @@ #include <vector> +#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/client_basic.h" #include "mongo/db/commands.h" #include "mongo/db/instance.h" #include "mongo/scripting/engine.h" @@ -72,7 +74,9 @@ namespace mongo { string& errmsg, BSONObjBuilder& result ) { - auto_ptr<Scope> s = globalScriptEngine->getPooledScope( realdbname, "group"); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + auto_ptr<Scope> s = globalScriptEngine->getPooledScope(realdbname, "group" + userToken); if ( reduceScope ) s->init( reduceScope ); diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 9528e495ded..1b237d3ed7f 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -22,6 +22,7 @@ #include "mongo/client/connpool.h" #include "mongo/client/parallel.h" +#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/db.h" @@ -300,8 +301,13 @@ namespace mongo { */ void State::dropTempCollections() { _db.dropCollection(_config.tempNamespace); - if (_useIncremental) + // Always forget about temporary namespaces, so we don't cache lots of them + ShardConnection::forgetNS( _config.tempNamespace ); + if (_useIncremental) { _db.dropCollection(_config.incLong); + ShardConnection::forgetNS( _config.incLong ); + } + } /** @@ -622,7 +628,10 @@ namespace mongo { */ void State::init() { // setup js - _scope.reset(globalScriptEngine->getPooledScope( _config.dbname, "mapreduce" ).release() ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + _scope.reset(globalScriptEngine->getPooledScope( + _config.dbname, "mapreduce" + userToken).release()); if ( ! _config.scopeSetup.isEmpty() ) _scope->init( &_config.scopeSetup ); @@ -1165,9 +1174,23 @@ namespace mongo { state.init(); state.prepTempCollection(); ON_BLOCK_EXIT_OBJ(state, &State::dropTempCollections); - ProgressMeterHolder pm(op->setMessage("m/r: (1/3) emit phase", - "M/R: (1/3) Emit Progress", - state.incomingDocuments())); + + int progressTotal = 0; + bool showTotal = true; + if ( state.config().filter.isEmpty() ) { + progressTotal = state.incomingDocuments(); + } + else { + showTotal = false; + // Set an arbitrary total > 0 so the meter will be activated. + progressTotal = 1; + } + + ProgressMeter& progress( op->setMessage("m/r: (1/3) emit phase", + "M/R: (1/3) Emit Progress", + progressTotal )); + progress.showTotal(showTotal); + ProgressMeterHolder pm(progress); wassert( config.limit < 0x4000000 ); // see case on next line to 32 bit unsigned long long mapTime = 0; @@ -1457,6 +1480,9 @@ namespace mongo { break; } + // Forget temporary input collection, if output is sharded collection + ShardConnection::forgetNS( inputNS ); + result.append( "chunkSizes" , chunkSizes.arr() ); long long outputCount = state.postProcessCollection(op, pm); diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2361ec2a945..ec337c89885 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -232,7 +232,6 @@ namespace mongo { virtual void disconnected( AbstractMessagingPort* p ) { Client * c = currentClient.get(); if( c ) c->shutdown(); - globalScriptEngine->threadDone(); } }; diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index 33b77f455d6..fb6be132171 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -178,9 +178,9 @@ namespace mongo { } } - if ( err ) { + if ( err && cmdObj["wOpTime"].eoo() ) { // doesn't make sense to wait for replication - // if there was an error + // if there was an error and we aren't explicitly waiting for another wOpTime return true; } @@ -200,7 +200,14 @@ namespace mongo { long long passes = 0; char buf[32]; - OpTime op(c.getLastOp()); + + OpTime op; + if ( cmdObj["wOpTime"].type() == Timestamp ) { + op = OpTime( cmdObj["wOpTime"].date() ); + } + else { + op = c.getLastOp(); + } if ( op.isNull() ) { if ( anyReplEnabled() ) { @@ -1689,122 +1696,6 @@ namespace mongo { } return Status::OK(); } - - class DBHashCmd : public Command { - public: - DBHashCmd() : Command( "dbHash", false, "dbhash" ) {} - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return READ; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector<Privilege>* out) { - ActionSet actions; - actions.addAction(ActionType::dbHash); - out->push_back(Privilege(dbname, actions)); - } - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - Timer timer; - - set<string> desiredCollections; - if ( cmdObj["collections"].type() == Array ) { - BSONObjIterator i( cmdObj["collections"].Obj() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.type() != String ) { - errmsg = "collections entries have to be strings"; - return false; - } - desiredCollections.insert( e.String() ); - } - } - - list<string> colls; - Database* db = cc().database(); - if ( db ) - db->namespaceIndex.getNamespaces( colls ); - colls.sort(); - - result.appendNumber( "numCollections" , (long long)colls.size() ); - result.append( "host" , prettyHostName() ); - - md5_state_t globalState; - md5_init(&globalState); - - BSONObjBuilder bb( result.subobjStart( "collections" ) ); - for ( list<string>::iterator i=colls.begin(); i != colls.end(); i++ ) { - string fullCollectionName = *i; - string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); - - if ( shortCollectionName.find( "system." ) == 0 ) - continue; - - if ( desiredCollections.size() > 0 && - desiredCollections.count( shortCollectionName ) == 0 ) - continue; - - shared_ptr<Cursor> cursor; - - NamespaceDetails * nsd = nsdetails( fullCollectionName ); - - // debug SERVER-761 - NamespaceDetails::IndexIterator ii = nsd->ii(); - while( ii.more() ) { - const IndexDetails &idx = ii.next(); - if ( !idx.head.isValid() || !idx.info.isValid() ) { - log() << "invalid index for ns: " << fullCollectionName << " " << idx.head << " " << idx.info; - if ( idx.info.isValid() ) - log() << " " << idx.info.obj(); - log() << endl; - } - } - - int idNum = nsd->findIdIndex(); - if ( idNum >= 0 ) { - cursor.reset( BtreeCursor::make( nsd, - nsd->idx( idNum ), - BSONObj(), - BSONObj(), - false, - 1 ) ); - } - else if ( nsd->isCapped() ) { - cursor = findTableScan( fullCollectionName.c_str() , BSONObj() ); - } - else { - log() << "can't find _id index for: " << fullCollectionName << endl; - continue; - } - - md5_state_t st; - md5_init(&st); - - long long n = 0; - while ( cursor->ok() ) { - BSONObj c = cursor->current(); - md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); - n++; - cursor->advance(); - } - md5digest d; - md5_finish(&st, d); - string hash = digestToString( d ); - - bb.append( shortCollectionName, hash ); - - md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); - } - bb.done(); - - md5digest d; - md5_finish(&globalState, d); - string hash = digestToString( d ); - - result.append( "md5" , hash ); - result.appendNumber( "timeMillis", timer.millis() ); - return 1; - } - - } dbhashCmd; /* for diagnostic / testing purposes. Enabled via command line. */ class CmdSleep : public Command { diff --git a/src/mongo/db/dbeval.cpp b/src/mongo/db/dbeval.cpp index 5a6cc464c34..9e6a3360a49 100644 --- a/src/mongo/db/dbeval.cpp +++ b/src/mongo/db/dbeval.cpp @@ -57,7 +57,9 @@ namespace mongo { return false; } - auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken ); ScriptingFunction f = s->createFunction(code); if ( f == 0 ) { errmsg = (string)"compile failed: " + s->getError(); diff --git a/src/mongo/db/fts/fts_spec.cpp b/src/mongo/db/fts/fts_spec.cpp index eb534f10a3e..f1829104796 100644 --- a/src/mongo/db/fts/fts_spec.cpp +++ b/src/mongo/db/fts/fts_spec.cpp @@ -30,10 +30,25 @@ namespace mongo { const double MAX_WEIGHT = 1000000000.0; const double MAX_WORD_WEIGHT = MAX_WEIGHT / 10000; + const int TEXT_INDEX_VERSION = 1; FTSSpec::FTSSpec( const BSONObj& indexInfo ) { - massert( 16739, "found invalid spec for text index", + // indexInfo is a text index spec. Text index specs pass through fixSpec() before + // being saved to the system.indexes collection. fixSpec() enforces a schema, such that + // required fields must exist and be of the correct type (e.g. weights, + // textIndexVersion). + massert( 16739, + "found invalid spec for text index, expected object for weights", indexInfo["weights"].isABSONObj() ); + BSONElement textIndexVersionElt = indexInfo["textIndexVersion"]; + massert( 17287, + "found invalid spec for text index, expected number for textIndexVersion", + textIndexVersionElt.isNumber() ); + massert( 17288, + str::stream() << "attempt to use unsupported textIndexVersion " << + textIndexVersionElt.numberInt() << ", only textIndexVersion " << + TEXT_INDEX_VERSION << " supported", + textIndexVersionElt.numberInt() == TEXT_INDEX_VERSION ); _defaultLanguage = indexInfo["default_language"].valuestrsafe(); _languageOverrideField = indexInfo["language_override"].valuestrsafe(); @@ -357,7 +372,7 @@ namespace mongo { language_override = "language"; int version = -1; - int textIndexVersion = 1; + int textIndexVersion = TEXT_INDEX_VERSION; BSONObjBuilder b; BSONObjIterator i( spec ); @@ -385,7 +400,7 @@ namespace mongo { textIndexVersion = e.numberInt(); uassert( 16730, str::stream() << "bad textIndexVersion: " << textIndexVersion, - textIndexVersion == 1 ); + textIndexVersion == TEXT_INDEX_VERSION ); } else { b.append( e ); diff --git a/src/mongo/db/fts/fts_spec_test.cpp b/src/mongo/db/fts/fts_spec_test.cpp index df4ff719f02..00c95261207 100644 --- a/src/mongo/db/fts/fts_spec_test.cpp +++ b/src/mongo/db/fts/fts_spec_test.cpp @@ -24,6 +24,36 @@ namespace mongo { namespace fts { + const BSONObj makeFixedSpec( int textIndexVersion ) { + return BSON( "v" << 1 << + "key" << BSON( "_fts" << "text" << + "_ftsx" << 1 ) << + "name" << "a_text" << + "ns" << "test.foo" << + "weights" << BSON( "a" << 1 ) << + "default_language" << "english" << + "language_override" << "language" << + "textIndexVersion" << textIndexVersion ); + } + + TEST( FTSSpec, TextIndexVersionCheck1 ) { + const int currentVersion = 1; + const int unsupportedVersion = 2; + + // Constructing an FTSSpec with the current textIndexVersion should succeed. + BSONObj validTextSpec = makeFixedSpec( currentVersion ); + try { + FTSSpec spec( validTextSpec ); + } + catch ( DBException& e ) { + ASSERT( false ); + } + + // Constructing an FTSSpec with an unsupported textIndexVersion should fail. + BSONObj invalidTextSpec = makeFixedSpec( unsupportedVersion ); + ASSERT_THROWS( FTSSpec spec( invalidTextSpec ), DBException ); + } + TEST( FTSSpec, Fix1 ) { BSONObj user = BSON( "key" << BSON( "title" << "fts" << "text" << "fts" ) << diff --git a/src/mongo/db/geo/geoparser.cpp b/src/mongo/db/geo/geoparser.cpp index 3de520be0be..ed86a2c3fa9 100644 --- a/src/mongo/db/geo/geoparser.cpp +++ b/src/mongo/db/geo/geoparser.cpp @@ -39,8 +39,15 @@ namespace mongo { static const string GEOJSON_COORDINATES = "coordinates"; //// Utility functions used by GeoParser functions below. - static S2Point coordToPoint(double p0, double p1) { - return S2LatLng::FromDegrees(p1, p0).Normalized().ToPoint(); + static S2Point coordToPoint(double lng, double lat) { + // Note that it's (lat, lng) for S2 but (lng, lat) for MongoDB. + S2LatLng ll = S2LatLng::FromDegrees(lat, lng).Normalized(); + if (!ll.is_valid()) { + stringstream ss; + ss << "coords invalid after normalization, lng = " << lng << " lat = " << lat << endl; + uasserted(17125, ss.str()); + } + return ll.ToPoint(); } static S2Point coordsToPoint(const vector<BSONElement>& coordElt) { diff --git a/src/mongo/db/matcher.cpp b/src/mongo/db/matcher.cpp index 2e679591f2c..532c18e1fe0 100644 --- a/src/mongo/db/matcher.cpp +++ b/src/mongo/db/matcher.cpp @@ -27,6 +27,7 @@ #include "db.h" #include "queryutil.h" #include "client.h" +#include "mongo/db/auth/authorization_manager.h" #include "pdfile.h" @@ -74,8 +75,10 @@ namespace mongo { return; _initCalled = true; + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); NamespaceString ns( _ns ); - _scope = globalScriptEngine->getPooledScope( ns.db.c_str(), "where" ); + _scope = globalScriptEngine->getPooledScope( ns.db.c_str(), "where" + userToken ); massert( 10341 , "code has to be set first!" , ! _jsCode.empty() ); @@ -429,6 +432,8 @@ namespace mongo { uassert( 10066 , "$where may only appear once in query", _where == 0 ); uassert( 10067 , "$where query, but no script engine", globalScriptEngine ); massert( 13089 , "no current client needed for $where" , haveClient() ); + uassert( 17126 , "no valid context found for $where", cc().getContext()); + _where = new Where( cc().ns() ); if ( e.type() == CodeWScope ) { diff --git a/src/mongo/db/namespace_details.cpp b/src/mongo/db/namespace_details.cpp index 94adac276a6..0fc7bdf5eb5 100644 --- a/src/mongo/db/namespace_details.cpp +++ b/src/mongo/db/namespace_details.cpp @@ -976,6 +976,10 @@ namespace mongo { bool legalClientSystemNS( const string& ns , bool write ) { if( ns == "local.system.replset" ) return true; + if( ns == "admin.system.version" ) return true; + if( ns == "admin.system.roles" ) return true; + if( ns == "admin.system.new_users" ) return true; + if( ns == "admin.system.backup_users" ) return true; if ( ns.find( ".system.users" ) != string::npos ) return true; diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp index 12f4742c256..b88ad66ac03 100644 --- a/src/mongo/db/oplog.cpp +++ b/src/mongo/db/oplog.cpp @@ -26,6 +26,7 @@ #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/dbhash.h" #include "mongo/db/index_update.h" #include "mongo/db/instance.h" #include "mongo/db/namespacestring.h" @@ -334,6 +335,7 @@ namespace mongo { } logOpForSharding( opstr , ns , obj , patt ); + logOpForDbHash( opstr , ns , obj , patt ); } void createOplog() { @@ -977,13 +979,17 @@ namespace mongo { BSONElement e = i.next(); const BSONObj& temp = e.Obj(); - Client::Context ctx(temp["ns"].String()); + string ns = temp["ns"].String(); + Client::Context ctx(ns); + bool failed = applyOperation_inlock(temp, false, alwaysUpsert); ab.append(!failed); if ( failed ) errors++; num++; + + logOpForDbHash( "u", ns.c_str(), BSONObj(), NULL ); } result.append( "applied" , num ); diff --git a/src/mongo/db/pdfile.cpp b/src/mongo/db/pdfile.cpp index 928b8560ced..4085dfd2b3f 100644 --- a/src/mongo/db/pdfile.cpp +++ b/src/mongo/db/pdfile.cpp @@ -267,7 +267,7 @@ namespace mongo { BSONElement e = options.getField("size"); if ( e.isNumber() ) { size = e.numberLong(); - uassert( 10083 , "create collection invalid size spec", size > 0 ); + uassert( 10083 , "create collection invalid size spec", size >= 0 ); size += 0xff; size &= 0xffffffffffffff00LL; diff --git a/src/mongo/db/pipeline/document.h b/src/mongo/db/pipeline/document.h index b51efa91497..5e7cf3f0579 100644 --- a/src/mongo/db/pipeline/document.h +++ b/src/mongo/db/pipeline/document.h @@ -162,6 +162,7 @@ namespace mongo { friend class FieldIterator; friend class ValueStorage; friend class MutableDocument; + friend class MutableValue; const DocumentStorage& storage() const { return (_storage ? *_storage : DocumentStorage::emptyDoc()); @@ -194,8 +195,17 @@ namespace mongo { /// Used by MutableDocument(MutableValue) const RefCountable*& getDocPtr() { - if (_val.getType() != Object) - *this = Value(Document()); + if (_val.getType() != Object || _val._storage.genericRCPtr == NULL) { + // If the current value isn't an object we replace it with a Object-typed Value. + // Note that we can't just use Document() here because that is a NULL pointer and + // Value doesn't refcount NULL pointers. This led to a memory leak (SERVER-10554) + // because MutableDocument::newStorage() would set a non-NULL pointer into the Value + // without setting the refCounter bit. While allocating a DocumentStorage here could + // result in an allocation where none is needed, in practice this is only called + // when we are about to add a field to the sub-document so this just changes where + // the allocation is done. + _val = Value(Document(new DocumentStorage())); + } return _val._storage.genericRCPtr; } diff --git a/src/mongo/db/pipeline/value.cpp b/src/mongo/db/pipeline/value.cpp index 5561cb41a59..207dc4e37b8 100644 --- a/src/mongo/db/pipeline/value.cpp +++ b/src/mongo/db/pipeline/value.cpp @@ -28,6 +28,48 @@ namespace mongo { using namespace mongoutils; + void ValueStorage::verifyRefCountingIfShould() const { + switch (type) { + case MinKey: + case MaxKey: + case jstOID: + case Date: + case Timestamp: + case EOO: + case jstNULL: + case Undefined: + case Bool: + case NumberInt: + case NumberLong: + case NumberDouble: + // the above types never reference external data + verify(!refCounter); + break; + + case String: + case RegEx: + case Code: + case Symbol: + // the above types reference data when not using short-string optimization + verify(refCounter == !shortStr); + break; + + case BinData: // TODO this should probably support short-string optimization + case Array: // TODO this should probably support empty-is-NULL optimization + case DBRef: + case CodeWScope: + // the above types always reference external data. + verify(refCounter); + verify(bool(genericRCPtr)); + break; + + case Object: + // Objects either hold a NULL ptr or should be ref-counting + verify(refCounter == bool(genericRCPtr)); + break; + } + } + void ValueStorage::putString(const StringData& s) { // Note: this also stores data portion of BinData const size_t sizeNoNUL = s.size(); diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index e3481951f9e..ffc7f42a7f7 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -89,6 +89,7 @@ namespace mongo { } ~ValueStorage() { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_release(genericRCPtr); DEV memset(this, 0xee, sizeof(*this)); @@ -109,6 +110,7 @@ namespace mongo { /// Call this after memcpying to update ref counts if needed void memcpyed() const { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_add_ref(genericRCPtr); } @@ -140,6 +142,7 @@ namespace mongo { intrusive_ptr_add_ref(genericRCPtr); refCounter = true; } + DEV verifyRefCountingIfShould(); } StringData getString() const { @@ -191,6 +194,8 @@ namespace mongo { && i64[1] == other.i64[1]); } + void verifyRefCountingIfShould() const; + // This data is public because this should only be used by Value which would be a friend union { struct { diff --git a/src/mongo/db/repl/consensus.cpp b/src/mongo/db/repl/consensus.cpp index f056befb605..dcb31408c11 100644 --- a/src/mongo/db/repl/consensus.cpp +++ b/src/mongo/db/repl/consensus.cpp @@ -61,7 +61,9 @@ namespace mongo { return true; } - if (primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime) { + if (primary && + (hopeful->hbinfo().id() != primary->hbinfo().id()) && + (primary->hbinfo().opTime >= hopeful->hbinfo().opTime)) { // other members might be aware of more up-to-date nodes errmsg = str::stream() << hopeful->fullName() << " is trying to elect itself but " << primary->fullName() << diff --git a/src/mongo/db/repl/heartbeat.cpp b/src/mongo/db/repl/heartbeat.cpp index f1bc18168f6..95bde6d185b 100644 --- a/src/mongo/db/repl/heartbeat.cpp +++ b/src/mongo/db/repl/heartbeat.cpp @@ -264,11 +264,13 @@ namespace mongo { down(mem, info.getStringField("errmsg")); } } - catch(DBException& e) { + catch (const DBException& e) { + log() << "replSet health poll task caught a DBException: " << e.what(); down(mem, e.what()); } - catch(...) { - down(mem, "replSet unexpected exception in ReplSetHealthPollTask"); + catch (const std::exception& e) { + log() << "replSet health poll task caught an exception: " << e.what(); + down(mem, e.what()); } m = mem; diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 68b7fc17529..841dcf7d898 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -177,6 +177,16 @@ namespace mongo { log() << "replSet " << o.toString() << rsLog; throw rsfatal(); } + else if( cmdname == "collMod" ) { + if ( o.nFields() == 2 && + o["usePowerOf2Sizes"].type() == Bool ) { + log() << "replSet not rolling back change of usePowerOf2Sizes: " << o; + } + else { + log() << "replSet error cannot rollback a collMod command: " << o; + throw rsfatal(); + } + } else { log() << "replSet error can't rollback this command yet: " << o.toString() << rsLog; log() << "replSet cmdname=" << cmdname << rsLog; diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 66c0ee7ee7f..4019a550b2b 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -81,6 +81,12 @@ namespace mongo { error() << "key for ttl index can only have 1 field" << endl; continue; } + if (!idx[secondsExpireField].isNumber()) { + error() << "ttl indexes require the " << secondsExpireField << " field to be " + << "numeric but received a type of: " + << typeName(idx[secondsExpireField].type()) << endl; + continue; + } BSONObj query; { diff --git a/src/mongo/dbtests/documenttests.cpp b/src/mongo/dbtests/documenttests.cpp index 57bf7f8b55d..56e51176f1d 100644 --- a/src/mongo/dbtests/documenttests.cpp +++ b/src/mongo/dbtests/documenttests.cpp @@ -170,6 +170,22 @@ namespace DocumentTests { ASSERT( md.peek().getValue( "c" ).missing() ); assertRoundTrips( md.peek() ); + // Set a nested field using [] + md["x"]["y"]["z"] = Value("nested"); + ASSERT_EQUALS(md.peek()["x"]["y"]["z"], Value("nested")); + + // Set a nested field using setNestedField + FieldPath xxyyzz = string("xx.yy.zz"); + md.setNestedField(xxyyzz, Value("nested")); + ASSERT_EQUALS(md.peek().getNestedField(xxyyzz), Value("nested") ); + + // Set a nested fields through an existing empty document + md["xxx"] = Value(Document()); + md["xxx"]["yyy"] = Value(Document()); + FieldPath xxxyyyzzz = string("xxx.yyy.zzz"); + md.setNestedField(xxxyyyzzz, Value("nested")); + ASSERT_EQUALS(md.peek().getNestedField(xxxyyyzzz), Value("nested") ); + // Make sure nothing moved ASSERT_EQUALS(apos, md.peek().positionOf("a")); ASSERT_EQUALS(bpos, md.peek().positionOf("c")); diff --git a/src/mongo/s/client_info.cpp b/src/mongo/s/client_info.cpp index b373e54c010..7dc83aff003 100644 --- a/src/mongo/s/client_info.cpp +++ b/src/mongo/s/client_info.cpp @@ -268,41 +268,39 @@ namespace mongo { } clearSinceLastGetError(); - LOG(4) << "checking " << writebacks.size() << " writebacks for" - << " gle (" << theShard << ")" << endl; - - if ( writebacks.size() ){ - vector<BSONObj> v = _handleWriteBacks( writebacks , fromWriteBackListener ); - if ( v.size() == 0 && fromWriteBackListener ) { - // ok + // We never need to handle writebacks if we're coming from the wbl itself + if ( writebacks.size() && !fromWriteBackListener ){ + + LOG(4) << "checking " << writebacks.size() << " writebacks for" + << " gle (" << theShard << ")" << endl; + + vector<BSONObj> v = _handleWriteBacks( writebacks , false ); + + // this will usually be 1 + // it can be greater than 1 if a write to a different shard + // than the last write op had a writeback + // all we're going to report is the first + // since that's the current write + // but we block for all + verify( v.size() >= 1 ); + + if ( res["writebackSince"].numberInt() > 0 ) { + // got writeback from older op + // ignore the result from it, just needed to wait + result.appendElements( res ); + } + else if ( writebacks[0].fromLastOperation ) { + result.appendElements( v[0] ); + result.appendElementsUnique( res ); + result.append( "writebackGLE" , v[0] ); + result.append( "initialGLEHost" , theShard ); + result.append( "initialGLE", res ); } else { - // this will usually be 1 - // it can be greater than 1 if a write to a different shard - // than the last write op had a writeback - // all we're going to report is the first - // since that's the current write - // but we block for all - verify( v.size() >= 1 ); - - if ( res["writebackSince"].numberInt() > 0 ) { - // got writeback from older op - // ignore the result from it, just needed to wait - result.appendElements( res ); - } - else if ( writebacks[0].fromLastOperation ) { - result.appendElements( v[0] ); - result.appendElementsUnique( res ); - result.append( "writebackGLE" , v[0] ); - result.append( "initialGLEHost" , theShard ); - result.append( "initialGLE", res ); - } - else { - // there was a writeback - // but its from an old operations - // so all that's important is that we block, not that we return stats - result.appendElements( res ); - } + // there was a writeback + // but its from an old operations + // so all that's important is that we block, not that we return stats + result.appendElements( res ); } } else { @@ -406,6 +404,10 @@ namespace mongo { LOG(4) << "checking " << writebacks.size() << " writebacks for" << " gle (" << shards->size() << " shards)" << endl; + // Multi-shard results from the writeback listener implicitly means that: + // A) no versioning was used (multi-update/delete) + // B) internal GLE was used (bulk insert) + if ( errors.size() == 0 ) { result.appendNull( "err" ); _handleWriteBacks( writebacks , fromWriteBackListener ); diff --git a/src/mongo/s/commands_public.cpp b/src/mongo/s/commands_public.cpp index 1c2ff52364a..f69ef9fe1bb 100644 --- a/src/mongo/s/commands_public.cpp +++ b/src/mongo/s/commands_public.cpp @@ -1771,6 +1771,8 @@ namespace mongo { int options, string &errmsg, BSONObjBuilder &result, bool fromRepl); + virtual bool passOptions() const { return true; } + private: }; @@ -1815,7 +1817,7 @@ namespace mongo { */ DBConfigPtr conf(grid.getDBConfig(dbName , false)); if (!conf || !conf->isShardingEnabled() || !conf->isSharded(fullns)) - return passthrough(conf, cmdObj, result); + return passthrough(conf, cmdObj, options, result); /* split the pipeline into pieces for mongods and this mongos */ intrusive_ptr<Pipeline> pShardPipeline( diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp index df620865603..28b281f0cbd 100644 --- a/src/mongo/s/shard.cpp +++ b/src/mongo/s/shard.cpp @@ -30,6 +30,7 @@ #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/jsobj.h" +#include "mongo/db/server_parameters.h" #include "mongo/s/client_info.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -39,6 +40,8 @@ namespace mongo { + MONGO_EXPORT_SERVER_PARAMETER(authOnPrimaryOnly, bool, true); + class StaticShardInfo { public: StaticShardInfo() : _mutex("StaticShardInfo"), _rsMutex("RSNameMap") { } @@ -411,14 +414,26 @@ namespace mongo { void ShardingConnectionHook::onCreate( DBClientBase * conn ) { if( !noauth ) { + bool result; string err; LOG(2) << "calling onCreate auth for " << conn->toString() << endl; - bool result = conn->auth( "local", - internalSecurity.user, - internalSecurity.pwd, - err, - false ); + if ( conn->type() == ConnectionString::SET && !authOnPrimaryOnly ) { + DBClientReplicaSet* setConn = dynamic_cast<DBClientReplicaSet*>(conn); + verify(setConn); + result = setConn->authAny( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } + else { + result = conn->auth( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } uassert( 15847, str::stream() << "can't authenticate to server " << conn->getServerAddress() << causedBy( err ), result ); diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h index e917baa0986..41f819d3ae9 100644 --- a/src/mongo/s/shard.h +++ b/src/mongo/s/shard.h @@ -290,8 +290,12 @@ namespace mongo { */ bool runCommand( const string& db , const BSONObj& cmd , BSONObj& res ); + // Whether or not we release connections from the thread-local cache after a read static bool releaseConnectionsAfterResponse; + // Controls whether we throw on initially failing to set a version + static bool ignoreInitialVersionFailure; + /** checks all of my thread local connections for the version of this ns */ static void checkMyConnectionVersions( const string & ns ); @@ -307,6 +311,11 @@ namespace mongo { */ static void clearPool(); + /** + * Forgets a namespace to prevent future versioning. + */ + static void forgetNS( const string& ns ); + private: void _init(); void _finishInit(); diff --git a/src/mongo/s/shardconnection.cpp b/src/mongo/s/shardconnection.cpp index 61b47904147..66a105f05e8 100644 --- a/src/mongo/s/shardconnection.cpp +++ b/src/mongo/s/shardconnection.cpp @@ -22,6 +22,7 @@ #include "mongo/db/client.h" #include "mongo/db/commands.h" +#include "mongo/db/lasterror.h" #include "mongo/db/server_parameters.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -33,6 +34,14 @@ namespace mongo { + bool ShardConnection::ignoreInitialVersionFailure( false ); + ExportedServerParameter<bool> + _ignoreInitialVersionFailure( ServerParameterSet::getGlobal(), + "ignoreInitialVersionFailure", + &ShardConnection::ignoreInitialVersionFailure, + true, + true ); + DBConnectionPool shardConnectionPool; class ClientConnections; @@ -235,6 +244,12 @@ namespace mongo { vector<Shard> all; Shard::getAllShards( all ); + scoped_ptr<LastError::Disabled> ignoreForGLE; + if ( ShardConnection::ignoreInitialVersionFailure ) { + // Don't report exceptions here as errors in GetLastError if ignoring failures + ignoreForGLE.reset( new LastError::Disabled( lastError.get( false ) ) ); + } + // Now only check top-level shard connections for ( unsigned i=0; i<all.size(); i++ ) { @@ -250,11 +265,18 @@ namespace mongo { versionManager.checkShardVersionCB( s->avail, ns, false, 1 ); } - catch ( const std::exception& e ) { + catch ( const DBException& ex ) { - warning() << "problem while initially checking shard versions on" - << " " << shard.getName() << causedBy(e) << endl; - throw; + warning() << "problem while initially checking shard versions on " + << shard.getName() << causedBy( ex ) << endl; + + if ( !ShardConnection::ignoreInitialVersionFailure ) { + throw; + } + else { + // We swallow the error here, checking shard version here is a heuristic to + // prevent later stale config exceptions, not required for correctness. + } } } } @@ -322,6 +344,11 @@ namespace mongo { _hosts.clear(); } + void forgetNS( const string& ns ) { + scoped_spinlock lock( _lock ); + _seenNS.erase( ns ); + } + // ----- static thread_specific_ptr<ClientConnections> _perThread; @@ -487,4 +514,8 @@ namespace mongo { shardConnectionPool.clear(); ClientConnections::threadInstance()->clearPool(); } + + void ShardConnection::forgetNS( const string& ns ) { + ClientConnections::threadInstance()->forgetNS( ns ); + } } diff --git a/src/mongo/s/strategy_shard.cpp b/src/mongo/s/strategy_shard.cpp index 0d4adcef155..560a4ac72d4 100644 --- a/src/mongo/s/strategy_shard.cpp +++ b/src/mongo/s/strategy_shard.cpp @@ -192,15 +192,10 @@ namespace mongo { string host = cursorCache.getRef( id ); if( host.size() == 0 ){ - - // - // Match legacy behavior here by throwing an exception when we can't find - // the cursor, but make the exception more informative - // - - uasserted( 16336, - str::stream() << "could not find cursor in cache for id " << id - << " over collection " << ns ); + LOG(3) << "could not find cursor in cache for id " << id + << " over collection " << ns << endl; + replyToQuery( ResultFlag_CursorNotFound , r.p() , r.m() , 0 , 0 , 0 ); + return; } // we used ScopedDbConnection because we don't get about config versions @@ -227,11 +222,9 @@ namespace mongo { int ntoreturn = r.d().pullInt(); long long id = r.d().pullInt64(); - LOG(6) << "want cursor : " << id << endl; - ShardedClientCursorPtr cursor = cursorCache.get( id ); if ( ! cursor ) { - LOG(6) << "\t invalid cursor :(" << endl; + LOG(3) << "Invalid cursor:" << id << endl; replyToQuery( ResultFlag_CursorNotFound , r.p() , r.m() , 0 , 0 , 0 ); return; } diff --git a/src/mongo/s/type_shard.cpp b/src/mongo/s/type_shard.cpp index 9b6fd2fdb78..a8f4236aa5d 100644 --- a/src/mongo/s/type_shard.cpp +++ b/src/mongo/s/type_shard.cpp @@ -87,7 +87,7 @@ namespace mongo { if (fieldState == FieldParser::FIELD_INVALID) return false; _isDrainingSet = fieldState == FieldParser::FIELD_SET; - fieldState = FieldParser::extract(source, maxSize, &_maxSize, errMsg); + fieldState = FieldParser::extractNumber(source, maxSize, &_maxSize, errMsg); if (fieldState == FieldParser::FIELD_INVALID) return false; _isMaxSizeSet = fieldState == FieldParser::FIELD_SET; diff --git a/src/mongo/s/type_shard_test.cpp b/src/mongo/s/type_shard_test.cpp index dda55a5daaf..c75db61ec26 100644 --- a/src/mongo/s/type_shard_test.cpp +++ b/src/mongo/s/type_shard_test.cpp @@ -74,6 +74,17 @@ namespace { ASSERT_TRUE(shard.isValid(NULL)); } + TEST(Validity, MaxSizeAsFloat) { + ShardType shard; + BSONObj obj = BSON(ShardType::name("shard0000") << + ShardType::host("localhost:27017") << + ShardType::maxSize() << 100.0); + string errMsg; + ASSERT(shard.parseBSON(obj, &errMsg)); + ASSERT_EQUALS(errMsg, ""); + ASSERT_TRUE(shard.isValid(NULL)); + } + TEST(Validity, BadType) { ShardType shard; BSONObj obj = BSON(ShardType::name() << 0); diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp index 1496ebc1407..3ba6254813b 100644 --- a/src/mongo/s/version_manager.cpp +++ b/src/mongo/s/version_manager.cpp @@ -103,23 +103,46 @@ namespace mongo { WriteBackListener::init( *conn_in ); - DBClientBase* conn = getVersionable( conn_in ); - verify( conn ); // errors thrown above + bool ok; + DBClientBase* conn = NULL; + try { + // May throw if replica set primary is down + conn = getVersionable( conn_in ); + dassert( conn ); // errors thrown above + + BSONObjBuilder cmdBuilder; + + cmdBuilder.append( "setShardVersion" , "" ); + cmdBuilder.appendBool( "init", true ); + cmdBuilder.append( "configdb" , configServer.modelServer() ); + cmdBuilder.appendOID( "serverID" , &serverID ); + cmdBuilder.appendBool( "authoritative" , true ); - BSONObjBuilder cmdBuilder; + BSONObj cmd = cmdBuilder.obj(); - cmdBuilder.append( "setShardVersion" , "" ); - cmdBuilder.appendBool( "init", true ); - cmdBuilder.append( "configdb" , configServer.modelServer() ); - cmdBuilder.appendOID( "serverID" , &serverID ); - cmdBuilder.appendBool( "authoritative" , true ); + LOG(1) << "initializing shard connection to " << conn->toString() << endl; + LOG(2) << "initial sharding settings : " << cmd << endl; + + ok = conn->runCommand("admin", cmd, result, 0); + } + catch( const DBException& ex ) { - BSONObj cmd = cmdBuilder.obj(); + bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure + && conn_in->type() == ConnectionString::SET; + if ( !ignoreFailure ) + throw; - LOG(1) << "initializing shard connection to " << conn->toString() << endl; - LOG(2) << "initial sharding settings : " << cmd << endl; + // Using initShardVersion is not strictly required when talking to replica sets - it is + // preferred to do so because it registers mongos early with the mongod. This info is + // also sent by checkShardVersion before a connection is used for a write or read. - bool ok = conn->runCommand("admin", cmd, result, 0); + OCCASIONALLY { + warning() << "failed to initialize new replica set connection version, " + << "will initialize on first use" << endl; + } + + return true; + } // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb diff --git a/src/mongo/s/writeback_listener.cpp b/src/mongo/s/writeback_listener.cpp index 255ab0160e5..0695c4cec56 100644 --- a/src/mongo/s/writeback_listener.cpp +++ b/src/mongo/s/writeback_listener.cpp @@ -345,6 +345,9 @@ namespace mongo { gle = b.obj(); } + dassert( !gle.isEmpty() ); + verify( !gle.isEmpty() ); + if ( gle["code"].numberInt() == 9517 ) { log() << "new version change detected, " diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index fc4d042c2af..e6bb8be819e 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -39,7 +39,7 @@ namespace mongo { Scope::Scope() : _localDBName(""), _loadedVersion(0), - _numTimeUsed(0), + _numTimesUsed(0), _lastRetIsNativeCode(false) { } @@ -259,90 +259,80 @@ namespace mongo { injectNative("benchFinish", BenchRunner::benchFinish); } - typedef map<string, list<Scope*> > PoolToScopes; - +namespace { class ScopeCache { public: - ScopeCache() : _mutex("ScopeCache") { - } + ScopeCache() : _mutex("ScopeCache") {} - ~ScopeCache() { - if (inShutdown()) - return; - clear(); - } - - void done(const string& pool, Scope* s) { + void release(const string& poolName, const boost::shared_ptr<Scope>& scope) { scoped_lock lk(_mutex); - list<Scope*>& l = _pools[pool]; - bool oom = s->hasOutOfMemoryException(); - // do not keep too many contexts, or use them for too long - if (l.size() > 10 || s->getTimeUsed() > 10 || oom || !s->getError().empty()) { - delete s; - } - else { - l.push_back(s); - s->reset(); + if (scope->hasOutOfMemoryException()) { + // make some room + log() << "Clearing all idle JS contexts due to out of memory" << endl; + _pools.clear(); + return; } - if (oom) { - // out of mem, make some room - log() << "Clearing all idle JS contexts due to out of memory" << endl; - clear(); + if (scope->getTimesUsed() > kMaxScopeReuse) + return; // used too many times to save + + if (!scope->getError().empty()) + return; // not saving errored scopes + + if (_pools.size() >= kMaxPoolSize) { + // prefer to keep recently-used scopes + _pools.pop_back(); } + + ScopeAndPool toStore = {scope, poolName}; + _pools.push_front(toStore); } - Scope* get(const string& pool) { + boost::shared_ptr<Scope> tryAcquire(const string& poolName) { scoped_lock lk(_mutex); - list<Scope*>& l = _pools[pool]; - if (l.size() == 0) - return 0; - - Scope* s = l.back(); - l.pop_back(); - s->reset(); - s->incTimeUsed(); - return s; - } - void clear() { - set<Scope*> seen; - for (PoolToScopes::iterator i = _pools.begin(); i != _pools.end(); ++i) { - for (list<Scope*>::iterator j = i->second.begin(); j != i->second.end(); ++j) { - Scope* s = *j; - fassert(16652, seen.insert(s).second); - delete s; + for (Pools::iterator it = _pools.begin(); it != _pools.end(); ++it) { + if (it->poolName == poolName) { + boost::shared_ptr<Scope> scope = it->scope; + _pools.erase(it); + scope->incTimesUsed(); + scope->reset(); + return scope; } } - _pools.clear(); + + return boost::shared_ptr<Scope>(); } private: - PoolToScopes _pools; + struct ScopeAndPool { + boost::shared_ptr<Scope> scope; + string poolName; + }; + + // Note: if these numbers change, reconsider choice of datastructure for _pools + static const unsigned kMaxPoolSize = 10; + static const int kMaxScopeReuse = 10; + + typedef deque<ScopeAndPool> Pools; // More-recently used Scopes are kept at the front. + Pools _pools; // protected by _mutex mongo::mutex _mutex; }; - thread_specific_ptr<ScopeCache> scopeCache; + ScopeCache scopeCache; +} // anonymous namespace class PooledScope : public Scope { public: - PooledScope(const std::string& pool, Scope* real) : _pool(pool), _real(real) { + PooledScope(const std::string& pool, const boost::shared_ptr<Scope>& real) + : _pool(pool) + , _real(real) { _real->loadStored(true); - }; + } + virtual ~PooledScope() { - ScopeCache* sc = scopeCache.get(); - if (sc) { - sc->done(_pool, _real); - _real = NULL; - } - else { - // this means that the Scope was killed from a different thread - // for example a cursor got timed out that has a $where clause - LOG(3) << "warning: scopeCache is empty!" << endl; - delete _real; - _real = 0; - } + scopeCache.release(_pool, _real); } // wrappers for the derived (_real) scope @@ -404,31 +394,24 @@ namespace mongo { private: string _pool; - Scope* _real; + boost::shared_ptr<Scope> _real; }; /** Get a scope from the pool of scopes matching the supplied pool name */ - auto_ptr<Scope> ScriptEngine::getPooledScope(const string& pool, const string& scopeType) { - if (!scopeCache.get()) - scopeCache.reset(new ScopeCache()); - - Scope* s = scopeCache->get(pool + scopeType); - if (!s) - s = newScope(); + auto_ptr<Scope> ScriptEngine::getPooledScope(const string& db, const string& scopeType) { + const string fullPoolName = db + scopeType; + boost::shared_ptr<Scope> s = scopeCache.tryAcquire(fullPoolName); + if (!s) { + s.reset(newScope()); + } auto_ptr<Scope> p; - p.reset(new PooledScope(pool + scopeType, s)); - p->setLocalDB(pool); + p.reset(new PooledScope(fullPoolName, s)); + p->setLocalDB(db); p->loadStored(true); return p; } - void ScriptEngine::threadDone() { - ScopeCache* sc = scopeCache.get(); - if (sc) - sc->clear(); - } - void (*ScriptEngine::_connectCallback)(DBClientWithCommands&) = 0; const char* (*ScriptEngine::_checkInterruptCallback)() = 0; unsigned (*ScriptEngine::_getCurrentOpIdCallback)() = 0; diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index 8233128e736..e0a3486a6ae 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -135,10 +135,10 @@ namespace mongo { static void validateObjectIdString(const string& str); /** increments the number of times a scope was used */ - void incTimeUsed() { ++_numTimeUsed; } + void incTimesUsed() { ++_numTimesUsed; } /** gets the number of times a scope was used */ - int getTimeUsed() { return _numTimeUsed; } + int getTimesUsed() { return _numTimesUsed; } /** return true if last invoke() return'd native code */ virtual bool isLastRetNativeCode() { return _lastRetIsNativeCode; } @@ -168,7 +168,7 @@ namespace mongo { set<string> _storedNames; static long long _lastVersion; FunctionCacheMap _cachedFunctions; - int _numTimeUsed; + int _numTimesUsed; bool _lastRetIsNativeCode; // v8 only: set to true if eval'd script returns a native func }; @@ -188,15 +188,12 @@ namespace mongo { static void setup(); /** gets a scope from the pool or a new one if pool is empty - * @param pool An identifier for the pool, usually the db name + * @param db The db name + * @param scopeType A unique id to limit scope sharing. + * This must include authenticated users. * @return the scope */ - auto_ptr<Scope> getPooledScope(const string& pool, const string& scopeType); - - /** - * call this method to release some JS resources when a thread is done - */ - void threadDone(); + auto_ptr<Scope> getPooledScope(const string& db, const string& scopeType); void setScopeInitCallback(void (*func)(Scope&)) { _scopeInitCallback = func; } static void setConnectCallback(void (*func)(DBClientWithCommands&)) { diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js index b1279028307..0e84c52d11c 100644 --- a/src/mongo/shell/assert.js +++ b/src/mongo/shell/assert.js @@ -225,3 +225,41 @@ assert.close = function(a, b, msg, places){ doassert(a + " is not equal to " + b + " within " + places + " places, diff: " + (a-b) + " : " + msg); }; + +assert.gleSuccess = function(db, msg) { + var gle = db.getLastErrorObj(); + if (gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null:" + tojson(gle) + " :" + msg); + } +} + +assert.gleError = function(db, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null: " + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorCode = function(db, code, msg) { + var gle = db.getLastErrorObj(); + if (gle.err && (gle.code == code)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null or missing code( " + code + "): " + + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorRegex = function(db, regex, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err || !regex.test(gle.err)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null or doesn't match regex (" + regex + "): " + + tojson(gle) + " :" + msg); + } +} diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 5934e862076..6a790ee2adb 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -75,7 +75,9 @@ void generateCompletions( const string& prefix , vector<string>& all ) { try { BSONObj args = BSON( "0" << prefix ); - shellMainScope->invokeSafe( "function callShellAutocomplete(x) {shellAutocomplete(x)}", &args, 0, 1000 ); + shellMainScope->invokeSafe("function callShellAutocomplete(x) {shellAutocomplete(x)}", + &args, + NULL); BSONObjBuilder b; shellMainScope->append( b , "" , "__autocomplete__" ); BSONObj res = b.obj(); diff --git a/src/mongo/tools/restore.cpp b/src/mongo/tools/restore.cpp index abd7f868afe..625902224ef 100644 --- a/src/mongo/tools/restore.cpp +++ b/src/mongo/tools/restore.cpp @@ -484,34 +484,29 @@ private: return nfields == obj2.nFields(); } - void createCollectionWithOptions(BSONObj cmdObj) { + void createCollectionWithOptions(BSONObj obj) { + BSONObjIterator i(obj); - // Create a new cmdObj to skip undefined fields and fix collection name + // Rebuild obj as a command object for the "create" command. + // - {create: <name>} comes first, where <name> is the new name for the collection + // - elements with type Undefined get skipped over BSONObjBuilder bo; - - // Add a "create" field if it doesn't exist - if (!cmdObj.hasField("create")) { - bo.append("create", _curcoll); - } - - BSONObjIterator i(cmdObj); - while ( i.more() ) { + bo.append("create", _curcoll); + while (i.more()) { BSONElement e = i.next(); - // Replace the "create" field with the name of the collection we are actually creating if (strcmp(e.fieldName(), "create") == 0) { - bo.append("create", _curcoll); + continue; } - else { - if (e.type() == Undefined) { - log() << _curns << ": skipping undefined field: " << e.fieldName() << endl; - } - else { - bo.append(e); - } + + if (e.type() == Undefined) { + log() << _curns << ": skipping undefined field: " << e.fieldName() << endl; + continue; } + + bo.append(e); } - cmdObj = bo.obj(); + obj = bo.obj(); BSONObj fields = BSON("options" << 1); scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", Query(BSON("name" << _curns)), 0, 0, &fields)); @@ -519,8 +514,8 @@ private: bool createColl = true; if (cursor->more()) { createColl = false; - BSONObj obj = cursor->next(); - if (!obj.hasField("options") || !optionsSame(cmdObj, obj["options"].Obj())) { + BSONObj nsObj = cursor->next(); + if (!nsObj.hasField("options") || !optionsSame(obj, nsObj["options"].Obj())) { log() << "WARNING: collection " << _curns << " exists with different options than are in the metadata.json file and not using --drop. Options in the metadata file will be ignored." << endl; } } @@ -530,10 +525,10 @@ private: } BSONObj info; - if (!conn().runCommand(_curdb, cmdObj, info)) { + if (!conn().runCommand(_curdb, obj, info)) { uasserted(15936, "Creating collection " + _curns + " failed. Errmsg: " + info["errmsg"].String()); } else { - log() << "\tCreated collection " << _curns << " with options: " << cmdObj.jsonString() << endl; + log() << "\tCreated collection " << _curns << " with options: " << obj.jsonString() << endl; } } diff --git a/src/mongo/util/net/listen.cpp b/src/mongo/util/net/listen.cpp index 4ba94d19c90..b92a3266a14 100644 --- a/src/mongo/util/net/listen.cpp +++ b/src/mongo/util/net/listen.cpp @@ -262,10 +262,14 @@ namespace mongo { int s = accept(*it, from.raw(), &from.addressSize); if ( s < 0 ) { int x = errno; // so no global issues - if ( x == ECONNABORTED || x == EBADF ) { - log() << "Listener on port " << _port << " aborted" << endl; + if (x == EBADF) { + log() << "Port " << _port << " is no longer valid" << endl; return; } + else if (x == ECONNABORTED) { + log() << "Connection on port " << _port << " aborted" << endl; + continue; + } if ( x == 0 && inShutdown() ) { return; // socket closed } @@ -461,9 +465,13 @@ namespace mongo { int s = accept(socks[eventIndex], from.raw(), &from.addressSize); if ( s < 0 ) { int x = errno; // so no global issues - if ( x == ECONNABORTED || x == EBADF ) { + if (x == EBADF) { + log() << "Port " << _port << " is no longer valid" << endl; + continue; + } + else if (x == ECONNABORTED) { log() << "Listener on port " << _port << " aborted" << endl; - return; + continue; } if ( x == 0 && inShutdown() ) { return; // socket closed diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 83ecd5923da..dd8b3a2fe6f 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -51,13 +51,9 @@ namespace mongo { SSLThreadInfo() { _id = ++_next; - CRYPTO_set_id_callback(_ssl_id_callback); - CRYPTO_set_locking_callback(_ssl_locking_callback); } - ~SSLThreadInfo() { - CRYPTO_set_id_callback(0); - } + ~SSLThreadInfo() {} unsigned long id() const { return _id; } @@ -150,14 +146,11 @@ namespace mongo { // Note: this is for blocking sockets only. SSL_CTX_set_mode(_context, SSL_MODE_AUTO_RETRY); - // Set context within which session can be reused - int status = SSL_CTX_set_session_id_context( - _context, - static_cast<unsigned char*>(static_cast<void*>(&_context)), - sizeof(_context)); - if (!status) { - uasserted(16768,"ssl initialization problem"); - } + // Disable session caching (see SERVER-10261) + SSL_CTX_set_session_cache_mode(_context, SSL_SESS_CACHE_OFF); + + CRYPTO_set_id_callback(_ssl_id_callback); + CRYPTO_set_locking_callback(_ssl_locking_callback); SSLThreadInfo::init(); SSLThreadInfo::get(); diff --git a/src/mongo/util/progress_meter.cpp b/src/mongo/util/progress_meter.cpp index 57ea9f728ec..d0f99ddff94 100644 --- a/src/mongo/util/progress_meter.cpp +++ b/src/mongo/util/progress_meter.cpp @@ -53,9 +53,13 @@ namespace mongo { if ( _total > 0 ) { int per = (int)( ( (double)_done * 100.0 ) / (double)_total ); - Nullstream& out = log() << "\t\t" << _name << ": " << _done - << '/' << _total << '\t' << per << '%'; + Nullstream& out = log(); + out << "\t\t" << _name << ": " << _done; + if (_showTotal) { + out << '/' << _total << '\t' << per << '%'; + } + if ( ! _units.empty() ) { out << "\t(" << _units << ")"; } diff --git a/src/mongo/util/progress_meter.h b/src/mongo/util/progress_meter.h index 88dbe29a0f5..baeb4680391 100644 --- a/src/mongo/util/progress_meter.h +++ b/src/mongo/util/progress_meter.h @@ -30,12 +30,13 @@ namespace mongo { int checkInterval = 100, std::string units = "", std::string name = "Progress") - : _units(units) - , _name(name) { + : _showTotal(true), + _units(units), + _name(name) { reset( total , secondsBetween , checkInterval ); } - ProgressMeter() : _active(0), _units(""), _name("Progress") {} + ProgressMeter() : _active(0), _showTotal(true), _units(""), _name("Progress") {} // typically you do ProgressMeterHolder void reset( unsigned long long total , int secondsBetween = 3 , int checkInterval = 100 ); @@ -65,6 +66,10 @@ namespace mongo { unsigned long long total() const { return _total; } + void showTotal(bool doShow) { + _showTotal = doShow; + } + std::string toString() const; bool operator==( const ProgressMeter& other ) const { return this == &other; } @@ -74,6 +79,7 @@ namespace mongo { bool _active; unsigned long long _total; + bool _showTotal; int _secondsBetween; int _checkInterval; diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp index 25fe64b5811..a5622e4dbbb 100644 --- a/src/mongo/util/version.cpp +++ b/src/mongo/util/version.cpp @@ -47,7 +47,7 @@ namespace mongo { * 1.2.3-rc4-pre- * If you really need to do something else you'll need to fix _versionArray() */ - const char versionString[] = "2.4.6"; + const char versionString[] = "2.4.9"; // See unit test for example outputs BSONArray toVersionArray(const char* version){ diff --git a/src/third_party/v8/src/spaces.h b/src/third_party/v8/src/spaces.h index 6602c899dfb..d7a79c6f983 100644 --- a/src/third_party/v8/src/spaces.h +++ b/src/third_party/v8/src/spaces.h @@ -321,7 +321,8 @@ class MemoryChunk { Space* owner() const { if ((reinterpret_cast<intptr_t>(owner_) & kFailureTagMask) == kFailureTag) { - return reinterpret_cast<Space*>(owner_ - kFailureTag); + return reinterpret_cast<Space*>(reinterpret_cast<intptr_t>(owner_) - + kFailureTag); } else { return NULL; } |