diff options
Diffstat (limited to 'src')
69 files changed, 1663 insertions, 418 deletions
diff --git a/src/SConscript.client b/src/SConscript.client index e30c73fcbb4..ff959e95f27 100644 --- a/src/SConscript.client +++ b/src/SConscript.client @@ -2,6 +2,8 @@ # This SConscript describes build and install rules for the Mongo C++ driver and associated exmaple # programs. +import os + Import('env has_option installSetup use_system_version_of_library') Import('nix linux darwin windows') @@ -52,6 +54,8 @@ clientSourceBasic = [ 'mongo/db/namespace.cpp', 'mongo/db/dbmessage.cpp', 'mongo/pch.cpp', + 'mongo/platform/backtrace.cpp', + 'mongo/platform/posix_fadvise.cpp', 'mongo/platform/random.cpp', 'mongo/util/assert_util.cpp', 'mongo/util/background.cpp', @@ -82,7 +86,6 @@ clientSourceBasic = [ 'mongo/util/net/ssl_manager.cpp', 'mongo/util/password.cpp', 'mongo/util/processinfo.cpp', - env.File('mongo/util/processinfo_${PYSYSPLATFORM}.cpp'), 'mongo/util/ramlog.cpp', 'mongo/util/signal_handlers.cpp', 'mongo/util/stringutils.cpp', @@ -98,14 +101,28 @@ clientSourceBasic = [ clientSourceSasl = ['mongo/client/sasl_client_authenticate_impl.cpp', 'mongo/client/sasl_client_session.cpp'] -clientSourceAll = clientSourceBasic + clientSourceSasl +clientSourceProcessInfo = [ + 'mongo/util/processinfo_darwin.cpp', + 'mongo/util/processinfo_freebsd.cpp', + 'mongo/util/processinfo_linux2.cpp', + 'mongo/util/processinfo_none.cpp', + 'mongo/util/processinfo_sunos5.cpp', + 'mongo/util/processinfo_win32.cpp' +] + +clientSourceAll = clientSourceBasic + clientSourceSasl + clientSourceProcessInfo usingSasl = env['MONGO_BUILD_SASL_CLIENT'] +clientSource = list(clientSourceBasic) if usingSasl: - clientSource = clientSourceAll -else: - clientSource = clientSourceBasic + clientSource += clientSourceSasl + +processInfoPlatformFile = env.File( "mongo/util/processinfo_${PYSYSPLATFORM}.cpp" ) +# NOTE: See comment about similar code in src/mongo/SConscript +if not os.path.exists( str( processInfoPlatformFile ) ): + processInfoPlatformFile = env.File( "mongo/util/processinfo_none.cpp" ) +clientSource += [processInfoPlatformFile] exampleSourceMap = [ ('authTest', 'mongo/client/examples/authTest.cpp'), diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 27067e32e9d..816a6bfacf8 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -479,6 +479,7 @@ mongosLibraryFiles = [ "s/commands_public.cpp", "s/request.cpp", "s/client_info.cpp", + "s/config_server_checker_service.cpp", "s/cursors.cpp", "s/s_only.cpp", "s/balance.cpp", diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 2bd395d0232..21a5b0810f4 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -31,10 +31,80 @@ #include "mongo/db/jsobj.h" #include "mongo/db/json.h" #include "mongo/util/background.h" +#include "mongo/util/concurrency/mutex.h" // for StaticObserver #include "mongo/util/scopeguard.h" #include "mongo/util/timer.h" namespace mongo { + + /* Replica Set statics: + * If a program (such as one built with the C++ driver) exits (by either calling exit() + * or by returning from main()), static objects will be destroyed in the reverse order + * of their creation (within each translation unit (source code file)). This makes it + * vital that the order be explicitly controlled within the source file so that destroyed + * objects never reference objects that have been destroyed earlier. + * + * The order chosen below is intended to allow safe destruction in reverse order from + * construction order: + * _setsLock -- mutex protecting _seedServers and _sets, destroyed last + * _seedServers -- list (map) of servers + * _sets -- list (map) of ReplicaSetMonitors + * replicaSetMonitorWatcher -- background job to check Replica Set members + * staticObserver -- sentinel to detect process termination + * + * Related to: + * SERVER-8891 -- Simple client fail with segmentation fault in mongoclient library + */ + mongo::mutex ReplicaSetMonitor::_setsLock( "ReplicaSetMonitor" ); + map<string, vector<HostAndPort> > ReplicaSetMonitor::_seedServers; + map<string, ReplicaSetMonitorPtr> ReplicaSetMonitor::_sets; + + // global background job responsible for checking every X amount of time + class ReplicaSetMonitorWatcher : public BackgroundJob { + public: + ReplicaSetMonitorWatcher() : _safego("ReplicaSetMonitorWatcher::_safego") , _started(false) {} + + virtual string name() const { return "ReplicaSetMonitorWatcher"; } + + void safeGo() { + // check outside of lock for speed + if ( _started ) + return; + + scoped_lock lk( _safego ); + if ( _started ) + return; + _started = true; + + go(); + } + + protected: + void run() { + log() << "starting" << endl; + sleepsecs( 10 ); + while ( !inShutdown() && !StaticObserver::_destroyingStatics ) { + try { + ReplicaSetMonitor::checkAll( true ); + } + catch ( std::exception& e ) { + error() << "check failed: " << e.what() << endl; + } + catch ( ... ) { + error() << "unknown error" << endl; + } + sleepsecs( 10 ); + } + } + + mongo::mutex _safego; + bool _started; + + } replicaSetMonitorWatcher; + + static StaticObserver staticObserver; + + /* * Set of commands that can be used with $readPreference */ @@ -154,6 +224,10 @@ namespace mongo { } if (secOnly && !node.okForSecondaryQueries()) { + LOG(3) << "dbclient_rs not selecting " << node + << ", not ok for secondary queries (" + << ( !node.secondary ? "not secondary" : "hidden" ) << ")" + << endl; continue; } @@ -164,8 +238,8 @@ namespace mongo { if (node.isLocalSecondary(localThresholdMillis)) { // found a local node. return early. - LOG(2) << "dbclient_rs _selectNode found local secondary for queries: " - << nextNodeIndex << ", ping time: " << node.pingTimeMillis << endl; + LOG(2) << "dbclient_rs selecting local secondary " << fallbackHost + << ", ping time: " << node.pingTimeMillis << endl; *lastHost = fallbackHost; return fallbackHost; } @@ -176,6 +250,14 @@ namespace mongo { *lastHost = fallbackHost; } + if ( fallbackHost.empty() ) { + LOG(3) << "dbclient_rs no node selected for tag " << readPreferenceTag << endl; + } + else { + LOG(3) << "dbclient_rs node " << fallbackHost << " selected for tag " + << readPreferenceTag << endl; + } + return fallbackHost; } @@ -283,47 +365,6 @@ namespace mongo { // ----- ReplicaSetMonitor --------- // -------------------------------- - // global background job responsible for checking every X amount of time - class ReplicaSetMonitorWatcher : public BackgroundJob { - public: - ReplicaSetMonitorWatcher() : _safego("ReplicaSetMonitorWatcher::_safego") , _started(false) {} - - virtual string name() const { return "ReplicaSetMonitorWatcher"; } - - void safeGo() { - // check outside of lock for speed - if ( _started ) - return; - - scoped_lock lk( _safego ); - if ( _started ) - return; - _started = true; - - go(); - } - protected: - void run() { - log() << "starting" << endl; - while ( ! inShutdown() ) { - sleepsecs( 10 ); - try { - ReplicaSetMonitor::checkAll( true ); - } - catch ( std::exception& e ) { - error() << "check failed: " << e.what() << endl; - } - catch ( ... ) { - error() << "unkown error" << endl; - } - } - } - - mongo::mutex _safego; - bool _started; - - } replicaSetMonitorWatcher; - string seedString( const vector<HostAndPort>& servers ){ string seedStr; for ( unsigned i = 0; i < servers.size(); i++ ){ @@ -1164,6 +1205,10 @@ namespace mongo { } if (candidate.empty()) { + + LOG( 3 ) << "dbclient_rs no compatible nodes found, refreshing view of replica set " + << _name << endl; + // mimic checkMaster behavior, which refreshes the local view of the replica set _check(false); @@ -1420,9 +1465,6 @@ namespace mongo { return builder.obj(); } - mongo::mutex ReplicaSetMonitor::_setsLock( "ReplicaSetMonitor" ); - map<string,ReplicaSetMonitorPtr> ReplicaSetMonitor::_sets; - map<string,vector<HostAndPort> > ReplicaSetMonitor::_seedServers; ReplicaSetMonitor::ConfigChangeHook ReplicaSetMonitor::_hook; int ReplicaSetMonitor::_maxFailedChecks = 30; // At 1 check every 10 seconds, 30 checks takes 5 minutes @@ -1625,9 +1667,21 @@ namespace mongo { const BSONObj *fieldsToReturn, int queryOptions, int batchSize) { - if (_isQueryOkToSecondary(ns, queryOptions, query.obj)) { + + if ( _isQueryOkToSecondary( ns, queryOptions, query.obj ) ) { + shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + LOG( 3 ) << "dbclient_rs query using secondary or tagged node selection in " + << _getMonitor()->getName() << ", read pref is " + << readPref->toBSON() << " (primary : " + << ( _master.get() != NULL ? + _master->getServerAddress() : "[not cached]" ) + << ", lastTagged : " + << ( _lastSlaveOkConn.get() != NULL ? + _lastSlaveOkConn->getServerAddress() : "[not cached]" ) + << ")" << endl; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { try { DBClientConnection* conn = selectNodeUsingTags(readPref); @@ -1643,16 +1697,19 @@ namespace mongo { return checkSlaveQueryResult(cursor); } catch (const DBException &dbExcep) { - LOG(1) << "can't query replica set slave " << _lastSlaveOkHost + LOG(1) << "can't query replica set node " << _lastSlaveOkHost << ": " << causedBy(dbExcep) << endl; invalidateLastSlaveOkCache(); } } - uasserted(16370, str::stream() << "Failed to do query, no good nodes in " - << _getMonitor()->getName()); + uasserted( 16370, + str::stream() << "Failed to do query, no good nodes in " + << _getMonitor()->getName() ); } + LOG( 3 ) << "dbclient_rs query to primary node in " << _getMonitor()->getName() << endl; + return checkMaster()->query(ns, query, nToReturn, nToSkip, fieldsToReturn, queryOptions, batchSize); } @@ -1662,8 +1719,19 @@ namespace mongo { const BSONObj *fieldsToReturn, int queryOptions) { if (_isQueryOkToSecondary(ns, queryOptions, query.obj)) { + shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + LOG( 3 ) << "dbclient_rs findOne using secondary or tagged node selection in " + << _getMonitor()->getName() << ", read pref is " + << readPref->toBSON() << " (primary : " + << ( _master.get() != NULL ? + _master->getServerAddress() : "[not cached]" ) + << ", lastTagged : " + << ( _lastSlaveOkConn.get() != NULL ? + _lastSlaveOkConn->getServerAddress() : "[not cached]" ) + << ")" << endl; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { try { DBClientConnection* conn = selectNodeUsingTags(readPref); @@ -1674,9 +1742,9 @@ namespace mongo { return conn->findOne(ns,query,fieldsToReturn,queryOptions); } - catch (const DBException &dbExcep) { - LOG(1) << "can't findone replica set slave " << _lastSlaveOkHost - << ": " << causedBy(dbExcep) << endl; + catch ( const DBException &dbExcep ) { + LOG(1) << "can't findone replica set node " << _lastSlaveOkHost << ": " + << causedBy( dbExcep ) << endl; invalidateLastSlaveOkCache(); } } @@ -1685,6 +1753,9 @@ namespace mongo { << _getMonitor()->getName()); } + LOG( 3 ) << "dbclient_rs findOne to primary node in " << _getMonitor()->getName() + << endl; + return checkMaster()->findOne(ns,query,fieldsToReturn,queryOptions); } @@ -1737,6 +1808,10 @@ namespace mongo { DBClientConnection* DBClientReplicaSet::selectNodeUsingTags( shared_ptr<ReadPreferenceSetting> readPref) { if (checkLastHost(readPref.get())) { + + LOG( 3 ) << "dbclient_rs selecting compatible last used node " << _lastSlaveOkHost + << endl; + return _lastSlaveOkConn.get(); } @@ -1746,6 +1821,9 @@ namespace mongo { &isPrimarySelected); if ( _lastSlaveOkHost.empty() ){ + + LOG( 3 ) << "dbclient_rs no compatible node found" << endl; + return NULL; } @@ -1759,6 +1837,9 @@ namespace mongo { checkMaster(); _lastSlaveOkConn = _master; _lastSlaveOkHost = _masterHost; // implied, but still assign just to be safe + + LOG( 3 ) << "dbclient_rs selecting primary node " << _lastSlaveOkHost << endl; + return _master.get(); } @@ -1772,14 +1853,16 @@ namespace mongo { // Assert here instead of returning NULL since the contract of this method is such // that returning NULL means none of the nodes were good, which is not the case here. - uassert(16532, str::stream() << "Failed to connect to " - << _lastSlaveOkHost.toString(true), + uassert(16532, str::stream() << "Failed to connect to " << _lastSlaveOkHost.toString(), newConn != NULL); _lastSlaveOkConn.reset(newConn); _lastSlaveOkConn->setReplSetClientCallback(this); _auth(_lastSlaveOkConn.get()); + + LOG( 3 ) << "dbclient_rs selecting node " << _lastSlaveOkHost << endl; + return _lastSlaveOkConn.get(); } @@ -1798,8 +1881,19 @@ namespace mongo { const bool slaveOk = qm.queryOptions & QueryOption_SlaveOk; if (_isQueryOkToSecondary(qm.ns, qm.queryOptions, qm.query)) { + shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + LOG( 3 ) << "dbclient_rs say using secondary or tagged node selection in " + << _getMonitor()->getName() << ", read pref is " + << readPref->toBSON() << " (primary : " + << ( _master.get() != NULL ? + _master->getServerAddress() : "[not cached]" ) + << ", lastTagged : " + << ( _lastSlaveOkConn.get() != NULL ? + _lastSlaveOkConn->getServerAddress() : "[not cached]" ) + << ")" << endl; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { _lazyState._retries = retry; try { @@ -1819,9 +1913,9 @@ namespace mongo { _lazyState._slaveOk = slaveOk; _lazyState._lastClient = conn; } - catch (const DBException& DBExcep) { - LOG(1) << "can't callLazy replica set slave " << _lastSlaveOkHost - << ": " << causedBy(DBExcep) << endl; + catch ( const DBException& DBExcep ) { + LOG(1) << "can't callLazy replica set node " << _lastSlaveOkHost << ": " + << causedBy( DBExcep ) << endl; invalidateLastSlaveOkCache(); continue; } @@ -1834,6 +1928,9 @@ namespace mongo { } } + LOG( 3 ) << "dbclient_rs say to primary node in " << _getMonitor()->getName() + << endl; + DBClientConnection* master = checkMaster(); if (actualServer) *actualServer = master->getServerAddress(); @@ -1940,8 +2037,19 @@ namespace mongo { ns = qm.ns; if (_isQueryOkToSecondary(ns, qm.queryOptions, qm.query)) { + shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + LOG( 3 ) << "dbclient_rs call using secondary or tagged node selection in " + << _getMonitor()->getName() << ", read pref is " + << readPref->toBSON() << " (primary : " + << ( _master.get() != NULL ? + _master->getServerAddress() : "[not cached]" ) + << ", lastTagged : " + << ( _lastSlaveOkConn.get() != NULL ? + _lastSlaveOkConn->getServerAddress() : "[not cached]" ) + << ")" << endl; + for (size_t retry = 0; retry < MAX_RETRY; retry++) { try { DBClientConnection* conn = selectNodeUsingTags(readPref); @@ -1956,12 +2064,11 @@ namespace mongo { return conn->call(toSend, response, assertOk); } - catch (const DBException& dbExcep) { - LOG(1) << "can't call replica set slave " << _lastSlaveOkHost - << ": " << causedBy(dbExcep) << endl; + catch ( const DBException& dbExcep ) { + LOG(1) << "can't call replica set node " << _lastSlaveOkHost << ": " + << causedBy( dbExcep ) << endl; - if (actualServer) - *actualServer = ""; + if ( actualServer ) *actualServer = ""; invalidateLastSlaveOkCache(); } @@ -1972,6 +2079,8 @@ namespace mongo { } } + LOG( 3 ) << "dbclient_rs call to primary node in " << _getMonitor()->getName() << endl; + DBClientConnection* m = checkMaster(); if ( actualServer ) *actualServer = m->getServerAddress(); @@ -2051,4 +2160,32 @@ namespace mongo { bool TagSet::equals(const TagSet& other) const { return _tags.equal(other._tags); } + + const BSONArray& TagSet::getTagBSON() const { + return _tags; + } + + string readPrefToString( ReadPreference pref ) { + switch ( pref ) { + case ReadPreference_PrimaryOnly: + return "primary only"; + case ReadPreference_PrimaryPreferred: + return "primary pref"; + case ReadPreference_SecondaryOnly: + return "secondary only"; + case ReadPreference_SecondaryPreferred: + return "secondary pref"; + case ReadPreference_Nearest: + return "nearest"; + default: + return "Unknown"; + } + } + + BSONObj ReadPreferenceSetting::toBSON() const { + BSONObjBuilder bob; + bob.append( "pref", readPrefToString( pref ) ); + bob.append( "tags", tags.getTagBSON() ); + return bob.obj(); + } } diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index ffdf0e0ecee..05f7faa9b2b 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -206,7 +206,7 @@ namespace mongo { /** * Removes the ReplicaSetMonitor for the given set name from _sets, which will delete it. * If clearSeedCache is true, then the cached seed string for this Replica Set will be removed - * from _setServers. + * from _seedServers. */ static void remove( const string& name, bool clearSeedCache = false ); @@ -333,7 +333,7 @@ namespace mongo { bool verbose, int nodesOffset ); /** - * Save the seed list for the current set into the _setServers map + * Save the seed list for the current set into the _seedServers map * Should only be called if you're already holding _setsLock and this * monitor's _lock. */ @@ -397,9 +397,12 @@ namespace mongo { // The number of consecutive times the set has been checked and every member in the set was down. int _failedChecks; - static mongo::mutex _setsLock; // protects _sets and _setServers - static map<string,ReplicaSetMonitorPtr> _sets; // set name to Monitor - static map<string,vector<HostAndPort> > _seedServers; // set name to seed list. Used to rebuild the monitor if it is cleaned up but then the set is accessed again. + static mongo::mutex _setsLock; // protects _seedServers and _sets + + // set name to seed list. + // Used to rebuild the monitor if it is cleaned up but then the set is accessed again. + static map<string, vector<HostAndPort> > _seedServers; + static map<string, ReplicaSetMonitorPtr> _sets; // set name to Monitor static ConfigChangeHook _hook; int _localThresholdMillis; // local ping latency threshold (protected by _lock) @@ -672,6 +675,8 @@ namespace mongo { */ bool equals(const TagSet& other) const; + const BSONArray& getTagBSON() const; + private: /** * This is purposely undefined as the semantics for assignment can be @@ -703,6 +708,8 @@ namespace mongo { return pref == other.pref && tags.equals(other.tags); } + BSONObj toBSON() const; + const ReadPreference pref; TagSet tags; }; diff --git a/src/mongo/db/clientcursor.cpp b/src/mongo/db/clientcursor.cpp index a0693dd8c7d..fa39170d73b 100644 --- a/src/mongo/db/clientcursor.cpp +++ b/src/mongo/db/clientcursor.cpp @@ -554,6 +554,16 @@ namespace mongo { return true; } + void yieldOrSleepFor1Microsecond() { +#ifdef _WIN32 + SwitchToThread(); +#elif defined(__linux__) + pthread_yield(); +#else + sleepmicros(1); +#endif + } + void ClientCursor::staticYield( int micros , const StringData& ns , Record * rec ) { bool haveReadLock = Lock::isReadLocked(); @@ -564,7 +574,7 @@ namespace mongo { // need to lock this else rec->touch won't be safe file could disappear lk.reset( new LockMongoFilesShared() ); } - + dbtempreleasecond unlock; if ( unlock.unlocked() ) { if ( haveReadLock ) { @@ -574,16 +584,26 @@ namespace mongo { #ifdef _WIN32 SwitchToThread(); #else - sleepmicros(1); + if ( micros == 0 ) { + yieldOrSleepFor1Microsecond(); + } + else { + sleepmicros(1); + } #endif } else { - if ( micros == -1 ) + if ( micros == -1 ) { micros = Client::recommendedYieldMicros(); - if ( micros > 0 ) + } + else if ( micros == 0 ) { + yieldOrSleepFor1Microsecond(); + } + else if ( micros > 0 ) { sleepmicros( micros ); + } } - + } else if ( Listener::getTimeTracker() == 0 ) { // we aren't running a server, so likely a repair, so don't complain diff --git a/src/mongo/db/clientcursor.h b/src/mongo/db/clientcursor.h index 7de161d6dd5..f2077282c66 100644 --- a/src/mongo/db/clientcursor.h +++ b/src/mongo/db/clientcursor.h @@ -178,7 +178,8 @@ namespace mongo { /** * @param microsToSleep -1 : ask client - * >=0 : sleep for that amount + * 0 : pthread_yield or equivilant + * >0 : sleep for that amount * @param recordToLoad after yielding lock, load this record with only mmutex * do a dbtemprelease * note: caller should check matcher.docMatcher().atomic() first and not yield if atomic - @@ -188,7 +189,7 @@ namespace mongo { * if false is returned, then this ClientCursor should be considered deleted - * in fact, the whole database could be gone. */ - bool yield( int microsToSleep = -1 , Record * recordToLoad = 0 ); + bool yield( int microsToSleep = -1, Record * recordToLoad = 0 ); enum RecordNeeds { DontNeed = -1 , MaybeCovered = 0 , WillNeed = 100 diff --git a/src/mongo/db/dbcommands_admin.cpp b/src/mongo/db/dbcommands_admin.cpp index 7ab769a9988..623a397201c 100644 --- a/src/mongo/db/dbcommands_admin.cpp +++ b/src/mongo/db/dbcommands_admin.cpp @@ -52,43 +52,6 @@ namespace mongo { - class CleanCmd : public Command { - public: - CleanCmd() : Command( "clean" ) {} - - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return WRITE; } - - virtual void help(stringstream& h) const { h << "internal"; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector<Privilege>* out) { - ActionSet actions; - actions.addAction(ActionType::clean); - out->push_back(Privilege(parseNs(dbname, cmdObj), actions)); - } - bool run(const string& dbname, BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool fromRepl ) { - string dropns = dbname + "." + cmdObj.firstElement().valuestrsafe(); - - if ( !cmdLine.quiet ) - tlog() << "CMD: clean " << dropns << endl; - - NamespaceDetails *d = nsdetails(dropns); - - if ( ! d ) { - errmsg = "ns not found"; - return 0; - } - - for ( int i = 0; i < Buckets; i++ ) - d->deletedList[i].Null(); - - result.append("ns", dropns.c_str()); - return 1; - } - - } cleanCmd; - namespace dur { boost::filesystem::path getJournalDir(); } diff --git a/src/mongo/db/dur.cpp b/src/mongo/db/dur.cpp index e49d8ef9da0..9c336751446 100644 --- a/src/mongo/db/dur.cpp +++ b/src/mongo/db/dur.cpp @@ -470,12 +470,14 @@ namespace mongo { fraction = 1; lastRemap = now; -#if defined(_WIN32) +#if defined(_WIN32) || defined(__sunos__) // Note that this negatively affects performance. - // We must grab the exclusive lock here because remapThePrivateView() on Windows needs - // to grab it as well, due to the lack of a non-atomic way to remap a memory mapped file. + // We must grab the exclusive lock here because remapPrivateView() on Windows and + // Solaris need to grab it as well, due to the lack of an atomic way to remap a + // memory mapped file. // See SERVER-5723 for performance improvement. - // See SERVER-5680 to see why this code is necessary. + // See SERVER-5680 to see why this code is necessary on Windows. + // See SERVER-8795 to see why this code is necessary on Solaris. LockMongoFilesExclusive lk; #else LockMongoFilesShared lk; diff --git a/src/mongo/db/extsort.cpp b/src/mongo/db/extsort.cpp index 195b01851a8..d955acb94cc 100644 --- a/src/mongo/db/extsort.cpp +++ b/src/mongo/db/extsort.cpp @@ -33,6 +33,7 @@ #include "mongo/db/kill_current_op.h" #include "mongo/db/namespace-inl.h" +#include "mongo/platform/posix_fadvise.h" #include "mongo/util/file.h" namespace mongo { diff --git a/src/mongo/db/fts/fts_matcher.cpp b/src/mongo/db/fts/fts_matcher.cpp index 313fdd5be9e..ee462bbb009 100644 --- a/src/mongo/db/fts/fts_matcher.cpp +++ b/src/mongo/db/fts/fts_matcher.cpp @@ -19,6 +19,7 @@ #include "mongo/pch.h" #include "mongo/db/fts/fts_matcher.h" +#include "mongo/platform/strcasestr.h" namespace mongo { @@ -226,20 +227,10 @@ namespace mongo { /* * Looks for phrase in a raw string * @param phrase, phrase to match - * @param raw, raw string to be parsed + * @param haystack, raw string to be parsed */ bool FTSMatcher::_phraseMatches( const string& phrase, const string& haystack ) const { -#ifdef _WIN32 - // windows doesn't have strcasestr - // for now, doing something very slow, bu correct - string p = phrase; - string h = haystack; - makeLower( &p ); - makeLower( &h ); - return strstr( h.c_str(), p.c_str() ) > 0; -#else return strcasestr( haystack.c_str(), phrase.c_str() ) > 0; -#endif } diff --git a/src/mongo/db/mongod.vcxproj b/src/mongo/db/mongod.vcxproj index 5b7dc27a53e..9b4811af659 100644 --- a/src/mongo/db/mongod.vcxproj +++ b/src/mongo/db/mongod.vcxproj @@ -1934,7 +1934,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\client\distlock_test.cpp" />
<ClCompile Include="..\client\model.cpp" />
<ClCompile Include="..\client\sasl_client_authenticate.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
@@ -3025,13 +3028,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
diff --git a/src/mongo/db/mongod.vcxproj.filters b/src/mongo/db/mongod.vcxproj.filters index 1b5e8788358..8e27c34a716 100644 --- a/src/mongo/db/mongod.vcxproj.filters +++ b/src/mongo/db/mongod.vcxproj.filters @@ -1814,6 +1814,15 @@ <ClCompile Include="index_set.cpp">
<Filter>db\Source Files\e to n</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\targetver.h" />
@@ -3828,6 +3837,15 @@ <ClInclude Include="index_set.h">
<Filter>db\Header Files\e to n</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="mongo.ico">
diff --git a/src/mongo/db/mongod_sm.vcxproj b/src/mongo/db/mongod_sm.vcxproj index df066a1baf8..b7add373e92 100644 --- a/src/mongo/db/mongod_sm.vcxproj +++ b/src/mongo/db/mongod_sm.vcxproj @@ -1588,7 +1588,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\client\distlock_test.cpp" />
<ClCompile Include="..\client\model.cpp" />
<ClCompile Include="..\client\sasl_client_authenticate.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
@@ -2377,13 +2380,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
diff --git a/src/mongo/db/mongod_sm.vcxproj.filters b/src/mongo/db/mongod_sm.vcxproj.filters index dbf6beb6b82..ee4a208c28b 100644 --- a/src/mongo/db/mongod_sm.vcxproj.filters +++ b/src/mongo/db/mongod_sm.vcxproj.filters @@ -1475,6 +1475,15 @@ <ClCompile Include="index_set.cpp">
<Filter>db\Source Files\e to n</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\targetver.h" />
@@ -3012,6 +3021,15 @@ <ClInclude Include="index_set.h">
<Filter>db\Header Files\e to n</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="mongo.ico">
diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp index 870601c9b4b..12f4742c256 100644 --- a/src/mongo/db/oplog.cpp +++ b/src/mongo/db/oplog.cpp @@ -576,7 +576,7 @@ namespace mongo { while( !finder->done() ) { if ( yieldCondition.intervalHasElapsed() ) { if ( finder->prepareToYield() ) { - ClientCursor::staticYield( -1, ns, 0 ); + ClientCursor::staticYield( 0, ns, 0 ); finder->recoverFromYield(); } } diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index 67c1eca8555..f2c22151636 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -63,7 +63,8 @@ namespace replset { BackgroundSyncInterface::~BackgroundSyncInterface() {} size_t getSize(const BSONObj& o) { - return o.objsize(); + // SERVER-9808 Avoid Fortify complaint about implicit signed->unsigned conversion + return static_cast<size_t>(o.objsize()); } BackgroundSync::BackgroundSync() : _buffer(bufferMaxSizeGauge, &getSize), @@ -240,11 +241,10 @@ namespace replset { try { _producerThread(); } - catch (DBException& e) { - sethbmsg(str::stream() << "db exception in producer: " << e.toString()); - sleepsecs(10); + catch (const DBException& e) { + sethbmsg(str::stream() << "sync source problem: " << e.toString()); } - catch (std::exception& e2) { + catch (const std::exception& e2) { sethbmsg(str::stream() << "exception in producer: " << e2.what()); sleepsecs(60); } @@ -288,7 +288,7 @@ namespace replset { // this oplog reader does not do a handshake because we don't want the server it's syncing // from to track how far it has synced OplogReader r(false /* doHandshake */); - + OpTime lastOpTimeFetched; // find a target to sync from the last op time written getOplogReader(r); @@ -302,10 +302,11 @@ namespace replset { // if there is no one to sync from return; } - - r.tailingQueryGTE(rsoplog, _lastOpTimeFetched); + lastOpTimeFetched = _lastOpTimeFetched; } + r.tailingQueryGTE(rsoplog, lastOpTimeFetched); + // if target cut connections between connecting and querying (for // example, because it stepped down) we might not have a cursor if (!r.haveCursor()) { @@ -320,72 +321,82 @@ namespace replset { } while (!inShutdown()) { - while (!inShutdown()) { - - if (!r.moreInCurrentBatch()) { - if (theReplSet->gotForceSync()) { - return; - } - if (isAssumingPrimary() || theReplSet->isPrimary()) { - return; - } - - // re-evaluate quality of sync target - if (shouldChangeSyncTarget()) { - return; - } - //record time for each getmore - { - TimerHolder batchTimer(&getmoreReplStats); - r.more(); - } - //increment - networkByteStats.increment(r.currentBatchMessageSize()); + if (!r.moreInCurrentBatch()) { + // Check some things periodically + // (whenever we run out of items in the + // current cursor batch) + if (theReplSet->gotForceSync()) { + return; + } + // If we are transitioning to primary state, we need to leave + // this loop in order to go into bgsync-pause mode. + if (isAssumingPrimary() || theReplSet->isPrimary()) { + return; } - if (!r.more()) - break; + // re-evaluate quality of sync target + if (shouldChangeSyncTarget()) { + return; + } - BSONObj o = r.nextSafe().getOwned(); - opsReadStats.increment(); { - boost::unique_lock<boost::mutex> lock(_mutex); - _appliedBuffer = false; + //record time for each getmore + TimerHolder batchTimer(&getmoreReplStats); + + // This calls receiveMore() on the oplogreader cursor. + // It can wait up to five seconds for more data. + r.more(); } + networkByteStats.increment(r.currentBatchMessageSize()); - OCCASIONALLY { - LOG(2) << "bgsync buffer has " << _buffer.size() << " bytes" << rsLog; - } - // the blocking queue will wait (forever) until there's room for us to push - _buffer.push(o); - bufferCountGauge.increment(); - bufferSizeGauge.increment(getSize(o)); + if (!r.moreInCurrentBatch()) { + // If there is still no data from upstream, check a few more things + // and then loop back for another pass at getting more data + { + boost::unique_lock<boost::mutex> lock(_mutex); + if (_pause || + !_currentSyncTarget || + !_currentSyncTarget->hbinfo().hbstate.readable()) { + return; + } + } - { - boost::unique_lock<boost::mutex> lock(_mutex); - _lastH = o["h"].numberLong(); - _lastOpTimeFetched = o["ts"]._opTime(); + r.tailCheck(); + if( !r.haveCursor() ) { + LOG(1) << "replSet end syncTail pass" << rsLog; + return; + } + + continue; } - } // end while + } + + // At this point, we are guaranteed to have at least one thing to read out + // of the oplogreader cursor. + BSONObj o = r.nextSafe().getOwned(); + opsReadStats.increment(); { boost::unique_lock<boost::mutex> lock(_mutex); - if (_pause || !_currentSyncTarget || !_currentSyncTarget->hbinfo().hbstate.readable()) { - return; - } + _appliedBuffer = false; } - - r.tailCheck(); - if( !r.haveCursor() ) { - LOG(1) << "replSet end syncTail pass" << rsLog; - return; + OCCASIONALLY { + LOG(2) << "bgsync buffer has " << _buffer.size() << " bytes" << rsLog; } + // the blocking queue will wait (forever) until there's room for us to push + _buffer.push(o); + bufferCountGauge.increment(); + bufferSizeGauge.increment(getSize(o)); - // looping back is ok because this is a tailable cursor + { + boost::unique_lock<boost::mutex> lock(_mutex); + _lastH = o["h"].numberLong(); + _lastOpTimeFetched = o["ts"]._opTime(); + } } } diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 8c8e89c20c6..68b7fc17529 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -68,7 +68,11 @@ namespace mongo { class rsfatal : public std::exception { public: - virtual const char* what() const throw() { return "replica set fatal exception"; } + rsfatal(std::string m = "replica set fatal exception") : msg(m) {} + virtual ~rsfatal() throw() {}; + virtual const char* what() const throw() { return msg.c_str(); } + private: + std::string msg; }; struct DocID { @@ -193,19 +197,13 @@ namespace mongo { int getRBID(DBClientConnection*); static void syncRollbackFindCommonPoint(DBClientConnection *them, HowToFixUp& h) { - static time_t last; - if( time(0)-last < 60 ) { - throw "findcommonpoint waiting a while before trying again"; - } - last = time(0); - verify( Lock::isLocked() ); Client::Context c(rsoplog); NamespaceDetails *nsd = nsdetails(rsoplog); verify(nsd); ReverseCappedCursor u(nsd); if( !u.ok() ) - throw "our oplog empty or unreadable"; + throw rsfatal("our oplog empty or unreadable"); const Query q = Query().sort(reverseNaturalObj); const bo fields = BSON( "ts" << 1 << "h" << 1 ); @@ -215,7 +213,7 @@ namespace mongo { h.rbid = getRBID(them); auto_ptr<DBClientCursor> t = them->query(rsoplog, q, 0, 0, &fields, 0, 0); - if( t.get() == 0 || !t->more() ) throw "remote oplog empty or unreadable"; + if( t.get() == 0 || !t->more() ) throw rsfatal("remote oplog empty or unreadable"); BSONObj ourObj = u.current(); OpTime ourTime = ourObj["ts"]._opTime(); @@ -230,7 +228,8 @@ namespace mongo { log() << "replSet info rollback diff in end of log times: " << diff << " seconds" << rsLog; if( diff > 1800 ) { log() << "replSet rollback too long a time period for a rollback." << rsLog; - throw "error not willing to roll back more than 30 minutes of data"; + throw rsfatal(str::stream() << "rollback error: not willing to roll back " + << "more than 30 minutes of data"); } } @@ -256,7 +255,7 @@ namespace mongo { log() << "replSet them: " << them->toString() << " scanned: " << scanned << rsLog; log() << "replSet theirTime: " << theirTime.toStringLong() << rsLog; log() << "replSet ourTime: " << ourTime.toStringLong() << rsLog; - throw "RS100 reached beginning of remote oplog [2]"; + throw rsfatal("RS100 reached beginning of remote oplog [2]"); } theirObj = t->nextSafe(); theirTime = theirObj["ts"]._opTime(); @@ -267,7 +266,7 @@ namespace mongo { log() << "replSet them: " << them->toString() << " scanned: " << scanned << rsLog; log() << "replSet theirTime: " << theirTime.toStringLong() << rsLog; log() << "replSet ourTime: " << ourTime.toStringLong() << rsLog; - throw "RS101 reached beginning of local oplog [1]"; + throw rsfatal("RS101 reached beginning of local oplog [1]"); } ourObj = u.current(); ourTime = ourObj["ts"]._opTime(); @@ -278,7 +277,7 @@ namespace mongo { log() << "replSet them: " << them->toString() << " scanned: " << scanned << rsLog; log() << "replSet theirTime: " << theirTime.toStringLong() << rsLog; log() << "replSet ourTime: " << ourTime.toStringLong() << rsLog; - throw "RS100 reached beginning of remote oplog [1]"; + throw rsfatal("RS100 reached beginning of remote oplog [1]"); } theirObj = t->nextSafe(); theirTime = theirObj["ts"]._opTime(); @@ -292,7 +291,7 @@ namespace mongo { log() << "replSet them: " << them->toString() << " scanned: " << scanned << rsLog; log() << "replSet theirTime: " << theirTime.toStringLong() << rsLog; log() << "replSet ourTime: " << ourTime.toStringLong() << rsLog; - throw "RS101 reached beginning of local oplog [2]"; + throw rsfatal("RS101 reached beginning of local oplog [2]"); } ourObj = u.current(); ourTime = ourObj["ts"]._opTime(); @@ -637,11 +636,8 @@ namespace mongo { try { syncRollbackFindCommonPoint(r.conn(), how); } - catch( const char *p ) { - sethbmsg(string("rollback 2 error ") + p); - return 10; - } - catch( rsfatal& ) { + catch( rsfatal& e ) { + sethbmsg(string(e.what())); _fatal(); return 2; } diff --git a/src/mongo/db/repl/rs_sync.cpp b/src/mongo/db/repl/rs_sync.cpp index f1c531b1c7d..e6e31943a9f 100644 --- a/src/mongo/db/repl/rs_sync.cpp +++ b/src/mongo/db/repl/rs_sync.cpp @@ -587,8 +587,9 @@ namespace replset { lock rsLock( this ); Lock::GlobalWrite writeLock; - // make sure we're not primary, secondary, or fatal already - if (box.getState().primary() || box.getState().secondary() || box.getState().fatal()) { + // make sure we're not primary, secondary, rollback, or fatal already + if (box.getState().primary() || box.getState().secondary() || + box.getState().fatal()) { return false; } diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 584bc20a3bc..66c0ee7ee7f 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -27,6 +27,7 @@ #include "mongo/db/instance.h" #include "mongo/db/ops/delete.h" #include "mongo/db/replutil.h" +#include "mongo/db/server_parameters.h" #include "mongo/util/background.h" namespace mongo { @@ -37,7 +38,7 @@ namespace mongo { ServerStatusMetricField<Counter64> ttlPassesDisplay("ttl.passes", &ttlPasses); ServerStatusMetricField<Counter64> ttlDeletedDocumentsDisplay("ttl.deletedDocuments", &ttlDeletedDocuments); - + MONGO_EXPORT_SERVER_PARAMETER( ttlMonitorEnabled, bool, true ); class TTLMonitor : public BackgroundJob { public: @@ -124,6 +125,11 @@ namespace mongo { sleepsecs( 60 ); LOG(3) << "TTLMonitor thread awake" << endl; + + if ( !ttlMonitorEnabled ) { + LOG(1) << "TTLMonitor is disabled" << endl; + continue; + } if ( lockedForWriting() ) { // note: this is not perfect as you can go into fsync+lock between diff --git a/src/mongo/dbtests/jsobjtests.cpp b/src/mongo/dbtests/jsobjtests.cpp index 80b88831c47..bb2c58207c3 100644 --- a/src/mongo/dbtests/jsobjtests.cpp +++ b/src/mongo/dbtests/jsobjtests.cpp @@ -1504,9 +1504,9 @@ namespace JsobjTests { public: void run() { Date_t before = jsTime(); - sleepmillis(1); - time_t now = time(NULL); - sleepmillis(1); + sleepmillis(2); + time_t now = jsTime().toTimeT(); + sleepmillis(2); Date_t after = jsTime(); BSONObjBuilder b; diff --git a/src/mongo/dbtests/sharding.cpp b/src/mongo/dbtests/sharding.cpp index f5e43be7687..e0f6218a0f4 100644 --- a/src/mongo/dbtests/sharding.cpp +++ b/src/mongo/dbtests/sharding.cpp @@ -98,6 +98,11 @@ namespace ShardingTests { _shard = Shard( "shard0000", "$hostFooBar:27017" ); // Need to run this to ensure the shard is in the global lookup table _shard.setAddress( _shard.getAddress() ); + + // Create an index so that diffing works correctly, otherwise no cursors from S&O + client().ensureIndex( "config.chunks", // br + BSON( "ns" << 1 << // br + "lastmod" << 1 ) ); } virtual ~ChunkManagerTest() { diff --git a/src/mongo/dbtests/test.vcxproj b/src/mongo/dbtests/test.vcxproj index 77b70f4ce25..827a155651b 100644 --- a/src/mongo/dbtests/test.vcxproj +++ b/src/mongo/dbtests/test.vcxproj @@ -711,13 +711,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
@@ -2578,7 +2581,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\db\scanandorder.cpp" />
<ClCompile Include="..\db\server_parameters.cpp" />
<ClCompile Include="..\db\stats\timer_stats.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
diff --git a/src/mongo/dbtests/test.vcxproj.filters b/src/mongo/dbtests/test.vcxproj.filters index 49ff9f467b2..d3216639cc7 100755 --- a/src/mongo/dbtests/test.vcxproj.filters +++ b/src/mongo/dbtests/test.vcxproj.filters @@ -2205,6 +2205,15 @@ <ClInclude Include="..\db\index_set.h">
<Filter>db\Header Files\e to n</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\client\connpool.cpp">
@@ -4201,6 +4210,15 @@ <ClCompile Include="..\db\hashindex.cpp">
<Filter>db\Source Files\e to n</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="btreetests.inl">
diff --git a/src/mongo/dbtests/test_sm.vcxproj b/src/mongo/dbtests/test_sm.vcxproj index 646d513dbbe..806c70053f1 100644 --- a/src/mongo/dbtests/test_sm.vcxproj +++ b/src/mongo/dbtests/test_sm.vcxproj @@ -767,13 +767,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
@@ -2869,7 +2872,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\db\scanandorder.cpp" />
<ClCompile Include="..\db\server_parameters.cpp" />
<ClCompile Include="..\db\stats\timer_stats.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
diff --git a/src/mongo/dbtests/test_sm.vcxproj.filters b/src/mongo/dbtests/test_sm.vcxproj.filters index 9855716ca4e..cfead0b3f30 100644 --- a/src/mongo/dbtests/test_sm.vcxproj.filters +++ b/src/mongo/dbtests/test_sm.vcxproj.filters @@ -1716,6 +1716,15 @@ <ClInclude Include="..\db\index_set.h">
<Filter>db\Header Files\e to n</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\client\connpool.cpp">
@@ -3373,6 +3382,15 @@ <ClCompile Include="..\db\hashindex.cpp">
<Filter>db\Source Files\e to n</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="btreetests.inl">
diff --git a/src/mongo/platform/SConscript b/src/mongo/platform/SConscript index c18fbae3ce8..99738b6693a 100644 --- a/src/mongo/platform/SConscript +++ b/src/mongo/platform/SConscript @@ -2,7 +2,12 @@ Import("env") -env.Library( "platform", "random.cpp" ) +env.Library('platform', [ + 'backtrace.cpp', + 'posix_fadvise.cpp', + 'random.cpp', + 'strcasestr.cpp', + ]) env.CppUnitTest('atomic_word_test', 'atomic_word_test.cpp') env.CppUnitTest('bits_test', 'bits_test.cpp') diff --git a/src/mongo/platform/backtrace.cpp b/src/mongo/platform/backtrace.cpp new file mode 100644 index 00000000000..89694425daf --- /dev/null +++ b/src/mongo/platform/backtrace.cpp @@ -0,0 +1,215 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined(_WIN32) +#if defined(__sunos__) || !defined(MONGO_HAVE_EXECINFO_BACKTRACE) + +#include "mongo/platform/backtrace.h" + +#include <boost/smart_ptr/scoped_array.hpp> +#include <cstdio> +#include <dlfcn.h> +#include <string> +#include <ucontext.h> +#include <vector> + +#include "mongo/base/init.h" +#include "mongo/base/status.h" + +using std::string; +using std::vector; + +namespace mongo { +namespace pal { + +namespace { + class WalkcontextCallback { + public: + WalkcontextCallback(uintptr_t* array, int size) + : _position(0), + _count(size), + _addresses(array) {} + + // This callback function is called from C code, and so must not throw exceptions + // + static int callbackFunction(uintptr_t address, + int signalNumber, + WalkcontextCallback* thisContext) { + if (thisContext->_position < thisContext->_count) { + thisContext->_addresses[thisContext->_position++] = address; + return 0; + } + return 1; + } + int getCount() const { return static_cast<int>(_position); } + private: + size_t _position; + size_t _count; + uintptr_t* _addresses; + }; + + // This function emulates a Solaris function that was added to Solaris 11 at the same time + // that the backtrace* functions were added. It is not important to match the function name + // or interface for this code, but this is a fine interface for our purposes and following + // an existing model seems potentially helpful ... hence the all-lowercase name with no + // underscores. The formatting of the output matches what Solaris 11 does; this is similar + // to Linux's display, but slightly different. + // + int addrtosymstr(void* address, char* outputBuffer, int outputBufferSize) { + Dl_info_t symbolInfo; + if (dladdr(address, &symbolInfo) == 0) { // no info: "[address]" + return snprintf(outputBuffer, outputBufferSize, "[0x%p]", address); + } + if (symbolInfo.dli_sname == NULL) { + return snprintf(outputBuffer, // no symbol: "filename'offset [address]" + outputBufferSize, + "%s'0x%p [0x%p]", + symbolInfo.dli_fname, + reinterpret_cast<char*>(reinterpret_cast<char*>(address) - + reinterpret_cast<char*>(symbolInfo.dli_fbase)), + address); + } + return snprintf(outputBuffer, // symbol: "filename'symbol+offset [address]" + outputBufferSize, + "%s'%s+0x%p [0x%p]", + symbolInfo.dli_fname, + symbolInfo.dli_sname, + reinterpret_cast<char*>(reinterpret_cast<char*>(address) - + reinterpret_cast<char*>(symbolInfo.dli_saddr)), + address); + } +} // namespace + + typedef int (*WalkcontextCallbackFunc)(uintptr_t address, int signalNumber, void* thisContext); + + int backtrace_emulation(void** array, int size) { + WalkcontextCallback walkcontextCallback(reinterpret_cast<uintptr_t*>(array), size); + ucontext_t context; + if (getcontext(&context) != 0) { + return 0; + } + int wcReturn = walkcontext( + &context, + reinterpret_cast<WalkcontextCallbackFunc>(WalkcontextCallback::callbackFunction), + static_cast<void*>(&walkcontextCallback)); + if (wcReturn == 0) { + return walkcontextCallback.getCount(); + } + return 0; + } + + // The API for backtrace_symbols() specifies that the caller must call free() on the char** + // returned by this function, and must *not* call free on the individual strings. + // In order to support this API, we allocate a single block of memory containing both the + // array of pointers to the strings and the strings themselves. Constructing this block + // requires two passes: one to collect the strings and calculate the required size of the + // combined single memory block; and a second pass to copy the strings into the block and + // set their pointers. + // The result is that we return a single memory block (allocated with malloc()) which begins + // with an array of pointers to strings (of count 'size'). This array is immediately + // followed by the strings themselves, each NUL terminated. + // + char** backtrace_symbols_emulation(void* const* array, int size) { + vector<string> stringVector; + vector<size_t> stringLengths; + size_t blockSize = size * sizeof(char*); + size_t blockPtr = blockSize; + const size_t kBufferSize = 8 * 1024; + boost::scoped_array<char> stringBuffer(new char[kBufferSize]); + for (int i = 0; i < size; ++i) { + size_t thisLength = 1 + addrtosymstr(array[i], stringBuffer.get(), kBufferSize); + stringVector.push_back(string(stringBuffer.get())); + stringLengths.push_back(thisLength); + blockSize += thisLength; + } + char** singleBlock = static_cast<char**>(malloc(blockSize)); + if (singleBlock == NULL) { + return NULL; + } + for (int i = 0; i < size; ++i) { + singleBlock[i] = reinterpret_cast<char*>(singleBlock) + blockPtr; + strncpy(singleBlock[i], stringVector[i].c_str(), stringLengths[i]); + blockPtr += stringLengths[i]; + } + return singleBlock; + } + + void backtrace_symbols_fd_emulation(void* const* array, int size, int fd) { + const int kBufferSize = 4 * 1024; + char stringBuffer[kBufferSize]; + for (int i = 0; i < size; ++i) { + int len = addrtosymstr(array[i], stringBuffer, kBufferSize); + if (len > kBufferSize - 1) { + len = kBufferSize - 1; + } + stringBuffer[len] = '\n'; + write(fd, stringBuffer, len + 1); + } + } + + typedef int (*BacktraceFunc)(void** array, int size); + static BacktraceFunc backtrace_switcher = + pal::backtrace_emulation; + + typedef char** (*BacktraceSymbolsFunc)(void* const* array, int size); + static BacktraceSymbolsFunc backtrace_symbols_switcher = + pal::backtrace_symbols_emulation; + + typedef void (*BacktraceSymbolsFdFunc)(void* const* array, int size, int fd); + static BacktraceSymbolsFdFunc backtrace_symbols_fd_switcher = + pal::backtrace_symbols_fd_emulation; + + int backtrace(void** array, int size) { + return backtrace_switcher(array, size); + } + + char** backtrace_symbols(void* const* array, int size) { + return backtrace_symbols_switcher(array, size); + } + + void backtrace_symbols_fd(void* const* array, int size, int fd) { + backtrace_symbols_fd_switcher(array, size, fd); + } + +} // namespace pal + + // 'backtrace()', 'backtrace_symbols()' and 'backtrace_symbols_fd()' on Solaris will call + // emulation functions if the symbols are not found + // + MONGO_INITIALIZER_GENERAL(SolarisBacktrace, + MONGO_NO_PREREQUISITES, + ("default"))(InitializerContext* context) { + void* functionAddress = dlsym(RTLD_DEFAULT, "backtrace"); + if (functionAddress != NULL) { + pal::backtrace_switcher = + reinterpret_cast<pal::BacktraceFunc>(functionAddress); + } + functionAddress = dlsym(RTLD_DEFAULT, "backtrace_symbols"); + if (functionAddress != NULL) { + pal::backtrace_symbols_switcher = + reinterpret_cast<pal::BacktraceSymbolsFunc>(functionAddress); + } + functionAddress = dlsym(RTLD_DEFAULT, "backtrace_symbols_fd"); + if (functionAddress != NULL) { + pal::backtrace_symbols_fd_switcher = + reinterpret_cast<pal::BacktraceSymbolsFdFunc>(functionAddress); + } + return Status::OK(); + } + +} // namespace mongo + +#endif // #if defined(__sunos__) +#endif // #if !defined(_WIN32) diff --git a/src/mongo/platform/backtrace.h b/src/mongo/platform/backtrace.h new file mode 100644 index 00000000000..962cfa43119 --- /dev/null +++ b/src/mongo/platform/backtrace.h @@ -0,0 +1,43 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if !defined(_WIN32) +#if defined(__sunos__) || !defined(MONGO_HAVE_EXECINFO_BACKTRACE) + +namespace mongo { + namespace pal { + int backtrace(void** array, int size); + char** backtrace_symbols(void* const* array, int size); + void backtrace_symbols_fd(void* const* array, int size, int fd); + } // namespace pal + using pal::backtrace; + using pal::backtrace_symbols; + using pal::backtrace_symbols_fd; +} // namespace mongo + +#else + +#include <execinfo.h> + +namespace mongo { + using ::backtrace; + using ::backtrace_symbols; + using ::backtrace_symbols_fd; +} // namespace mongo + +#endif +#endif diff --git a/src/mongo/platform/posix_fadvise.cpp b/src/mongo/platform/posix_fadvise.cpp new file mode 100644 index 00000000000..3053b02fbec --- /dev/null +++ b/src/mongo/platform/posix_fadvise.cpp @@ -0,0 +1,56 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if defined(__sunos__) + +#include "mongo/platform/posix_fadvise.h" + +#include <dlfcn.h> + +#include "mongo/base/init.h" +#include "mongo/base/status.h" + +namespace mongo { +namespace pal { + + int posix_fadvise_emulation(int fd, off_t offset, off_t len, int advice) { + return 0; + } + + typedef int (*PosixFadviseFunc)(int fd, off_t offset, off_t len, int advice); + static PosixFadviseFunc posix_fadvise_switcher = mongo::pal::posix_fadvise_emulation; + + int posix_fadvise(int fd, off_t offset, off_t len, int advice) { + return posix_fadvise_switcher(fd, offset, len, advice); + } + +} // namespace pal + + // 'posix_fadvise()' on Solaris will call the emulation if the symbol is not found + // + MONGO_INITIALIZER_GENERAL(SolarisPosixFadvise, + MONGO_NO_PREREQUISITES, + ("default"))(InitializerContext* context) { + void* functionAddress = dlsym(RTLD_DEFAULT, "posix_fadvise"); + if (functionAddress != NULL) { + mongo::pal::posix_fadvise_switcher = + reinterpret_cast<mongo::pal::PosixFadviseFunc>(functionAddress); + } + return Status::OK(); + } + +} // namespace mongo + +#endif // #if defined(__sunos__) diff --git a/src/mongo/platform/posix_fadvise.h b/src/mongo/platform/posix_fadvise.h new file mode 100644 index 00000000000..117b6b917f4 --- /dev/null +++ b/src/mongo/platform/posix_fadvise.h @@ -0,0 +1,41 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if !defined(_WIN32) + +#include <fcntl.h> + +#if defined(__sunos__) + +#include <sys/types.h> + +namespace mongo { + namespace pal { + int posix_fadvise(int fd, off_t offset, off_t len, int advice); + } // namespace pal + using pal::posix_fadvise; +} // namespace mongo + +#elif defined(POSIX_FADV_DONTNEED) + +namespace mongo { + using ::posix_fadvise; +} // namespace mongo + +#endif + +#endif diff --git a/src/mongo/platform/strcasestr.cpp b/src/mongo/platform/strcasestr.cpp new file mode 100644 index 00000000000..1bcd0a37094 --- /dev/null +++ b/src/mongo/platform/strcasestr.cpp @@ -0,0 +1,105 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/strcasestr.h" + +#if defined(__sunos__) +#include <dlfcn.h> + +#include "mongo/base/init.h" +#include "mongo/base/status.h" +#endif + +#if defined(_WIN32) || defined(__sunos__) + +#include <algorithm> +#include <cctype> +#include <cstring> +#include <string> + +#if defined(__sunos__) +#define STRCASESTR_EMULATION_NAME strcasestr_emulation +#else +#define STRCASESTR_EMULATION_NAME strcasestr +#endif + +namespace mongo { +namespace pal { + + /** + * strcasestr -- case-insensitive search for a substring within another string. + * + * @param haystack ptr to C-string to search + * @param needle ptr to C-string to try to find within 'haystack' + * @return ptr to start of 'needle' within 'haystack' if found, NULL otherwise + */ + const char* STRCASESTR_EMULATION_NAME(const char* haystack, const char* needle) { + + std::string haystackLower(haystack); + std::transform(haystackLower.begin(), + haystackLower.end(), + haystackLower.begin(), + ::tolower); + + std::string needleLower(needle); + std::transform(needleLower.begin(), + needleLower.end(), + needleLower.begin(), + ::tolower); + + // Use strstr() to find 'lowercased needle' in 'lowercased haystack' + // If found, use the location to compute the matching location in the original string + // If not found, return NULL + const char* haystackLowerStart = haystackLower.c_str(); + const char* location = strstr(haystackLowerStart, needleLower.c_str()); + return location ? (haystack + (location - haystackLowerStart)) : NULL; + } + +#if defined(__sunos__) + + typedef const char* (*StrCaseStrFunc)(const char* haystack, const char* needle); + static StrCaseStrFunc strcasestr_switcher = mongo::pal::strcasestr_emulation; + + const char* strcasestr(const char* haystack, const char* needle) { + return strcasestr_switcher(haystack, needle); + } + +#endif // #if defined(__sunos__) + +} // namespace pal +} // namespace mongo + +#endif // #if defined(_WIN32) || defined(__sunos__) + +#if defined(__sunos__) + +namespace mongo { + + // 'strcasestr()' on Solaris will call the emulation if the symbol is not found + // + MONGO_INITIALIZER_GENERAL(SolarisStrCaseCmp, + MONGO_NO_PREREQUISITES, + ("default"))(InitializerContext* context) { + void* functionAddress = dlsym(RTLD_DEFAULT, "strcasestr"); + if (functionAddress != NULL) { + mongo::pal::strcasestr_switcher = + reinterpret_cast<mongo::pal::StrCaseStrFunc>(functionAddress); + } + return Status::OK(); + } + +} // namespace mongo + +#endif // __sunos__ diff --git a/src/mongo/platform/strcasestr.h b/src/mongo/platform/strcasestr.h new file mode 100644 index 00000000000..844374f6915 --- /dev/null +++ b/src/mongo/platform/strcasestr.h @@ -0,0 +1,35 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(_WIN32) || defined(__sunos__) + +namespace mongo { +namespace pal { + const char* strcasestr(const char* haystack, const char* needle); +} + using mongo::pal::strcasestr; +} + +#else + +#include <cstring> + +namespace mongo { + using ::strcasestr; +} + +#endif diff --git a/src/mongo/s/balance.cpp b/src/mongo/s/balance.cpp index de72b25dada..2cf9d06d131 100644 --- a/src/mongo/s/balance.cpp +++ b/src/mongo/s/balance.cpp @@ -24,6 +24,7 @@ #include "mongo/db/jsobj.h" #include "mongo/s/chunk.h" #include "mongo/s/config.h" +#include "mongo/s/config_server_checker_service.h" #include "mongo/s/grid.h" #include "mongo/s/server.h" #include "mongo/s/shard.h" @@ -461,7 +462,14 @@ namespace mongo { sleepsecs( sleepTime ); // no need to wake up soon continue; } - + + if ( !isConfigServerConsistent() ) { + conn.done(); + warning() << "Skipping balancing round because data inconsistency" + << " was detected amongst the config servers." << endl; + continue; + } + LOG(1) << "*** start balancing round" << endl; bool waitForDelete = false; diff --git a/src/mongo/s/balancer_policy.cpp b/src/mongo/s/balancer_policy.cpp index 184451637ed..e5edf340c1c 100644 --- a/src/mongo/s/balancer_policy.cpp +++ b/src/mongo/s/balancer_policy.cpp @@ -77,8 +77,18 @@ namespace mongo { unsigned minChunks = numeric_limits<unsigned>::max(); for ( ShardInfoMap::const_iterator i = _shardInfo.begin(); i != _shardInfo.end(); ++i ) { - if ( i->second.isSizeMaxed() || i->second.isDraining() || i->second.hasOpsQueued() ) { - LOG(1) << i->first << " is unavailable" << endl; + if ( i->second.isSizeMaxed() ) { + LOG(1) << i->first << " has already reached the maximum total chunk size." << endl; + continue; + } + + if ( i->second.isDraining() ) { + LOG(1) << i->first << " is currently draining." << endl; + continue; + } + + if ( i->second.hasOpsQueued() ) { + LOG(1) << i->first << " has writebacks queued." << endl; continue; } diff --git a/src/mongo/s/chunk_diff.hpp b/src/mongo/s/chunk_diff.hpp index 0206851d26e..b5b0bab410b 100644 --- a/src/mongo/s/chunk_diff.hpp +++ b/src/mongo/s/chunk_diff.hpp @@ -173,8 +173,11 @@ namespace mongo { if( isTracked( diffChunkDoc ) ) newTracked.push_back( diffChunkDoc.getOwned() ); } - LOG(3) << "found " << _validDiffs << " new chunks for collection " << _ns - << " (tracking " << newTracked.size() << "), new version is " << _maxVersion << endl; + LOG(3) << "found " << _validDiffs + << " new chunks for collection " << _ns + << " (tracking " << newTracked.size() + << "), new version is " << *_maxVersion + << endl; for( vector<BSONObj>::iterator it = newTracked.begin(); it != newTracked.end(); it++ ){ @@ -279,9 +282,6 @@ namespace mongo { BSONObj query = queryB.obj(); - LOG(2) << "major version query from " << *_maxVersion << " and over " - << _maxShardVersions->size() << " shards is " << query << endl; - // // NOTE: IT IS IMPORTANT FOR CONSISTENCY THAT WE SORT BY ASC VERSION, TO HANDLE // CURSOR YIELDING BETWEEN CHUNKS BEING MIGRATED. @@ -293,7 +293,10 @@ namespace mongo { Query queryObj(query); queryObj.sort(BSON( "lastmod" << 1 )); - return Query( query ); + LOG(2) << "major version query from " << *_maxVersion << " and over " + << _maxShardVersions->size() << " shards is " << queryObj << endl; + + return queryObj; } } // namespace mongo diff --git a/src/mongo/s/config_server_checker_service.cpp b/src/mongo/s/config_server_checker_service.cpp new file mode 100644 index 00000000000..4d30cdcf50f --- /dev/null +++ b/src/mongo/s/config_server_checker_service.cpp @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <boost/scoped_ptr.hpp> +#include <boost/thread/thread.hpp> + +#include "mongo/s/config.h" +#include "mongo/s/config_server_checker_service.h" +#include "mongo/util/concurrency/mutex.h" + +namespace mongo { + + namespace { + + // Thread that runs dbHash on config servers for checking data consistency. + boost::scoped_ptr<boost::thread> _checkerThread; + + // Protects _isConsistentFromLastCheck. + mutex _isConsistentMutex( "ConfigServerConsistent" ); + bool _isConsistentFromLastCheck = true; + + void checkConfigConsistency() { + while ( !inShutdown() ) { + bool isConsistent = configServer.ok( true ); + + { + scoped_lock sl( _isConsistentMutex ); + _isConsistentFromLastCheck = isConsistent; + } + + sleepsecs( 60 ); + } + } + } + + bool isConfigServerConsistent() { + scoped_lock sl( _isConsistentMutex ); + return _isConsistentFromLastCheck; + } + + bool startConfigServerChecker() { + if ( _checkerThread == NULL ) { + _checkerThread.reset( new boost::thread( checkConfigConsistency )); + } + + return _checkerThread != NULL; + } +} + diff --git a/src/mongo/s/config_server_checker_service.h b/src/mongo/s/config_server_checker_service.h new file mode 100644 index 00000000000..4661d0fc756 --- /dev/null +++ b/src/mongo/s/config_server_checker_service.h @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +namespace mongo { + + /** + * Returns true if the config servers have the same contents since the last check + * was performed. Currently checks only the config.chunks and config.databases. + */ + bool isConfigServerConsistent(); + + /** + * Starts the thread that periodically checks data consistency amongst the config servers. + * Note: this is not thread safe. + */ + bool startConfigServerChecker(); +} + diff --git a/src/mongo/s/d_migrate.cpp b/src/mongo/s/d_migrate.cpp index 139a640f5ce..85eb18294dd 100644 --- a/src/mongo/s/d_migrate.cpp +++ b/src/mongo/s/d_migrate.cpp @@ -589,10 +589,11 @@ namespace mongo { } BSONObj o = dl.obj(); - + // use the builder size instead of accumulating 'o's size so that we take into consideration - // the overhead of BSONArray indices - if ( a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) { + // the overhead of BSONArray indices, and *always* append one doc + if ( a.arrSize() != 0 && + a.len() + o.objsize() + 1024 > BSONObjMaxUserSize ) { filledBuffer = true; // break out of outer while loop break; } @@ -638,6 +639,11 @@ namespace mongo { _cloneLocs.erase( dl ); } + std::size_t cloneLocsRemaining() { + scoped_spinlock lk( _trackerLocks ); + return _cloneLocs.size(); + } + long long mbUsed() const { return _memoryUsed / ( 1024 * 1024 ); } bool getInCriticalSection() const { @@ -1161,15 +1167,20 @@ namespace mongo { timing.done( 3 ); // 4. + + // Track last result from TO shard for sanity check + BSONObj res; for ( int i=0; i<86400; i++ ) { // don't want a single chunk move to take more than a day verify( !Lock::isLocked() ); // Exponential sleep backoff, up to 1024ms. Don't sleep much on the first few // iterations, since we want empty chunk migrations to be fast. sleepmillis( 1 << std::min( i , 10 ) ); + scoped_ptr<ScopedDbConnection> conn( ScopedDbConnection::getScopedDbConnection( toShard.getConnString() ) ); - BSONObj res; + bool ok; + res = BSONObj(); try { ok = conn->get()->runCommand( "admin" , BSON( "_recvChunkStatus" << 1 ) , res ); res = res.getOwned(); @@ -1217,9 +1228,26 @@ namespace mongo { // 5. // Before we get into the critical section of the migration, let's double check - // that the config servers are reachable and the lock is in place. - log() << "About to check if it is safe to enter critical section"; + // that the docs have been cloned, the config servers are reachable, + // and the lock is in place. + log() << "About to check if it is safe to enter critical section" << endl; + + // Ensure all cloned docs have actually been transferred + std::size_t locsRemaining = migrateFromStatus.cloneLocsRemaining(); + if ( locsRemaining != 0 ) { + + errmsg = + str::stream() << "moveChunk cannot enter critical section before all data is" + << " cloned, " << locsRemaining << " locs were not transferred" + << " but to-shard reported " << res; + + // Should never happen, but safe to abort before critical section + error() << errmsg << migrateLog; + dassert( false ); + return false; + } + // Ensure distributed lock still held string lockHeldMsg; bool lockHeld = dlk.isLockHeld( 30.0 /* timeout */, &lockHeldMsg ); if ( !lockHeld ) { @@ -1252,49 +1280,48 @@ namespace mongo { // 5.b // we're under the collection lock here, too, so we can undo the chunk donation because no other state change // could be ongoing - { - BSONObj res; + + BSONObj res; + bool ok; + + try { scoped_ptr<ScopedDbConnection> connTo( ScopedDbConnection::getScopedDbConnection( toShard.getConnString(), 35.0 ) ); - - bool ok; - - try{ - ok = connTo->get()->runCommand( "admin" , - BSON( "_recvChunkCommit" << 1 ) , - res ); - } - catch( DBException& e ){ - errmsg = str::stream() << "moveChunk could not contact to: shard " << toShard.getConnString() << " to commit transfer" << causedBy( e ); - warning() << errmsg << endl; - ok = false; - } - + ok = connTo->get()->runCommand( "admin", BSON( "_recvChunkCommit" << 1 ), res ); connTo->done(); + } + catch ( DBException& e ) { + errmsg = str::stream() << "moveChunk could not contact to: shard " + << toShard.getConnString() << " to commit transfer" + << causedBy( e ); + warning() << errmsg << endl; + ok = false; + } - if ( ! ok ) { - log() << "moveChunk migrate commit not accepted by TO-shard: " << res - << " resetting shard version to: " << startingVersion << migrateLog; - { - Lock::GlobalWrite lk; - log() << "moveChunk global lock acquired to reset shard version from " - "failed migration" << endl; - - // revert the chunk manager back to the state before "forgetting" about the chunk - shardingState.undoDonateChunk( ns , min , max , startingVersion ); - } - log() << "Shard version successfully reset to clean up failed migration" - << endl; + if ( !ok ) { + log() << "moveChunk migrate commit not accepted by TO-shard: " << res + << " resetting shard version to: " << startingVersion << migrateLog; + { + Lock::GlobalWrite lk; + log() << "moveChunk global lock acquired to reset shard version from " + "failed migration" + << endl; - errmsg = "_recvChunkCommit failed!"; - result.append( "cause" , res ); - return false; + // revert the chunk manager back to the state before "forgetting" about the + // chunk + shardingState.undoDonateChunk( ns, min, max, startingVersion ); } + log() << "Shard version successfully reset to clean up failed migration" + << endl; - log() << "moveChunk migrate commit accepted by TO-shard: " << res << migrateLog; + errmsg = "_recvChunkCommit failed!"; + result.append( "cause", res ); + return false; } + log() << "moveChunk migrate commit accepted by TO-shard: " << res << migrateLog; + // 5.c // version at which the next highest lastmod will be set @@ -1404,7 +1431,7 @@ namespace mongo { LOG(7) << "moveChunk update: " << cmd << migrateLog; int exceptionCode = OkCode; - bool ok = false; + ok = false; BSONObj cmdResult; try { scoped_ptr<ScopedDbConnection> conn( diff --git a/src/mongo/s/d_split.cpp b/src/mongo/s/d_split.cpp index 5231e9e571b..86f9d5ef728 100644 --- a/src/mongo/s/d_split.cpp +++ b/src/mongo/s/d_split.cpp @@ -289,13 +289,14 @@ namespace mongo { // 'force'-ing a split is equivalent to having maxChunkSize be the size of the current chunk, i.e., the // logic below will split that chunk in half long long maxChunkSize = 0; - bool force = false; + bool forceMedianSplit = false; { BSONElement maxSizeElem = jsobj[ "maxChunkSize" ]; BSONElement forceElem = jsobj[ "force" ]; if ( forceElem.trueValue() ) { - force = true; + forceMedianSplit = true; + // This chunk size is effectively ignored if force is true maxChunkSize = dataSize; } @@ -365,7 +366,8 @@ namespace mongo { while ( cc->ok() ) { currCount++; - if ( currCount > keyCount ) { + if ( currCount > keyCount && !forceMedianSplit ) { + BSONObj currKey = bc->prettyKey( c->currKey() ).extractFields(keyPattern); // Do not use this split key if it is the same used in the previous split point. if ( currKey.woCompare( splitKeys.back() ) == 0 ) { @@ -403,10 +405,15 @@ namespace mongo { } } - if ( splitKeys.size() > 1 || ! force ) + if ( ! forceMedianSplit ) break; - force = false; + // + // If we're forcing a split at the halfway point, then the first pass was just + // to count the keys, and we still need a second pass. + // + + forceMedianSplit = false; keyCount = currCount / 2; currCount = 0; log() << "splitVector doing another cycle because of force, keyCount now: " << keyCount << endl; diff --git a/src/mongo/s/d_state.cpp b/src/mongo/s/d_state.cpp index 95ba21ab8f9..bca04ff565a 100644 --- a/src/mongo/s/d_state.cpp +++ b/src/mongo/s/d_state.cpp @@ -275,6 +275,10 @@ namespace mongo { } { + // NOTE: This lock prevents the ns version from changing while a write operation occurs. + Lock::DBRead readLk(ns); + + // This lock prevents simultaneous metadata changes using the same map scoped_lock lk( _mutex ); // since we loaded the chunk manager unlocked, other thread may have done the same diff --git a/src/mongo/s/mongos.vcxproj b/src/mongo/s/mongos.vcxproj index fb597f75bce..97300042b3f 100644 --- a/src/mongo/s/mongos.vcxproj +++ b/src/mongo/s/mongos.vcxproj @@ -2442,7 +2442,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\db\server_parameters.cpp" />
<ClCompile Include="..\db\stats\timer_stats.cpp" />
<ClCompile Include="..\db\stats\top.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bench.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
@@ -2538,6 +2541,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="commands_admin.cpp" />
<ClCompile Include="commands_public.cpp" />
<ClCompile Include="config.cpp" />
+ <ClCompile Include="config_server_checker_service.cpp" />
<ClCompile Include="config_upgrade.cpp" />
<ClCompile Include="config_upgrade_helpers.cpp" />
<ClCompile Include="config_upgrade_v0_to_v4.cpp" />
@@ -3136,13 +3140,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
@@ -3259,6 +3266,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="cluster_client_internal.h" />
<ClInclude Include="collection_manager.h" />
<ClInclude Include="config.h" />
+ <ClInclude Include="config_server_checker_service.h" />
<ClInclude Include="config_upgrade.h" />
<ClInclude Include="config_upgrade_helpers.h" />
<ClInclude Include="cursors.h" />
diff --git a/src/mongo/s/mongos.vcxproj.filters b/src/mongo/s/mongos.vcxproj.filters index 691485ab06c..c55dc8ff667 100755 --- a/src/mongo/s/mongos.vcxproj.filters +++ b/src/mongo/s/mongos.vcxproj.filters @@ -1632,6 +1632,18 @@ <ClCompile Include="..\client\sasl_client_authenticate.cpp">
<Filter>client</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="config_server_checker_service.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\pch.h" />
@@ -3210,6 +3222,18 @@ <ClInclude Include="..\client\sasl_client_authenticate.h">
<Filter>client</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="config_server_checker_service.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\base\error_codes.err">
diff --git a/src/mongo/s/mongos_sm.vcxproj b/src/mongo/s/mongos_sm.vcxproj index e02683838f5..bb4d032f8e3 100644 --- a/src/mongo/s/mongos_sm.vcxproj +++ b/src/mongo/s/mongos_sm.vcxproj @@ -2271,7 +2271,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\db\server_parameters.cpp" />
<ClCompile Include="..\db\stats\timer_stats.cpp" />
<ClCompile Include="..\db\stats\top.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bench.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
@@ -2394,6 +2397,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="commands_admin.cpp" />
<ClCompile Include="commands_public.cpp" />
<ClCompile Include="config.cpp" />
+ <ClCompile Include="config_server_checker_service.cpp" />
<ClCompile Include="config_upgrade.cpp" />
<ClCompile Include="config_upgrade_helpers.cpp" />
<ClCompile Include="config_upgrade_v0_to_v4.cpp" />
@@ -2719,13 +2723,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
@@ -2869,6 +2876,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="cluster_client_internal.h" />
<ClInclude Include="collection_manager.h" />
<ClInclude Include="config.h" />
+ <ClInclude Include="config_server_checker_service.h" />
<ClInclude Include="config_upgrade.h" />
<ClInclude Include="config_upgrade_helpers.h" />
<ClInclude Include="cursors.h" />
diff --git a/src/mongo/s/mongos_sm.vcxproj.filters b/src/mongo/s/mongos_sm.vcxproj.filters index 8b107b3dc2f..7660ede7014 100644 --- a/src/mongo/s/mongos_sm.vcxproj.filters +++ b/src/mongo/s/mongos_sm.vcxproj.filters @@ -1233,6 +1233,18 @@ <ClCompile Include="..\client\sasl_client_authenticate.cpp">
<Filter>client</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="config_server_checker_service.cpp">
+ <Filter>Source Files</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\pch.h" />
@@ -2349,6 +2361,18 @@ <ClInclude Include="..\client\sasl_client_authenticate.h">
<Filter>client</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="config_server_checker_service.h">
+ <Filter>Header Files</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="..\..\third_party\js-1.7\jskeyword.tbl">
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 6f5f4ed52f7..e6aa728d7a3 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -45,6 +45,7 @@ #include "cursors.h" #include "../util/processinfo.h" #include "mongo/db/lasterror.h" +#include "mongo/s/config_server_checker_service.h" #include "mongo/s/config_upgrade.h" #include "mongo/util/stacktrace.h" #include "mongo/util/exception_filter_win32.h" @@ -292,14 +293,7 @@ static bool runMongosServer( bool doUpgrade ) { return false; } - { - class CheckConfigServers : public task::Task { - virtual string name() const { return "CheckConfigServers"; } - virtual void doWork() { configServer.ok(true); } - }; - - task::repeat(new CheckConfigServers, 60*1000); - } + startConfigServerChecker(); VersionType initVersionInfo; VersionType versionInfo; diff --git a/src/mongo/scripting/engine_v8.cpp b/src/mongo/scripting/engine_v8.cpp index 7ffebec383e..5aa40b1abf0 100644 --- a/src/mongo/scripting/engine_v8.cpp +++ b/src/mongo/scripting/engine_v8.cpp @@ -919,7 +919,7 @@ namespace mongo { // script loaded from file ss << " at " << *resourceName; const int linenum = message->GetLineNumber(); - if (linenum != 1) ss << ":L" << linenum; + if (linenum != 1) ss << ":" << linenum; } return ss.str(); } @@ -1572,6 +1572,14 @@ namespace mongo { void V8Scope::v8ToMongoElement(BSONObjBuilder & b, const StringData& sname, v8::Handle<v8::Value> value, int depth, BSONObj* originalParent) { + + // Null char should be at the end, not in the string + uassert(16985, + str::stream() << "JavaScript property (name) contains a null char " + << "which is not allowed in BSON. " + << originalParent->jsonString(), + (string::npos == sname.find('\0')) ); + if (value->IsString()) { b.append(sname, V8String(value)); return; diff --git a/src/mongo/scripting/engine_v8.h b/src/mongo/scripting/engine_v8.h index 448ab38af26..442441c55ce 100644 --- a/src/mongo/scripting/engine_v8.h +++ b/src/mongo/scripting/engine_v8.h @@ -530,7 +530,7 @@ namespace mongo { if (try_catch.HasCaught() && try_catch.CanContinue()) { // normal JS exception - _error = string("JavaScript execution failed: ") + v8ExceptionToSTLString(&try_catch); + _error = v8ExceptionToSTLString(&try_catch); haveError = true; } else if (hasOutOfMemoryException()) { diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 4d71d2b8d19..5934e862076 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -1043,10 +1043,6 @@ int _main( int argc, char* argv[], char **envp ) { #ifdef _WIN32 int wmain( int argc, wchar_t* argvW[] ) { static mongo::StaticObserver staticObserver; - UINT initialConsoleInputCodePage = GetConsoleCP(); - UINT initialConsoleOutputCodePage = GetConsoleOutputCP(); - SetConsoleCP( CP_UTF8 ); - SetConsoleOutputCP( CP_UTF8 ); int returnCode; try { WindowsCommandLine wcl( argc, argvW ); @@ -1056,8 +1052,6 @@ int wmain( int argc, wchar_t* argvW[] ) { cerr << "exception: " << e.what() << endl; returnCode = 1; } - SetConsoleCP( initialConsoleInputCodePage ); - SetConsoleOutputCP( initialConsoleOutputCodePage ); ::_exit(returnCode); } #else // #ifdef _WIN32 diff --git a/src/mongo/shell/mongo.js b/src/mongo/shell/mongo.js index e4786a7cc17..2f3667f53f3 100644 --- a/src/mongo/shell/mongo.js +++ b/src/mongo/shell/mongo.js @@ -98,26 +98,58 @@ Mongo.prototype.getReadPrefTagSet = function () { return this._readPrefTagSet; }; -connect = function( url , user , pass ){ - chatty( "connecting to: " + url ) - - if ( user && ! pass ) - throw "you specified a user and not a password. either you need a password, or you're using the old connect api"; +connect = function(url, user, pass) { + if (user && !pass) + throw Error("you specified a user and not a password. " + + "either you need a password, or you're using the old connect api"); + + // Validate connection string "url" as "hostName:portNumber/databaseName" + // or "hostName/databaseName" + // or "databaseName" + // hostName may be an IPv6 address (with colons), in which case ":portNumber" is required + // + var urlType = typeof url; + if (urlType == "undefined") { + throw Error("Missing connection string"); + } + if (urlType != "string") { + throw Error("Incorrect type \"" + urlType + + "\" for connection string \"" + tojson(url) + "\""); + } + url = url.trim(); + if (0 == url.length) { + throw Error("Empty connection string"); + } + var colon = url.lastIndexOf(":"); + var slash = url.lastIndexOf("/"); + if (0 == colon || 0 == slash) { + throw Error("Missing host name in connection string \"" + url + "\""); + } + if (colon == slash - 1 || colon == url.length - 1) { + throw Error("Missing port number in connection string \"" + url + "\""); + } + if (colon != -1 && colon < slash) { + var portNumber = url.substring(colon + 1, slash); + if (portNumber.length > 5 || !/^\d*$/.test(portNumber) || parseInt(portNumber) > 65535) { + throw Error("Invalid port number \"" + portNumber + + "\" in connection string \"" + url + "\""); + } + } + if (slash == url.length - 1) { + throw Error("Missing database name in connection string \"" + url + "\""); + } - var idx = url.lastIndexOf( "/" ); - + chatty("connecting to: " + url) var db; - - if ( idx < 0 ) - db = new Mongo().getDB( url ); + if (slash == -1) + db = new Mongo().getDB(url); else - db = new Mongo( url.substring( 0 , idx ) ).getDB( url.substring( idx + 1 ) ); - - if ( user && pass ){ - if ( ! db.auth( user , pass ) ){ - throw "couldn't login"; + db = new Mongo(url.substring(0, slash)).getDB(url.substring(slash + 1)); + + if (user && pass) { + if (!db.auth(user, pass)) { + throw Error("couldn't login"); } } - return db; } diff --git a/src/mongo/shell/mongo.vcxproj b/src/mongo/shell/mongo.vcxproj index 48bc5625c62..219f43e2488 100755 --- a/src/mongo/shell/mongo.vcxproj +++ b/src/mongo/shell/mongo.vcxproj @@ -382,6 +382,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj </PreBuildEvent>
</ItemDefinitionGroup>
<ItemGroup>
+ <ClCompile Include="..\..\third_party\murmurhash3\MurmurHash3.cpp">
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|Win32'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Release|x64'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|x64'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'">4355;4800;4267;4244;4351;4141</DisableSpecificWarnings>
+ </ClCompile>
<ClCompile Include="..\..\third_party\v8\src\experimental-libraries.cc">
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">..\..\third_party\v8\src</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">..\..\third_party\v8\src</AdditionalIncludeDirectories>
@@ -1074,7 +1084,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\client\connection_factory.cpp" />
<ClCompile Include="..\client\sasl_client_authenticate.cpp" />
<ClCompile Include="..\db\dbmessage.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
@@ -1271,6 +1284,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="shell_utils_launcher.cpp" />
</ItemGroup>
<ItemGroup>
+ <None Include="..\..\third_party\murmurhash3\SConscript" />
<None Include="..\..\third_party\run_if_newer.js" />
<None Include="..\..\third_party\v8\ChangeLog" />
<None Include="..\..\third_party\v8\ChangeLog_10gen" />
@@ -1309,6 +1323,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <None Include="utils_sh.js" />
</ItemGroup>
<ItemGroup>
+ <ClInclude Include="..\..\third_party\murmurhash3\MurmurHash3.h" />
<ClInclude Include="..\..\third_party\v8\include\v8-debug.h" />
<ClInclude Include="..\..\third_party\v8\include\v8-preparser.h" />
<ClInclude Include="..\..\third_party\v8\include\v8-profiler.h" />
@@ -1639,13 +1654,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
diff --git a/src/mongo/shell/mongo.vcxproj.filters b/src/mongo/shell/mongo.vcxproj.filters index 0af752a3e3b..4f0661b84e3 100644 --- a/src/mongo/shell/mongo.vcxproj.filters +++ b/src/mongo/shell/mongo.vcxproj.filters @@ -158,6 +158,9 @@ <Filter Include="JavaScript source files\Included in shell only">
<UniqueIdentifier>{0d7ccbd5-b4e4-4e57-ae0b-b4a80a6fa6bb}</UniqueIdentifier>
</Filter>
+ <Filter Include="third_party\MurmurHash3">
+ <UniqueIdentifier>{5ced80f0-4dce-421a-ba59-1b338973dd97}</UniqueIdentifier>
+ </Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\scripting\engine.cpp">
@@ -1016,6 +1019,18 @@ <ClCompile Include="..\client\sasl_client_authenticate.cpp">
<Filter>client</Filter>
</ClCompile>
+ <ClCompile Include="..\..\third_party\murmurhash3\MurmurHash3.cpp">
+ <Filter>third_party\MurmurHash3</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="..\..\third_party\v8\src\apinatives.js">
@@ -1126,6 +1141,9 @@ <None Include="types.js">
<Filter>JavaScript source files\Included in all executables</Filter>
</None>
+ <None Include="..\..\third_party\murmurhash3\SConscript">
+ <Filter>third_party\MurmurHash3</Filter>
+ </None>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\pch.h" />
@@ -2387,6 +2405,18 @@ <ClInclude Include="..\client\sasl_client_authenticate.h">
<Filter>client</Filter>
</ClInclude>
+ <ClInclude Include="..\..\third_party\murmurhash3\MurmurHash3.h">
+ <Filter>third_party\MurmurHash3</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\db\db.rc">
diff --git a/src/mongo/shell/mongo_sm.vcxproj b/src/mongo/shell/mongo_sm.vcxproj index 0f2bd65b5b8..82381afe47b 100644 --- a/src/mongo/shell/mongo_sm.vcxproj +++ b/src/mongo/shell/mongo_sm.vcxproj @@ -876,6 +876,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|x64'">4355;4800;4267;4244;4334</DisableSpecificWarnings>
<DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'">4355;4800;4267;4244;4334</DisableSpecificWarnings>
</ClCompile>
+ <ClCompile Include="..\..\third_party\murmurhash3\MurmurHash3.cpp">
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|Win32'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Release|x64'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|x64'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ <DisableSpecificWarnings Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'">4355;4800;4267;4244;4141</DisableSpecificWarnings>
+ </ClCompile>
<ClCompile Include="..\base\configuration_variable_manager.cpp" />
<ClCompile Include="..\base\error_codes.cpp" />
<ClCompile Include="..\base\global_initializer.cpp" />
@@ -897,7 +907,10 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClCompile Include="..\client\connection_factory.cpp" />
<ClCompile Include="..\client\sasl_client_authenticate.cpp" />
<ClCompile Include="..\db\dbmessage.cpp" />
+ <ClCompile Include="..\platform\backtrace.cpp" />
+ <ClCompile Include="..\platform\posix_fadvise.cpp" />
<ClCompile Include="..\platform\random.cpp" />
+ <ClCompile Include="..\platform\strcasestr.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator.cpp" />
<ClCompile Include="..\scripting\bson_template_evaluator_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
@@ -1114,6 +1127,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ItemGroup>
<None Include="..\..\third_party\js-1.7\jskeyword.tbl" />
<None Include="..\..\third_party\js-1.7\jsopcode.tbl" />
+ <None Include="..\..\third_party\murmurhash3\SConscript" />
<None Include="..\base\error_codes.err" />
<None Include="..\base\generate_error_codes.py" />
<None Include="assert.js" />
@@ -1180,6 +1194,7 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\..\third_party\js-1.7\jsxml.h" />
<ClInclude Include="..\..\third_party\js-1.7\prmjtime.h" />
<ClInclude Include="..\..\third_party\js-1.7\resource.h" />
+ <ClInclude Include="..\..\third_party\murmurhash3\MurmurHash3.h" />
<ClInclude Include="..\base\configuration_variable_manager.h" />
<ClInclude Include="..\base\counter.h" />
<ClInclude Include="..\base\disallow_copying.h" />
@@ -1211,13 +1226,16 @@ cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(Proj <ClInclude Include="..\platform\atomic_intrinsics.h" />
<ClInclude Include="..\platform\atomic_intrinsics_win32.h" />
<ClInclude Include="..\platform\atomic_word.h" />
+ <ClInclude Include="..\platform\backtrace.h" />
<ClInclude Include="..\platform\basic.h" />
<ClInclude Include="..\platform\bits.h" />
<ClInclude Include="..\platform\compiler.h" />
<ClInclude Include="..\platform\compiler_msvc.h" />
<ClInclude Include="..\platform\cstdint.h" />
<ClInclude Include="..\platform\float_utils.h" />
+ <ClInclude Include="..\platform\posix_fadvise.h" />
<ClInclude Include="..\platform\random.h" />
+ <ClInclude Include="..\platform\strcasestr.h" />
<ClInclude Include="..\platform\strtoll.h" />
<ClInclude Include="..\platform\unordered_map.h" />
<ClInclude Include="..\platform\unordered_set.h" />
diff --git a/src/mongo/shell/mongo_sm.vcxproj.filters b/src/mongo/shell/mongo_sm.vcxproj.filters index 1c015105890..33678d8fb74 100644 --- a/src/mongo/shell/mongo_sm.vcxproj.filters +++ b/src/mongo/shell/mongo_sm.vcxproj.filters @@ -104,6 +104,9 @@ <Filter Include="JavaScript source files\Included in shell only">
<UniqueIdentifier>{283d932c-5594-47a6-9b1b-8f5af53c0337}</UniqueIdentifier>
</Filter>
+ <Filter Include="third_party\MurmurHash3">
+ <UniqueIdentifier>{a07219b6-a351-468a-a705-2b691500c617}</UniqueIdentifier>
+ </Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\scripting\engine.cpp">
@@ -623,6 +626,18 @@ <ClCompile Include="..\client\sasl_client_authenticate.cpp">
<Filter>client</Filter>
</ClCompile>
+ <ClCompile Include="..\platform\strcasestr.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\..\third_party\murmurhash3\MurmurHash3.cpp">
+ <Filter>third_party\MurmurHash3</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\backtrace.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
+ <ClCompile Include="..\platform\posix_fadvise.cpp">
+ <Filter>platform</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="..\..\third_party\js-1.7\jskeyword.tbl">
@@ -679,6 +694,9 @@ <None Include="utils.js">
<Filter>JavaScript source files\Included in all executables</Filter>
</None>
+ <None Include="..\..\third_party\murmurhash3\SConscript">
+ <Filter>third_party\MurmurHash3</Filter>
+ </None>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\pch.h" />
@@ -1454,6 +1472,18 @@ <ClInclude Include="..\client\sasl_client_authenticate.h">
<Filter>client</Filter>
</ClInclude>
+ <ClInclude Include="..\platform\strcasestr.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\..\third_party\murmurhash3\MurmurHash3.h">
+ <Filter>third_party\MurmurHash3</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\backtrace.h">
+ <Filter>platform</Filter>
+ </ClInclude>
+ <ClInclude Include="..\platform\posix_fadvise.h">
+ <Filter>platform</Filter>
+ </ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\db\db.rc">
diff --git a/src/mongo/tools/tool.cpp b/src/mongo/tools/tool.cpp index c2728b88af0..2539e5cacb4 100644 --- a/src/mongo/tools/tool.cpp +++ b/src/mongo/tools/tool.cpp @@ -18,20 +18,20 @@ #include "mongo/tools/tool.h" +#include <boost/filesystem/operations.hpp> #include <fstream> #include <iostream> #include "pcrecpp.h" +#include "mongo/client/dbclient_rs.h" #include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/json.h" #include "mongo/db/namespace_details.h" +#include "mongo/platform/posix_fadvise.h" #include "mongo/util/file_allocator.h" #include "mongo/util/password.h" #include "mongo/util/version.h" -#include "mongo/client/dbclient_rs.h" -#include "mongo/db/json.h" - -#include <boost/filesystem/operations.hpp> using namespace std; using namespace mongo; @@ -480,7 +480,7 @@ namespace mongo { return 0; } -#if !defined(__sunos__) && defined(POSIX_FADV_SEQUENTIAL) +#ifdef POSIX_FADV_SEQUENTIAL posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL); #endif diff --git a/src/mongo/util/file_allocator.cpp b/src/mongo/util/file_allocator.cpp index 07a962b42a2..e59bce6934b 100644 --- a/src/mongo/util/file_allocator.cpp +++ b/src/mongo/util/file_allocator.cpp @@ -16,11 +16,13 @@ */ #include "mongo/pch.h" + #include "mongo/util/file_allocator.h" + #include <boost/thread.hpp> #include <boost/filesystem/operations.hpp> -#include <fcntl.h> #include <errno.h> +#include <fcntl.h> #if defined(__freebsd__) || defined(__openbsd__) # include <sys/stat.h> @@ -34,10 +36,11 @@ # include <io.h> #endif -#include "mongo/util/time_support.h" -#include "mongo/util/timer.h" +#include "mongo/platform/posix_fadvise.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/paths.h" +#include "mongo/util/time_support.h" +#include "mongo/util/timer.h" using namespace mongoutils; diff --git a/src/mongo/util/log.cpp b/src/mongo/util/log.cpp index 00845d07da1..9ed7c224f6e 100644 --- a/src/mongo/util/log.cpp +++ b/src/mongo/util/log.cpp @@ -18,6 +18,7 @@ #include "mongo/pch.h" +#include "mongo/platform/posix_fadvise.h" #include "mongo/util/assert_util.h" #include "mongo/util/concurrency/threadlocal.h" #include "mongo/util/stacktrace.h" diff --git a/src/mongo/util/logfile.cpp b/src/mongo/util/logfile.cpp index 23b3b597e19..9ab57deac3a 100644 --- a/src/mongo/util/logfile.cpp +++ b/src/mongo/util/logfile.cpp @@ -16,12 +16,16 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include "pch.h" -#include "logfile.h" -#include "text.h" -#include "mongoutils/str.h" -#include "mongo/util/startup_test.h" +#include "mongo/pch.h" + +#include "mongo/util/logfile.h" + +#include "mongo/platform/posix_fadvise.h" #include "mongo/util/mmap.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/startup_test.h" +#include "mongo/util/text.h" + using namespace mongoutils; @@ -64,7 +68,7 @@ namespace mongo { FILE_SHARE_READ, NULL, OPEN_ALWAYS, - FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH, + FILE_FLAG_NO_BUFFERING, NULL); if( _fd == INVALID_HANDLE_VALUE ) { DWORD e = GetLastError(); diff --git a/src/mongo/util/mmap_posix.cpp b/src/mongo/util/mmap_posix.cpp index 7eb5f062045..46f8c481ae3 100644 --- a/src/mongo/util/mmap_posix.cpp +++ b/src/mongo/util/mmap_posix.cpp @@ -174,6 +174,11 @@ namespace mongo { } void* MemoryMappedFile::remapPrivateView(void *oldPrivateAddr) { +#if defined(__sunos__) // SERVER-8795 + verify( Lock::isW() ); + LockMongoFilesExclusive lockMongoFiles; +#endif + // don't unmap, just mmap over the old region void * x = mmap( oldPrivateAddr, len , PROT_READ|PROT_WRITE , MAP_PRIVATE|MAP_NORESERVE|MAP_FIXED , fd , 0 ); if( x == MAP_FAILED ) { diff --git a/src/mongo/util/net/hostandport.h b/src/mongo/util/net/hostandport.h index 2d8e435293c..ff47d7f9ef4 100644 --- a/src/mongo/util/net/hostandport.h +++ b/src/mongo/util/net/hostandport.h @@ -170,7 +170,7 @@ namespace mongo { const char *colon = strrchr(p, ':'); if( colon ) { int port = atoi(colon+1); - uassert(13095, "HostAndPort: bad port #", port > 0); + massert(13095, "HostAndPort: bad port #", port > 0); _host = string(p,colon-p); _port = port; } diff --git a/src/mongo/util/net/sock.cpp b/src/mongo/util/net/sock.cpp index 2f155c5295c..f8eb34414d6 100644 --- a/src/mongo/util/net/sock.cpp +++ b/src/mongo/util/net/sock.cpp @@ -377,7 +377,7 @@ namespace mongo { string SocketException::toString() const { stringstream ss; - ss << _ei.code << " socket exception [" << _type << "] "; + ss << _ei.code << " socket exception [" << _getStringType(_type) << "] "; if ( _server.size() ) ss << "server [" << _server << "] "; @@ -425,6 +425,12 @@ namespace mongo { void Socket::close() { if ( _fd >= 0 ) { + // Stop any blocking reads/writes, and prevent new reads/writes +#if defined(_WIN32) + shutdown( _fd, SD_BOTH ); +#else + shutdown( _fd, SHUT_RDWR ); +#endif closesocket( _fd ); _fd = -1; } diff --git a/src/mongo/util/net/sock_test.cpp b/src/mongo/util/net/sock_test.cpp index c1e48c63d43..1bc9df617d6 100644 --- a/src/mongo/util/net/sock_test.cpp +++ b/src/mongo/util/net/sock_test.cpp @@ -48,9 +48,9 @@ namespace { typedef boost::shared_ptr<Socket> SocketPtr; typedef std::pair<SocketPtr, SocketPtr> SocketPair; - // On UNIX, make a connected pair of PF_LOCAL sockets via the native 'socketpair' call. The - // 'type' parameter should be one of SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET, etc. For - // Win32, we don't have a native socketpair function, so we hack up a connected PF_INET + // On UNIX, make a connected pair of PF_LOCAL (aka PF_UNIX) sockets via the native 'socketpair' + // call. The 'type' parameter should be one of SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET, etc. + // For Win32, we don't have a native socketpair function, so we hack up a connected PF_INET // pair on a random port. SocketPair socketPair(const int type, const int protocol = 0); @@ -184,7 +184,10 @@ namespace { #else // We can just use ::socketpair and wrap up the result in a Socket. SocketPair socketPair(const int type, const int protocol) { - const int domain = PF_LOCAL; + // PF_LOCAL is the POSIX name for Unix domain sockets, while PF_UNIX + // is the name that BSD used. We use the BSD name because it is more + // widely supported (e.g. Solaris 10). + const int domain = PF_UNIX; int socks[2]; const int result = ::socketpair(domain, type, protocol, socks); diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index dadc6b66f1b..83ecd5923da 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -298,17 +298,21 @@ namespace mongo { } SSL* SSLManager::connect(int fd) { SSL* ssl = _secure(fd); + ScopeGuard guard = MakeGuard(::SSL_free, ssl); int ret = _ssl_connect(ssl); if (ret != 1) _handleSSLError(SSL_get_error(ssl, ret)); + guard.Dismiss(); return ssl; } SSL* SSLManager::accept(int fd) { SSL* ssl = _secure(fd); + ScopeGuard guard = MakeGuard(::SSL_free, ssl); int ret = SSL_accept(ssl); if (ret != 1) _handleSSLError(SSL_get_error(ssl, ret)); + guard.Dismiss(); return ssl; } @@ -362,35 +366,32 @@ namespace mongo { // accepts the socket connection but fails to do the SSL handshake in a timely // manner. error() << "SSL error: " << code << ", possibly timed out during connect" << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); break; case SSL_ERROR_SYSCALL: if (code < 0) { error() << "socket error: " << errnoWithDescription() << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); } - error() << "could not negotiate SSL connection: EOF detected" << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); + else { + error() << "could not negotiate SSL connection: EOF detected" << endl; + } break; case SSL_ERROR_SSL: { int ret = ERR_get_error(); error() << _getSSLErrorMessage(ret) << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); break; } case SSL_ERROR_ZERO_RETURN: error() << "could not negotiate SSL connection: EOF detected" << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); break; default: error() << "unrecognized SSL error" << endl; - throw SocketException(SocketException::CONNECT_ERROR, ""); break; } + throw SocketException(SocketException::CONNECT_ERROR, ""); } } diff --git a/src/mongo/util/processinfo_sunos5.cpp b/src/mongo/util/processinfo_sunos5.cpp index 089edc24eed..22a0b5e6d7d 100644 --- a/src/mongo/util/processinfo_sunos5.cpp +++ b/src/mongo/util/processinfo_sunos5.cpp @@ -1,6 +1,4 @@ -// processinfo_none.cpp - -/* Copyright 2009 10gen Inc. +/* Copyright 2013 10gen Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,54 +13,162 @@ * limitations under the License. */ -#include "pch.h" -#include "processinfo.h" - +#include <boost/filesystem.hpp> +#include <fstream> #include <iostream> -using namespace std; +#include <malloc.h> +#include <procfs.h> +#include <stdio.h> +#include <string> +#include <sys/mman.h> +#include <sys/systeminfo.h> +#include <sys/utsname.h> +#include <unistd.h> +#include <vector> + +#include "mongo/util/file.h" +#include "mongo/util/mongoutils/str.h" +#include "mongo/util/processinfo.h" namespace mongo { - ProcessInfo::ProcessInfo( pid_t pid ) { + /** + * Read the first line from a file; return empty string on failure + */ + static string readLineFromFile(const char* fname) { + std::string fstr; + std::ifstream f(fname); + if (f.is_open()) { + std::getline(f, fstr); + } + return fstr; } - ProcessInfo::~ProcessInfo() { - } + struct ProcPsinfo { + ProcPsinfo() { + FILE* f = fopen("/proc/self/psinfo", "r"); + massert(16846, + mongoutils::str::stream() << "couldn't open \"/proc/self/psinfo\": " + << errnoWithDescription(), + f); + size_t num = fread(&psinfo, sizeof(psinfo), 1, f); + int err = errno; + fclose(f); + massert(16847, + mongoutils::str::stream() << "couldn't read from \"/proc/self/psinfo\": " + << errnoWithDescription(err), + num == 1); + } + psinfo_t psinfo; + }; + + struct ProcUsage { + ProcUsage() { + FILE* f = fopen("/proc/self/usage", "r"); + massert(16848, + mongoutils::str::stream() << "couldn't open \"/proc/self/usage\": " + << errnoWithDescription(), + f); + size_t num = fread(&prusage, sizeof(prusage), 1, f); + int err = errno; + fclose(f); + massert(16849, + mongoutils::str::stream() << "couldn't read from \"/proc/self/usage\": " + << errnoWithDescription(err), + num == 1); + } + prusage_t prusage; + }; + + ProcessInfo::ProcessInfo(pid_t pid) : _pid(pid) { } + ProcessInfo::~ProcessInfo() { } bool ProcessInfo::supported() { - return false; + return true; } int ProcessInfo::getVirtualMemorySize() { - return -1; + ProcPsinfo p; + return static_cast<int>(p.psinfo.pr_size / 1024); } int ProcessInfo::getResidentSize() { - return -1; - } - - bool ProcessInfo::checkNumaEnabled() { - return false; + ProcPsinfo p; + return static_cast<int>(p.psinfo.pr_rssize / 1024); } - bool ProcessInfo::blockCheckSupported() { - return false; + void ProcessInfo::getExtraInfo(BSONObjBuilder& info) { + ProcUsage p; + info.appendNumber("page_faults", static_cast<long long>(p.prusage.pr_majf)); } + /** + * Save a BSON obj representing the host system's details + */ void ProcessInfo::SystemInfo::collectSystemInfo() { + struct utsname unameData; + if (uname(&unameData) == -1) { + log() << "Unable to collect detailed system information: " << strerror(errno) << endl; + } + + char buf_64[32]; + char buf_native[32]; + if (sysinfo(SI_ARCHITECTURE_64, buf_64, sizeof(buf_64)) != -1 && + sysinfo(SI_ARCHITECTURE_NATIVE, buf_native, sizeof(buf_native)) != -1) { + addrSize = mongoutils::str::equals(buf_64, buf_native) ? 64 : 32; + } + else { + log() << "Unable to determine system architecture: " << strerror(errno) << endl; + } + + osType = unameData.sysname; + osName = mongoutils::str::ltrim(readLineFromFile("/etc/release")); + osVersion = unameData.version; + pageSize = static_cast<unsigned long long>(sysconf(_SC_PAGESIZE)); + memSize = pageSize * static_cast<unsigned long long>(sysconf(_SC_PHYS_PAGES)); + numCores = static_cast<unsigned>(sysconf(_SC_NPROCESSORS_CONF)); + cpuArch = unameData.machine; + hasNuma = checkNumaEnabled(); + + BSONObjBuilder bExtra; + bExtra.append("kernelVersion", unameData.release); + bExtra.append("pageSize", static_cast<long long>(pageSize)); + bExtra.append("numPages", static_cast<int>(sysconf(_SC_PHYS_PAGES))); + bExtra.append("maxOpenFiles", static_cast<int>(sysconf(_SC_OPEN_MAX))); + _extraStats = bExtra.obj(); + } + bool ProcessInfo::checkNumaEnabled() { + return false; } - void ProcessInfo::getExtraInfo( BSONObjBuilder& info ) { - + bool ProcessInfo::blockCheckSupported() { + return true; } bool ProcessInfo::blockInMemory(const void* start) { - verify(0); + char x = 0; + if (mincore(static_cast<char*>(const_cast<void*>(alignToStartOfPage(start))), + getPageSize(), + &x)) { + log() << "mincore failed: " << errnoWithDescription() << endl; + return 1; + } + return x & 0x1; } - bool ProcessInfo::pagesInMemory(const void* start, size_t numPages, vector<char>* out) { - verify(0); + bool ProcessInfo::pagesInMemory(const void* start, size_t numPages, std::vector<char>* out) { + out->resize(numPages); + if (mincore(static_cast<char*>(const_cast<void*>(alignToStartOfPage(start))), + numPages * getPageSize(), + &out->front())) { + log() << "mincore failed: " << errnoWithDescription() << endl; + return false; + } + for (size_t i = 0; i < numPages; ++i) { + (*out)[i] &= 0x1; + } + return true; } } diff --git a/src/mongo/util/signal_handlers.cpp b/src/mongo/util/signal_handlers.cpp index 52694900448..241d824b1a0 100644 --- a/src/mongo/util/signal_handlers.cpp +++ b/src/mongo/util/signal_handlers.cpp @@ -16,7 +16,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include "pch.h" +#include "mongo/pch.h" #include <cstdarg> #include <cstdio> @@ -26,20 +26,17 @@ #include <unistd.h> #endif -#ifdef MONGO_HAVE_EXECINFO_BACKTRACE -#include <execinfo.h> -#endif - -#include "log.h" -#include "signal_handlers.h" +#include "mongo/platform/backtrace.h" +#include "mongo/util/log.h" +#include "mongo/util/signal_handlers.h" namespace mongo { /* * WARNING: PLEASE READ BEFORE CHANGING THIS MODULE * - * All code in this module should be singal-friendly. Before adding any system - * call or other dependency, please make sure the latter still holds. + * All code in this module must be signal-friendly. Before adding any system + * call or other dependency, please make sure that this still holds. * */ @@ -84,7 +81,7 @@ namespace mongo { static void formattedBacktrace( int fd ) { -#ifdef MONGO_HAVE_EXECINFO_BACKTRACE +#if !defined(_WIN32) int numFrames; const int MAX_DEPTH = 20; diff --git a/src/mongo/util/stack_introspect.cpp b/src/mongo/util/stack_introspect.cpp index 27ebb0267cf..ce69b988bb7 100644 --- a/src/mongo/util/stack_introspect.cpp +++ b/src/mongo/util/stack_introspect.cpp @@ -18,22 +18,21 @@ #include "mongo/util/stack_introspect.h" +#if !defined(_WIN32) + #include <cstdlib> +#include <cxxabi.h> #include <iostream> -#include <string> #include <map> +#include <string> #include <vector> +#include "mongo/platform/backtrace.h" #include "mongo/util/concurrency/mutex.h" #include "mongo/util/text.h" using namespace std; -#ifdef MONGO_HAVE_EXECINFO_BACKTRACE - -#include <execinfo.h> -#include <cxxabi.h> - namespace mongo { namespace { @@ -106,7 +105,7 @@ namespace mongo { bool inConstructorChain( bool printOffending ){ void* b[maxBackTraceFrames]; - int size = ::backtrace( b, maxBackTraceFrames ); + int size = backtrace( b, maxBackTraceFrames ); char** strings = 0; @@ -122,7 +121,7 @@ namespace mongo { } if ( ! strings ) - strings = ::backtrace_symbols( b, size ); + strings = backtrace_symbols( b, size ); string symbol = strings[i]; @@ -176,5 +175,4 @@ namespace mongo { bool inConstructorChainSupported() { return false; } } -#endif // defined(MONGO_HAVE_EXECINFO_BACKTRACE) - +#endif // #if !defined(_WIN32) diff --git a/src/mongo/util/stacktrace.cpp b/src/mongo/util/stacktrace.cpp index ab405c9a717..c1348a00240 100644 --- a/src/mongo/util/stacktrace.cpp +++ b/src/mongo/util/stacktrace.cpp @@ -1,4 +1,17 @@ -// Copyright 2009. 10gen, Inc. +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include "mongo/util/stacktrace.h" @@ -11,53 +24,18 @@ #include "mongo/util/log.h" #ifdef _WIN32 -#include <sstream> -#include <stdio.h> #include <boost/filesystem/operations.hpp> #include <boost/smart_ptr/scoped_array.hpp> +#include <sstream> +#include <stdio.h> #include "mongo/platform/windows_basic.h" #include <DbgHelp.h> #include "mongo/util/assert_util.h" +#else +#include "mongo/platform/backtrace.h" #endif -#ifdef MONGO_HAVE_EXECINFO_BACKTRACE - -#include <execinfo.h> - -namespace mongo { - static const int maxBackTraceFrames = 20; - - /** - * Print a stack backtrace for the current thread to the specified ostream. - * - * @param os ostream& to receive printed stack backtrace - */ - void printStackTrace( std::ostream& os ) { - - void *b[maxBackTraceFrames]; - - int size = ::backtrace( b, maxBackTraceFrames ); - for ( int i = 0; i < size; i++ ) - os << std::hex << b[i] << std::dec << ' '; - os << std::endl; - - char **strings; - - strings = ::backtrace_symbols( b, size ); - if (strings == NULL) { - const int err = errno; - os << "Unable to collect backtrace symbols (" << errnoWithDescription(err) << ")" - << std::endl; - return; - } - for ( int i = 0; i < size; i++ ) - os << ' ' << strings[i] << '\n'; - os.flush(); - ::free( strings ); - } -} - -#elif defined(_WIN32) +#if defined(_WIN32) namespace mongo { @@ -306,12 +284,43 @@ namespace mongo { } } - #else namespace mongo { - void printStackTrace( std::ostream &os ) {} + static const int maxBackTraceFrames = 20; + + /** + * Print a stack backtrace for the current thread to the specified ostream. + * + * @param os ostream& to receive printed stack backtrace + */ + void printStackTrace( std::ostream& os ) { + + void* addresses[maxBackTraceFrames]; + + int addressCount = backtrace(addresses, maxBackTraceFrames); + if (addressCount == 0) { + const int err = errno; + os << "Unable to collect backtrace addresses (" << errnoWithDescription(err) << ")" + << std::endl; + return; + } + for (int i = 0; i < addressCount; i++) + os << std::hex << addresses[i] << std::dec << ' '; + os << std::endl; + + char** backtraceStrings = backtrace_symbols(addresses, addressCount); + if (backtraceStrings == NULL) { + const int err = errno; + os << "Unable to collect backtrace symbols (" << errnoWithDescription(err) << ")" + << std::endl; + return; + } + for (int i = 0; i < addressCount; i++) + os << ' ' << backtraceStrings[i] << '\n'; + os.flush(); + free(backtraceStrings); + } } #endif - diff --git a/src/mongo/util/stacktrace.h b/src/mongo/util/stacktrace.h index 1a5d62dcdf1..f1cd6279107 100644 --- a/src/mongo/util/stacktrace.h +++ b/src/mongo/util/stacktrace.h @@ -1,4 +1,17 @@ -// Copyright 2009. 10gen, Inc. +/* Copyright 2009 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Tools for working with in-process stack traces. diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp index 0756bca0901..25fe64b5811 100644 --- a/src/mongo/util/version.cpp +++ b/src/mongo/util/version.cpp @@ -47,7 +47,7 @@ namespace mongo { * 1.2.3-rc4-pre- * If you really need to do something else you'll need to fix _versionArray() */ - const char versionString[] = "2.4.5"; + const char versionString[] = "2.4.6"; // See unit test for example outputs BSONArray toVersionArray(const char* version){ diff --git a/src/third_party/s2/util/math/SConscript b/src/third_party/s2/util/math/SConscript index 8ad3ee125f1..af51445de1d 100755 --- a/src/third_party/s2/util/math/SConscript +++ b/src/third_party/s2/util/math/SConscript @@ -7,6 +7,10 @@ env = env.Clone() env.Append(CCFLAGS=['-Isrc/third_party/s2']) env.Append(CCFLAGS=['-Isrc/third_party/gflags-2.0/src']) +if solaris: + # Enables declaration of isinf() on Solaris + env.Append(CPPDEFINES=['__C99FEATURES__']) + env.StaticLibrary("math", [ "mathutil.cc", # "mathlimits.cc", |