diff options
Diffstat (limited to 'src/mongo/tools')
53 files changed, 4959 insertions, 2151 deletions
diff --git a/src/mongo/tools/bridge.cpp b/src/mongo/tools/bridge.cpp index 4d13b64a938..422237c5b00 100644 --- a/src/mongo/tools/bridge.cpp +++ b/src/mongo/tools/bridge.cpp @@ -1,5 +1,3 @@ -// bridge.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,35 +12,63 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include <boost/thread.hpp> #include "mongo/base/initializer.h" +#include "mongo/client/dbclientinterface.h" #include "mongo/db/dbmessage.h" +#include "mongo/tools/mongobridge_options.h" #include "mongo/util/net/listen.h" #include "mongo/util/net/message.h" #include "mongo/util/stacktrace.h" +#include "mongo/util/text.h" +#include "mongo/util/timer.h" using namespace mongo; using namespace std; -int port = 0; -int delay = 0; -string destUri; void cleanup( int sig ); class Forwarder { public: Forwarder( MessagingPort &mp ) : mp_( mp ) { } + void operator()() const { DBClientConnection dest; string errmsg; - while( !dest.connect( destUri, errmsg ) ) - sleepmillis( 500 ); + + Timer connectTimer; + while (!dest.connect(mongoBridgeGlobalParams.destUri, errmsg)) { + // If we can't connect for the configured timeout, give up + // + if (connectTimer.seconds() >= mongoBridgeGlobalParams.connectTimeoutSec) { + cout << "Unable to establish connection from " << mp_.psock->remoteString() + << " to " << mongoBridgeGlobalParams.destUri + << " after " << connectTimer.seconds() << " seconds. Giving up." << endl; + mp_.shutdown(); + return; + } + + sleepmillis(500); + } + Message m; while( 1 ) { try { @@ -52,7 +78,7 @@ public: mp_.shutdown(); break; } - sleepmillis( delay ); + sleepmillis(mongoBridgeGlobalParams.delay); int oldId = m.header()->id; if ( m.operation() == dbQuery || m.operation() == dbMsg || m.operation() == dbGetMore ) { @@ -118,8 +144,7 @@ void cleanup( int sig ) { } #if !defined(_WIN32) void myterminate() { - rawOut( "bridge terminate() called, printing stack:" ); - printStackTrace(); + printStackTrace(severe().stream() << "bridge terminate() called, printing stack:\n"); ::abort(); } @@ -137,47 +162,34 @@ void setupSignals() { inline void setupSignals() {} #endif -void helpExit() { - cout << "usage mongobridge --port <port> --dest <destUri> [ --delay <ms> ]" << endl; - cout << " port: port to listen for mongo messages" << endl; - cout << " destUri: uri of remote mongod instance" << endl; - cout << " ms: transfer delay in milliseconds (default = 0)" << endl; - ::_exit( -1 ); -} - -void check( bool b ) { - if ( !b ) - helpExit(); -} - -int main( int argc, char **argv, char** envp ) { +int toolMain( int argc, char **argv, char** envp ) { mongo::runGlobalInitializersOrDie(argc, argv, envp); static StaticObserver staticObserver; setupSignals(); - check( argc == 5 || argc == 7 ); - - for( int i = 1; i < argc; ++i ) { - check( i % 2 != 0 ); - if ( strcmp( argv[ i ], "--port" ) == 0 ) { - port = strtol( argv[ ++i ], 0, 10 ); - } - else if ( strcmp( argv[ i ], "--dest" ) == 0 ) { - destUri = argv[ ++i ]; - } - else if ( strcmp( argv[ i ], "--delay" ) == 0 ) { - delay = strtol( argv[ ++i ], 0, 10 ); - } - else { - check( false ); - } - } - check( port != 0 && !destUri.empty() ); - - listener.reset( new MyListener( port ) ); + listener.reset(new MyListener(mongoBridgeGlobalParams.port)); + listener->setupSockets(); listener->initAndListen(); return 0; } + +#if defined(_WIN32) +// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters +// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The +// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent +// and makes them available through the argv() and envp() members. This enables toolMain() +// to process UTF-8 encoded arguments and environment variables without regard to platform. +int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { + WindowsCommandLine wcl(argc, argvW, envpW); + int exitCode = toolMain(argc, wcl.argv(), wcl.envp()); + ::_exit(exitCode); +} +#else +int main(int argc, char* argv[], char** envp) { + int exitCode = toolMain(argc, argv, envp); + ::_exit(exitCode); +} +#endif diff --git a/src/mongo/tools/bsondump.cpp b/src/mongo/tools/bsondump.cpp index 461737af739..ef0c02be5b3 100644 --- a/src/mongo/tools/bsondump.cpp +++ b/src/mongo/tools/bsondump.cpp @@ -1,5 +1,3 @@ -// restore.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,22 +12,32 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "../pch.h" -#include "mongo/base/initializer.h" -#include "mongo/client/dbclientcursor.h" -#include "../util/mmap.h" -#include "../util/text.h" -#include "tool.h" - -#include <boost/program_options.hpp> +#include "mongo/pch.h" #include <fcntl.h> -using namespace mongo; +#include "mongo/client/dbclientcursor.h" +#include "mongo/tools/bsondump_options.h" +#include "mongo/tools/tool.h" +#include "mongo/util/mmap.h" +#include "mongo/util/options_parser/option_section.h" +#include "mongo/util/text.h" -namespace po = boost::program_options; +using namespace mongo; class BSONDump : public BSONTool { @@ -37,38 +45,27 @@ class BSONDump : public BSONTool { public: - BSONDump() : BSONTool( "bsondump", NONE ) { - add_options() - ("type" , po::value<string>()->default_value("json") , "type of output: json,debug" ) - ; - add_hidden_options() - ("file" , po::value<string>() , ".bson file" ) - ; - addPositionArg( "file" , 1 ); - _noconnection = true; - } + BSONDump() : BSONTool() { } - virtual void printExtraHelp(ostream& out) { - out << "Display BSON objects in a data file.\n" << endl; - out << "usage: " << _name << " [options] <bson filename>" << endl; + virtual void printHelp(ostream& out) { + printBSONDumpHelp(&out); } virtual int doRun() { { - string t = getParam( "type" ); - if ( t == "json" ) + if (bsonDumpGlobalParams.type == "json") _type = JSON; - else if ( t == "debug" ) + else if (bsonDumpGlobalParams.type == "debug") _type = DEBUG; else { - cerr << "bad type: " << t << endl; + cerr << "bad type: " << bsonDumpGlobalParams.type << endl; return 1; } } - boost::filesystem::path root = getParam( "file" ); + boost::filesystem::path root = bsonDumpGlobalParams.file; if ( root == "" ) { - printExtraHelp(cout); + printBSONDumpHelp(&std::cout); return 1; } @@ -87,17 +84,31 @@ public: try { cout << prefix << "--- new object ---\n"; cout << prefix << "\t size : " << o.objsize() << "\n"; + + // Note: this will recursively check each level of the bson and will also be called by + // this function at each level. While inefficient, it shouldn't effect correctness. + const Status status = validateBSON(o.objdata(), o.objsize()); + if (!status.isOK()) { + cout << prefix << "\t OBJECT IS INVALID: " << status.reason() << '\n' + << prefix << "\t attempting to print as much as possible" << endl; + } + BSONObjIterator i(o); while ( i.more() ) { - BSONElement e = i.next(); - cout << prefix << "\t\t " << e.fieldName() << "\n" << prefix << "\t\t\t type:" << setw(3) << e.type() << " size: " << e.size() << endl; + // This call verifies it is safe to call size() and fieldName() but doesn't check + // whether the element extends past the end of the object. That is done below. + BSONElement e = i.next(/*checkEnd=*/true); + + cout << prefix << "\t\t " << e.fieldName() << "\n" + << prefix << "\t\t\t type:" << setw(3) << e.type() << " size: " << e.size() + << endl; + if ( ( read + e.size() ) > o.objsize() ) { cout << prefix << " SIZE DOES NOT WORK" << endl; return false; } read += e.size(); try { - e.validate(); if ( e.isABSONObj() ) { if ( ! debug( e.Obj() , depth + 1 ) ) { //return false; @@ -111,10 +122,9 @@ public: else if ( e.type() == String && ! isValidUTF8( e.valuestr() ) ) { cout << prefix << "\t\t\t" << "bad utf8 String!" << endl; } - else if ( logLevel > 0 ) { + else if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)) ) { cout << prefix << "\t\t\t" << e << endl; } - } catch ( std::exception& e ) { cout << prefix << "\t\t\t bad value: " << e.what() << endl; @@ -142,8 +152,4 @@ public: } }; -int main( int argc , char ** argv, char **envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - BSONDump dump; - return dump.main( argc , argv ); -} +REGISTER_MONGO_TOOL(BSONDump); diff --git a/src/mongo/tools/bsondump_options.cpp b/src/mongo/tools/bsondump_options.cpp new file mode 100644 index 00000000000..ebbd0e877e5 --- /dev/null +++ b/src/mongo/tools/bsondump_options.cpp @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/bsondump_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + BSONDumpGlobalParams bsonDumpGlobalParams; + + Status addBSONDumpOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addBSONToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("type", "type", moe::String, "type of output: json,debug") + .setDefault(moe::Value(std::string("json"))); + + options->addOptionChaining("file", "file", moe::String, "path to BSON file to dump") + .hidden() + .setSources(moe::SourceCommandLine) + .positional(1, 1); + + + return Status::OK(); + } + + void printBSONDumpHelp(std::ostream* out) { + *out << "Display BSON objects in a data file.\n" << std::endl; + *out << "usage: bsondump [options] <bson filename>" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationBSONDumpOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printBSONDumpHelp(&std::cout); + return false; + } + return true; + } + + Status storeBSONDumpOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + ret = storeBSONToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + // BSONDump never has a db connection + toolGlobalParams.noconnection = true; + + bsonDumpGlobalParams.type = getParam("type"); + bsonDumpGlobalParams.file = getParam("file"); + + // Make the default db "" if it was not explicitly set + if (!params.count("db")) { + toolGlobalParams.db = ""; + } + + // bsondump always outputs data to stdout, so we can't send messages there + toolGlobalParams.canUseStdout = false; + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/bsondump_options.h b/src/mongo/tools/bsondump_options.h new file mode 100644 index 00000000000..b2e71a1a095 --- /dev/null +++ b/src/mongo/tools/bsondump_options.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct BSONDumpGlobalParams { + std::string type; + std::string file; + }; + + extern BSONDumpGlobalParams bsonDumpGlobalParams; + + Status addBSONDumpOptions(moe::OptionSection* options); + + void printBSONDumpHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationBSONDumpOptions(const moe::Environment& params); + + Status storeBSONDumpOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/bsondump_options_init.cpp b/src/mongo/tools/bsondump_options_init.cpp new file mode 100644 index 00000000000..59d50d7e7cc --- /dev/null +++ b/src/mongo/tools/bsondump_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/bsondump_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(BSONDumpOptions)(InitializerContext* context) { + return addBSONDumpOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(BSONDumpOptions)(InitializerContext* context) { + if (!handlePreValidationBSONDumpOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(BSONDumpOptions)(InitializerContext* context) { + Status ret = storeBSONDumpOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/docgenerator.cpp b/src/mongo/tools/docgenerator.cpp deleted file mode 100644 index ba614b53d8f..00000000000 --- a/src/mongo/tools/docgenerator.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/** @file docgenerator.cpp - -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -#include "mongo/tools/docgenerator.h" - -#include "mongo/util/assert_util.h" -#include "mongo/util/md5.hpp" -#include "mongo/util/mongoutils/str.h" - -namespace mongo { - - void DocumentGenerator::init( BSONObj& args ) { - uassert( 16363, "_id is not a number", args["_id"].isNumber() ); - config.id = args["_id"].numberLong(); - - uassert( 16364, "blob is not a string", (args["blob"].type() == String) ); - config.blob = args["blob"].String(); - - uassert( 16365, "nestedDoc is not an object", (args["nestedDoc"].type() == Object) ); - config.nestedDoc = args["nestedDoc"].embeddedObject(); - - uassert( 16366, "list is not an array", args["list"].type() == Array ); - BSONObj list = args["list"].embeddedObject(); - for( int i = 0; i < 10; i++ ) { - uassert( 16367, "list member is not a string", list[i].type() == String ); - config.list.push_back( list[i].String() ); - } - - uassert( 16368, "counter is not a number", args["counter"].isNumber() ); - config.counter = args["counter"].numberLong(); - } - - - BSONObj DocumentGenerator::createDocument() { - BSONObjBuilder doc; - doc.append( "_id", config.id ); - config.id++; - doc.append( "blob", config.blob ); - doc.append( "nestedDoc", config.nestedDoc ); - doc.append( "list", config.list ); - doc.append( "counter", config.counter ); - config.counter++; - return doc.obj(); - } -} - diff --git a/src/mongo/tools/docgenerator.h b/src/mongo/tools/docgenerator.h deleted file mode 100644 index 8ca2233947c..00000000000 --- a/src/mongo/tools/docgenerator.h +++ /dev/null @@ -1,80 +0,0 @@ -/** @file docgenerator.h - -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -/** -* This is a simple document generator. It generates documents of the format -* { _id: xxx, blob: "xxx", nestedDoc: {xxx}, list: [xxx,yyy,zzz], counter: xxx } -*/ - -#pragma once - -#include <string> -#include <vector> - -#include "mongo/db/jsobj.h" - -namespace mongo { - - struct DocGeneratorOptions { - DocGeneratorOptions() : - hostname(""), - dbSize( 0.0 ), - numdbs( 0 ), - prefix("") - { } - std::string hostname; - double dbSize; - int numdbs; - std::string prefix; - }; - - struct DocConfig { - DocConfig() : - id( 0 ), - blob( "" ), - counter( 0 ) - { } - long long id; - std::string blob; - BSONObj nestedDoc; - std::vector<std::string> list; - long long counter; - }; - - // Creates a documentGenerator that can be used to create sample documents - class DocumentGenerator { - public: - DocumentGenerator( ) { } - ~DocumentGenerator() { } - - void init( BSONObj& args ); - BSONObj createDocument(); - - // caller is responsible for managing this raw pointer - static DocumentGenerator* makeDocumentGenerator( BSONObj args ) { - DocumentGenerator* runner = new DocumentGenerator() ; - runner->init( args ); - return runner; - } - DocConfig config; - }; - -} // end namespace - - - - diff --git a/src/mongo/tools/docgeneratormain.cpp b/src/mongo/tools/docgeneratormain.cpp deleted file mode 100644 index 2cae11fa2e4..00000000000 --- a/src/mongo/tools/docgeneratormain.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/** @file docgeneratormain.cpp - -* Copyright (C) 2012 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -/** -* Uses the DocumentGenerator class to populate databases with documents of the format -* { _id: xxx, blob: "xxx", nestedDoc: {xxx}, list: [xxx,yyy,zzz], counter: xxx } -* -*/ - -#include <iostream> - -#include <boost/program_options.hpp> - -#include "mongo/base/initializer.h" -#include "mongo/util/assert_util.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/tools/docgenerator.h" -#include "mongo/db/jsobj.h" -#include "mongo/util/mongoutils/str.h" - -using namespace mongo; -using std::exception; -using std::cout; - -namespace po = boost::program_options; - -//------------- Define globals and constants-------- - -// global options object -DocGeneratorOptions globalDocGenOption; - -//----------- End globals and constants------------ - -int parseCmdLineOptions( int argc, char **argv ) { - - try { - po::options_description general_options( "General options" ); - general_options.add_options() - ( "help", "produce help message" ) - ( "hostname,H", po::value<string>() , "ip address of the host where mongod is running " ) - ( "dbSize", po::value<double>(), "size of each database in megabytes(MB)" ) - ( "numdbs", po::value<int>(), "number of databases you want" ) - ( "prefix", po::value<string>(), - "prefix the resultant db name where the documents " - "will be saved. DB's will be named prefix0, prefix1 etc." ) - ; - - po::variables_map params; - po::store( po::parse_command_line(argc, argv, general_options), params ); - po::notify( params ); - - // Parse the values if supplied by the user. No data sanity check is performed - // here so meaningless values may result in unexpected behavior. - // TODO: Perform data sanity check - if ( params.count("help") ) { - cout << general_options << "\n"; - return 1; - } - if ( params.count("hostname") ) { - globalDocGenOption.hostname = params["hostname"].as<string>(); - } - if ( params.count("dbSize") ) { - globalDocGenOption.dbSize = params["dbSize"].as<double>(); - } - if ( params.count("numdbs") ) { - globalDocGenOption.numdbs = params["numdbs"].as<int>(); - } - if ( params.count("prefix") ) { - globalDocGenOption.prefix = params["prefix"].as<string>(); - } - } - catch(exception& e) { - cerr << "error: " << e.what() << "\n"; - return 1; - } - return 0; -} - - -int main( int argc, char* argv[], char* envp[] ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - if( parseCmdLineOptions( argc, argv) ) - return 1; - - BSONObj nestedDoc = BSON("Firstname" << "David" << - "Lastname" << "Smith" << - "Address" << BSON( "Street" << "5th Av" << - "City" << "New York" ) - ); - std::vector<std::string> list; - list.push_back("mongo new york city"); - list.push_back("mongo rome"); - list.push_back("mongo dublin"); - list.push_back("mongo seoul"); - list.push_back("mongo barcelona"); - list.push_back("mongo madrid"); - list.push_back("mongo chicago"); - list.push_back("mongo amsterdam"); - list.push_back("mongo delhi"); - list.push_back("mongo beijing"); - - BSONObj args = BSONObjBuilder() - .append( "_id", 0 ) - .append( "blob", "MongoDB is an open source document-oriented database " - "system designed with scalability and developer." ) - .append( "nestedDoc", nestedDoc ) - .append( "list", list ) - .append( "counter", 0 ).obj(); - - const int numDocsPerDB = - static_cast<int>( globalDocGenOption.dbSize * 1024 * 1024 / args.objsize() ); - cout << "numDocsPerDB:" << numDocsPerDB << endl; - try { - DBClientConnection conn; - conn.connect( globalDocGenOption.hostname ); - cout << "successfully connected to the host" << endl; - for( int i=0; i < globalDocGenOption.numdbs; ++i ) { - scoped_ptr<DocumentGenerator> docGen( DocumentGenerator::makeDocumentGenerator(args) ); - cout << "populating database " << globalDocGenOption.prefix << i << endl; - long long j = 0; - string ns = mongoutils::str::stream() << globalDocGenOption.prefix << i << ".sampledata"; - while( j != numDocsPerDB ) { - BSONObj doc = docGen->createDocument(); - conn.insert( ns, doc ); - ++j; - } - BSONObj blobIndex = BSON("blob" << 1); - conn.ensureIndex(ns, blobIndex); - BSONObj listIndex = BSON("list" << 1); - conn.ensureIndex(ns, listIndex); - } - } catch( DBException &e ) { - cout << "caught " << e.what() << endl; - } - return 0; -} diff --git a/src/mongo/tools/dump.cpp b/src/mongo/tools/dump.cpp index 93d0446c3cd..7f8c6b42f63 100644 --- a/src/mongo/tools/dump.cpp +++ b/src/mongo/tools/dump.cpp @@ -1,5 +1,3 @@ -// dump.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,27 +12,42 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ #include "mongo/pch.h" -#include <fcntl.h> -#include <map> -#include <fstream> - #include <boost/filesystem/operations.hpp> #include <boost/filesystem/convenience.hpp> +#include <fcntl.h> +#include <fstream> +#include <map> -#include "mongo/base/initializer.h" +#include "mongo/base/status.h" +#include "mongo/client/auth_helpers.h" #include "mongo/client/dbclientcursor.h" +#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/db.h" -#include "mongo/db/namespacestring.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/catalog/collection.h" +#include "mongo/tools/mongodump_options.h" #include "mongo/tools/tool.h" +#include "mongo/util/options_parser/option_section.h" +#include "mongo/util/mongoutils/str.h" using namespace mongo; -namespace po = boost::program_options; - class Dump : public Tool { class FilePtr : boost::noncopyable { public: @@ -45,27 +58,10 @@ class Dump : public Tool { FILE* _f; }; public: - Dump() : Tool( "dump" , ALL , "" , "" , true ) { - add_options() - ("out,o", po::value<string>()->default_value("dump"), "output directory or \"-\" for stdout") - ("query,q", po::value<string>() , "json query" ) - ("oplog", "Use oplog for point-in-time snapshotting" ) - ("repair", "try to recover a crashed database" ) - ("forceTableScan", "force a table scan (do not use $snapshot)" ) - ; - } - - virtual void preSetup() { - string out = getParam("out"); - if ( out == "-" ) { - // write output to standard error to avoid mangling output - // must happen early to avoid sending junk to stdout - useStandardOutput(false); - } - } + Dump() : Tool() { } - virtual void printExtraHelp(ostream& out) { - out << "Export MongoDB data to BSON files.\n" << endl; + virtual void printHelp(ostream& out) { + printMongoDumpHelp(&out); } // This is a functor that writes a BSONObj to a file @@ -93,13 +89,12 @@ public: ProgressMeter* _m; }; - void doCollection( const string coll , FILE* out , ProgressMeter *m ) { - Query q = _query; - + void doCollection( const string coll , Query q, FILE* out , ProgressMeter *m, + bool usingMongos ) { int queryOptions = QueryOption_SlaveOk | QueryOption_NoCursorTimeout; - if (startsWith(coll.c_str(), "local.oplog.")) + if (startsWith(coll.c_str(), "local.oplog.") && q.obj.hasField("ts")) queryOptions |= QueryOption_OplogReplay; - else if ( _query.isEmpty() && !hasParam("dbpath") && !hasParam("forceTableScan") ) { + else if (mongoDumpGlobalParams.snapShotQuery) { q.snapshot(); } @@ -107,7 +102,7 @@ public: Writer writer(out, m); // use low-latency "exhaust" mode if going over the network - if (!_usingMongos && typeid(connBase) == typeid(DBClientConnection&)) { + if (!usingMongos && typeid(connBase) == typeid(DBClientConnection&)) { DBClientConnection& conn = static_cast<DBClientConnection&>(connBase); boost::function<void(const BSONObj&)> castedWriter(writer); // needed for overload resolution conn.query( castedWriter, coll.c_str() , q , NULL, queryOptions | QueryOption_Exhaust); @@ -121,24 +116,25 @@ public: } } - void writeCollectionFile( const string coll , boost::filesystem::path outputFile ) { - log() << "\t" << coll << " to " << outputFile.string() << endl; + void writeCollectionFile( const string coll , Query q, boost::filesystem::path outputFile, + bool usingMongos ) { + toolInfoLog() << "\t" << coll << " to " << outputFile.string() << std::endl; FilePtr f (fopen(outputFile.string().c_str(), "wb")); uassert(10262, errnoWithPrefix("couldn't open file"), f); ProgressMeter m(conn(true).count(coll.c_str(), BSONObj(), QueryOption_SlaveOk)); m.setName("Collection File Writing Progress"); - m.setUnits("objects"); + m.setUnits("documents"); - doCollection(coll, f, &m); + doCollection(coll, q, f, &m, usingMongos); - log() << "\t\t " << m.done() << " objects" << endl; + toolInfoLog() << "\t\t " << m.done() << " documents" << std::endl; } void writeMetadataFile( const string coll, boost::filesystem::path outputFile, map<string, BSONObj> options, multimap<string, BSONObj> indexes ) { - log() << "\tMetadata for " << coll << " to " << outputFile.string() << endl; + toolInfoLog() << "\tMetadata for " << coll << " to " << outputFile.string() << std::endl; bool hasOptions = options.count(coll) > 0; bool hasIndexes = indexes.count(coll) > 0; @@ -170,13 +166,18 @@ public: - void writeCollectionStdout( const string coll ) { - doCollection(coll, stdout, NULL); + void writeCollectionStdout( const string coll, const BSONObj& dumpQuery, bool usingMongos ) { + doCollection(coll, dumpQuery, stdout, NULL, usingMongos); } - void go( const string db , const boost::filesystem::path outdir ) { - log() << "DATABASE: " << db << "\t to \t" << outdir.string() << endl; - + void go(const string& db, + const string& coll, + const Query& query, + const boost::filesystem::path& outdir, + const string& outFilename, + bool usingMongos) { + // Can only provide outFilename if db and coll are provided + fassert(17368, outFilename.empty() || (!coll.empty() && !db.empty())); boost::filesystem::create_directories( outdir ); map <string, BSONObj> collectionOptions; @@ -202,81 +203,74 @@ public: } // skip namespaces with $ in them only if we don't specify a collection to dump - if ( _coll == "" && name.find( ".$" ) != string::npos ) { - LOG(1) << "\tskipping collection: " << name << endl; + if (coll == "" && name.find(".$") != string::npos) { + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << "\tskipping collection: " << name << std::endl; + } continue; } const string filename = name.substr( db.size() + 1 ); //if a particular collections is specified, and it's not this one, skip it - if ( _coll != "" && db + "." + _coll != name && _coll != name ) + if (coll != "" && db + "." + coll != name && coll != name) { continue; + } // raise error before writing collection with non-permitted filename chars in the name size_t hasBadChars = name.find_first_of("/\0"); if (hasBadChars != string::npos){ - error() << "Cannot dump " << name << ". Collection has '/' or null in the collection name." << endl; - continue; + toolError() << "Cannot dump " << name + << ". Collection has '/' or null in the collection name." << std::endl; + continue; } - if (NamespaceString(name).coll == "system.indexes") { + if (nsToCollectionSubstring(name) == "system.indexes") { // Create system.indexes.bson for compatibility with pre 2.2 mongorestore const string filename = name.substr( db.size() + 1 ); - writeCollectionFile( name.c_str() , outdir / ( filename + ".bson" ) ); + writeCollectionFile( name.c_str(), query, outdir / ( filename + ".bson" ), + usingMongos ); // Don't dump indexes as *.metadata.json continue; } - - if ( _coll != "" && db + "." + _coll != name && _coll != name ) - continue; - + + if (nsToCollectionSubstring(name) == "system.users" && + !mongoDumpGlobalParams.dumpUsersAndRoles) { + continue; + } + collections.push_back(name); } for (vector<string>::iterator it = collections.begin(); it != collections.end(); ++it) { string name = *it; - const string filename = name.substr( db.size() + 1 ); - writeCollectionFile( name , outdir / ( filename + ".bson" ) ); + const string filename = outFilename != "" ? outFilename : name.substr( db.size() + 1 ); + writeCollectionFile( name , query, outdir / ( filename + ".bson" ), usingMongos ); writeMetadataFile( name, outdir / (filename + ".metadata.json"), collectionOptions, indexes); } } int repair() { - if ( ! hasParam( "dbpath" ) ){ - log() << "repair mode only works with --dbpath" << endl; - return -1; - } - - if ( ! hasParam( "db" ) ){ - log() << "repair mode only works on 1 db at a time right now" << endl; - return -1; - } - - string dbname = getParam( "db" ); - log() << "going to try and recover data from: " << dbname << endl; - - return _repair( dbname ); + toolInfoLog() << "going to try and recover data from: " << toolGlobalParams.db << std::endl; + return _repair(toolGlobalParams.db); } DiskLoc _repairExtent( Database* db , string ns, bool forward , DiskLoc eLoc , Writer& w ){ LogIndentLevel lil; if ( eLoc.getOfs() <= 0 ){ - error() << "invalid extent ofs: " << eLoc.getOfs() << endl; + toolError() << "invalid extent ofs: " << eLoc.getOfs() << std::endl; return DiskLoc(); } - - - MongoDataFile * mdf = db->getFile( eLoc.a() ); - Extent * e = mdf->debug_getExtent( eLoc ); + Extent * e = db->getExtentManager().getExtent( eLoc, false ); if ( ! e->isOk() ){ - warning() << "Extent not ok magic: " << e->magic << " going to try to continue" << endl; + toolError() << "Extent not ok magic: " << e->magic << " going to try to continue" + << std::endl; } - - log() << "length:" << e->length << endl; + + toolInfoLog() << "length:" << e->length << std::endl; LogIndentLevel lil2; @@ -286,34 +280,38 @@ public: while ( ! loc.isNull() ){ if ( ! seen.insert( loc ).second ) { - error() << "infinite loop in extent, seen: " << loc << " before" << endl; + toolError() << "infinite loop in extent, seen: " << loc << " before" << std::endl; break; } if ( loc.getOfs() <= 0 ){ - error() << "offset is 0 for record which should be impossible" << endl; + toolError() << "offset is 0 for record which should be impossible" << std::endl; break; } - LOG(1) << loc << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << loc << std::endl; + } Record* rec = loc.rec(); BSONObj obj; try { obj = loc.obj(); verify( obj.valid() ); - LOG(1) << obj << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << obj << std::endl; + } w( obj ); } catch ( std::exception& e ) { - log() << "found invalid document @ " << loc << " " << e.what() << endl; + toolError() << "found invalid document @ " << loc << " " << e.what() << std::endl; if ( ! obj.isEmpty() ) { try { BSONElement e = obj.firstElement(); stringstream ss; ss << "first element: " << e; - log() << ss.str(); + toolError() << ss.str() << std::endl; } catch ( std::exception& ) { - log() << "unable to log invalid document @ " << loc << endl; + toolError() << "unable to log invalid document @ " << loc << std::endl; } } } @@ -326,76 +324,81 @@ public: break; } } - log() << "wrote " << seen.size() << " documents" << endl; + toolInfoLog() << "wrote " << seen.size() << " documents" << std::endl; return forward ? e->xnext : e->xprev; } + /* + * NOTE: The "outfile" parameter passed in should actually represent a directory, but it is + * called "outfile" because we append the filename and use it as our output file. + */ void _repair( Database* db , string ns , boost::filesystem::path outfile ){ - NamespaceDetails * nsd = nsdetails( ns ); - log() << "nrecords: " << nsd->stats.nrecords - << " datasize: " << nsd->stats.datasize - << " firstExtent: " << nsd->firstExtent - << endl; - - if ( nsd->firstExtent.isNull() ){ - log() << " ERROR fisrtExtent is null" << endl; + Collection* collection = db->getCollection( ns ); + const NamespaceDetails * nsd = collection->details(); + toolInfoLog() << "nrecords: " << nsd->numRecords() + << " datasize: " << nsd->dataSize() + << " firstExtent: " << nsd->firstExtent() + << std::endl; + + if ( nsd->firstExtent().isNull() ){ + toolError() << " ERROR fisrtExtent is null" << std::endl; return; } - - if ( ! nsd->firstExtent.isValid() ){ - log() << " ERROR fisrtExtent is not valid" << endl; + + if ( ! nsd->firstExtent().isValid() ){ + toolError() << " ERROR fisrtExtent is not valid" << std::endl; return; } outfile /= ( ns.substr( ns.find( "." ) + 1 ) + ".bson" ); - log() << "writing to: " << outfile.string() << endl; - + toolInfoLog() << "writing to: " << outfile.string() << std::endl; + FilePtr f (fopen(outfile.string().c_str(), "wb")); // init with double the docs count because we make two passes - ProgressMeter m( nsd->stats.nrecords * 2 ); + ProgressMeter m( nsd->numRecords() * 2 ); m.setName("Repair Progress"); - m.setUnits("objects"); - + m.setUnits("documents"); + Writer w( f , &m ); try { - log() << "forward extent pass" << endl; + toolInfoLog() << "forward extent pass" << std::endl; LogIndentLevel lil; - DiskLoc eLoc = nsd->firstExtent; + DiskLoc eLoc = nsd->firstExtent(); while ( ! eLoc.isNull() ){ - log() << "extent loc: " << eLoc << endl; + toolInfoLog() << "extent loc: " << eLoc << std::endl; eLoc = _repairExtent( db , ns , true , eLoc , w ); } } catch ( DBException& e ){ - error() << "forward extent pass failed:" << e.toString() << endl; + toolError() << "forward extent pass failed:" << e.toString() << std::endl; } try { - log() << "backwards extent pass" << endl; + toolInfoLog() << "backwards extent pass" << std::endl; LogIndentLevel lil; - DiskLoc eLoc = nsd->lastExtent; + DiskLoc eLoc = nsd->lastExtent(); while ( ! eLoc.isNull() ){ - log() << "extent loc: " << eLoc << endl; + toolInfoLog() << "extent loc: " << eLoc << std::endl; eLoc = _repairExtent( db , ns , false , eLoc , w ); } } catch ( DBException& e ){ - error() << "ERROR: backwards extent pass failed:" << e.toString() << endl; + toolError() << "ERROR: backwards extent pass failed:" << e.toString() << std::endl; } - log() << "\t\t " << m.done() << " objects" << endl; + toolInfoLog() << "\t\t " << m.done() << " documents" << std::endl; } int _repair( string dbname ) { Client::WriteContext cx( dbname ); Database * db = cx.ctx().db(); - + list<string> namespaces; - db->namespaceIndex.getNamespaces( namespaces ); - - boost::filesystem::path root = getParam( "out" ); + db->namespaceIndex().getNamespaces( namespaces ); + + boost::filesystem::path root = mongoDumpGlobalParams.outputDirectory; root /= dbname; boost::filesystem::create_directories( root ); @@ -409,17 +412,19 @@ public: if ( str::contains( ns , ".tmp.mr." ) ) continue; - if ( _coll != "" && ! str::endsWith( ns , _coll ) ) + if (toolGlobalParams.coll != "" && + !str::endsWith(ns, toolGlobalParams.coll)) { continue; + } - log() << "trying to recover: " << ns << endl; + toolInfoLog() << "trying to recover: " << ns << std::endl; LogIndentLevel lil2; try { _repair( db , ns , root ); } catch ( DBException& e ){ - log() << "ERROR recovering: " << ns << " " << e.toString() << endl; + toolError() << "ERROR recovering: " << ns << " " << e.toString() << std::endl; } } @@ -427,26 +432,34 @@ public: } int run() { - - if ( hasParam( "repair" ) ){ - warning() << "repair is a work in progress" << endl; + bool usingMongos = isMongos(); + int serverAuthzVersion = 0; + BSONObj dumpQuery; + + if (mongoDumpGlobalParams.repair){ return repair(); } { - string q = getParam("query"); - if ( q.size() ) - _query = fromjson( q ); + if (mongoDumpGlobalParams.query.size()) { + dumpQuery = fromjson(mongoDumpGlobalParams.query); + } + } + + if (mongoDumpGlobalParams.dumpUsersAndRoles) { + uassertStatusOK(auth::getRemoteStoredAuthorizationVersion(&conn(true), + &serverAuthzVersion)); + uassert(17369, + mongoutils::str::stream() << "Backing up users and roles is only supported for " + "clusters with auth schema versions 1 or 3, found: " << + serverAuthzVersion, + serverAuthzVersion == AuthorizationManager::schemaVersion24 || + serverAuthzVersion == AuthorizationManager::schemaVersion26Final); } string opLogName = ""; unsigned long long opLogStart = 0; - if (hasParam("oplog")) { - if (hasParam("query") || hasParam("db") || hasParam("collection")) { - log() << "oplog mode is only supported on full dumps" << endl; - return -1; - } - + if (mongoDumpGlobalParams.useOplog) { BSONObj isMaster; conn("true").simpleCommand("admin", &isMaster, "isMaster"); @@ -457,14 +470,16 @@ public: else { opLogName = "local.oplog.$main"; if ( ! isMaster["ismaster"].trueValue() ) { - log() << "oplog mode is only supported on master or replica set member" << endl; + toolError() << "oplog mode is only supported on master or replica set member" + << std::endl; return -1; } } BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk); if (op.isEmpty()) { - log() << "No operations in oplog. Please ensure you are connecting to a master." << endl; + toolError() << "No operations in oplog. Please ensure you are connecting to a " + << "master." << std::endl; return -1; } @@ -473,34 +488,33 @@ public: } // check if we're outputting to stdout - string out = getParam("out"); - if ( out == "-" ) { - if ( _db != "" && _coll != "" ) { - writeCollectionStdout( _db+"."+_coll ); + if (mongoDumpGlobalParams.outputDirectory == "-") { + if (toolGlobalParams.db != "" && toolGlobalParams.coll != "") { + writeCollectionStdout(toolGlobalParams.db + "." + toolGlobalParams.coll, dumpQuery, + usingMongos); return 0; } else { - log() << "You must specify database and collection to print to stdout" << endl; + toolError() << "You must specify database and collection to print to stdout" + << std::endl; return -1; } } - _usingMongos = isMongos(); + boost::filesystem::path root(mongoDumpGlobalParams.outputDirectory); - boost::filesystem::path root( out ); - string db = _db; - - if ( db == "" ) { - if ( _coll != "" ) { - error() << "--db must be specified with --collection" << endl; + if (toolGlobalParams.db == "") { + if (toolGlobalParams.coll != "") { + toolError() << "--db must be specified with --collection" << std::endl; return -1; } - log() << "all dbs" << endl; + toolInfoLog() << "all dbs" << std::endl; BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); if ( ! res["databases"].isABSONObj() ) { - error() << "output of listDatabases isn't what we expected, no 'databases' field:\n" << res << endl; + toolError() << "output of listDatabases isn't what we expected, no 'databases' " + << "field:\n" << res << std::endl; return -2; } BSONObj dbs = res["databases"].embeddedObjectUserCheck(); @@ -510,7 +524,8 @@ public: string key = *i; if ( ! dbs[key].isABSONObj() ) { - error() << "database field not an object key: " << key << " value: " << dbs[key] << endl; + toolError() << "database field not an document key: " << key << " value: " + << dbs[key] << std::endl; return -3; } @@ -520,31 +535,39 @@ public: if ( (string)dbName == "local" ) continue; - go ( dbName , root / dbName ); + boost::filesystem::path outdir = root / dbName; + toolInfoLog() << "DATABASE: " << dbName << "\t to \t" << outdir.string() + << std::endl; + go ( dbName , "", dumpQuery, outdir, "", usingMongos ); } } else { - go( db , root / db ); + boost::filesystem::path outdir = root / toolGlobalParams.db; + toolInfoLog() << "DATABASE: " << toolGlobalParams.db << "\t to \t" << outdir.string() + << std::endl; + go(toolGlobalParams.db, toolGlobalParams.coll, dumpQuery, outdir, "", usingMongos); + if (mongoDumpGlobalParams.dumpUsersAndRoles && + serverAuthzVersion == AuthorizationManager::schemaVersion26Final && + toolGlobalParams.db != "admin") { + toolInfoLog() << "Backing up user and role data for the " << toolGlobalParams.db << + " database"; + Query query = Query(BSON("db" << toolGlobalParams.db)); + go("admin", "system.users", query, outdir, "$admin.system.users", usingMongos); + go("admin", "system.roles", query, outdir, "$admin.system.roles", usingMongos); + } } if (!opLogName.empty()) { BSONObjBuilder b; b.appendTimestamp("$gt", opLogStart); - _query = BSON("ts" << b.obj()); + dumpQuery = BSON("ts" << b.obj()); - writeCollectionFile( opLogName , root / "oplog.bson" ); + writeCollectionFile( opLogName , dumpQuery, root / "oplog.bson", usingMongos ); } return 0; } - - bool _usingMongos; - BSONObj _query; }; -int main( int argc , char ** argv, char ** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - Dump d; - return d.main( argc , argv ); -} +REGISTER_MONGO_TOOL(Dump); diff --git a/src/mongo/tools/export.cpp b/src/mongo/tools/export.cpp index ad2195846f0..75460f76a15 100644 --- a/src/mongo/tools/export.cpp +++ b/src/mongo/tools/export.cpp @@ -1,5 +1,3 @@ -// export.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,55 +12,42 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" -#include "db/json.h" -#include "mongo/base/initializer.h" -#include "mongo/client/dbclientcursor.h" - -#include "tool.h" +#include "mongo/pch.h" +#include <boost/filesystem/convenience.hpp> +#include <boost/filesystem/operations.hpp> #include <fstream> #include <iostream> -#include <boost/filesystem/convenience.hpp> -#include <boost/filesystem/operations.hpp> -#include <boost/program_options.hpp> +#include "mongo/client/dbclientcursor.h" +#include "mongo/db/json.h" +#include "mongo/tools/mongoexport_options.h" +#include "mongo/tools/tool.h" +#include "mongo/tools/tool_logger.h" +#include "mongo/util/options_parser/option_section.h" using namespace mongo; -namespace po = boost::program_options; - class Export : public Tool { public: - Export() : Tool( "export" ) { - addFieldOptions(); - add_options() - ("query,q" , po::value<string>() , "query filter, as a JSON string" ) - ("csv","export to csv instead of json") - ("out,o", po::value<string>(), "output file; if not specified, stdout is used") - ("jsonArray", "output to a json array rather than one object per line") - ("slaveOk,k", po::value<bool>()->default_value(true) , "use secondaries for export if available, default true") - ("forceTableScan", "force a table scan (do not use $snapshot)" ) - ; - _usesstdout = false; - } - - virtual void preSetup() { - if ( hasParam("out") ) { - string out = getParam("out"); - if ( out != "-" ) { - // we write output to standard error by default to avoid - // mangling output, but we don't need to do this if an output - // file was specified - useStandardOutput(true); - } - } - } + Export() : Tool() { } - virtual void printExtraHelp( ostream & out ) { - out << "Export MongoDB data to CSV, TSV or JSON files.\n" << endl; + virtual void printHelp( ostream & out ) { + printMongoExportHelp(&out); } // Turn every double quote character into two double quote characters @@ -107,7 +92,10 @@ public: case jstOID: return "ObjectID(" + object.OID().toString() + ")"; // OIDs are always 24 bytes case Date: - return timeToISOString(object.Date() / 1000); + // We need to check if we can actually format this date. See SERVER-13760. + return object.Date().isFormatable() ? + dateToISOStringUTC(object.Date()) : + csvEscape(object.jsonString(Strict, false)); case Timestamp: return csvEscape(object.jsonString(Strict, false)); case RegEx: @@ -134,22 +122,19 @@ public: int run() { string ns; - const bool csv = hasParam( "csv" ); - const bool jsonArray = hasParam( "jsonArray" ); ostream *outPtr = &cout; - string outfile = getParam( "out" ); auto_ptr<ofstream> fileStream; - if ( hasParam( "out" ) && outfile != "-" ) { - size_t idx = outfile.rfind( "/" ); + if (mongoExportGlobalParams.outputFileSpecified && mongoExportGlobalParams.outputFile != "-") { + size_t idx = mongoExportGlobalParams.outputFile.rfind("/"); if ( idx != string::npos ) { - string dir = outfile.substr( 0 , idx + 1 ); + string dir = mongoExportGlobalParams.outputFile.substr( 0 , idx + 1 ); boost::filesystem::create_directories( dir ); } - ofstream * s = new ofstream( outfile.c_str() , ios_base::out ); + ofstream * s = new ofstream(mongoExportGlobalParams.outputFile.c_str(), ios_base::out); fileStream.reset( s ); outPtr = s; if ( ! s->good() ) { - cerr << "couldn't open [" << outfile << "]" << endl; + cerr << "couldn't open [" << mongoExportGlobalParams.outputFile << "]" << endl; return -1; } } @@ -166,18 +151,17 @@ public: return 1; } - if ( hasParam( "fields" ) || csv ) { - needFields(); + if (toolGlobalParams.fieldsSpecified || mongoExportGlobalParams.csv) { - // we can't use just _fieldsObj since we support everything getFieldDotted does + // we can't use just toolGlobalParams.fields since we support everything getFieldDotted + // does set<string> seen; BSONObjBuilder b; - BSONObjIterator i( _fieldsObj ); - while ( i.more() ){ - BSONElement e = i.next(); - string f = str::before( e.fieldName() , '.' ); + for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); + i != toolGlobalParams.fields.end(); i++) { + std::string f = str::before(*i, '.'); if ( seen.insert( f ).second ) b.append( f , 1 ); } @@ -187,38 +171,47 @@ public: } - if ( csv && _fields.size() == 0 ) { + if (mongoExportGlobalParams.csv && !toolGlobalParams.fieldsSpecified) { cerr << "csv mode requires a field list" << endl; return -1; } - Query q( getParam( "query" , "" ) ); - if ( q.getFilter().isEmpty() && !hasParam("dbpath") && !hasParam("forceTableScan") ) - q.snapshot(); + Query q(mongoExportGlobalParams.query); + if (mongoExportGlobalParams.sort != "") { + BSONObj sortSpec = mongo::fromjson(mongoExportGlobalParams.sort); + q.sort(sortSpec); + } - bool slaveOk = _params["slaveOk"].as<bool>(); + if (mongoExportGlobalParams.snapShotQuery) { + q.snapshot(); + } - auto_ptr<DBClientCursor> cursor = conn().query( ns.c_str() , q , 0 , 0 , fieldsToReturn , ( slaveOk ? QueryOption_SlaveOk : 0 ) | QueryOption_NoCursorTimeout ); + auto_ptr<DBClientCursor> cursor = conn().query(ns.c_str(), q, + mongoExportGlobalParams.limit, mongoExportGlobalParams.skip, fieldsToReturn, + (mongoExportGlobalParams.slaveOk ? QueryOption_SlaveOk : 0) | + QueryOption_NoCursorTimeout); - if ( csv ) { - for ( vector<string>::iterator i=_fields.begin(); i != _fields.end(); i++ ) { - if ( i != _fields.begin() ) + if (mongoExportGlobalParams.csv) { + for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); + i != toolGlobalParams.fields.end(); i++) { + if (i != toolGlobalParams.fields.begin()) out << ","; out << *i; } out << endl; } - if (jsonArray) + if (mongoExportGlobalParams.jsonArray) out << '['; long long num = 0; while ( cursor->more() ) { num++; BSONObj obj = cursor->next(); - if ( csv ) { - for ( vector<string>::iterator i=_fields.begin(); i != _fields.end(); i++ ) { - if ( i != _fields.begin() ) + if (mongoExportGlobalParams.csv) { + for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); + i != toolGlobalParams.fields.end(); i++) { + if (i != toolGlobalParams.fields.begin()) out << ","; const BSONElement & e = obj.getFieldDotted(i->c_str()); if ( ! e.eoo() ) { @@ -228,27 +221,23 @@ public: out << endl; } else { - if (jsonArray && num != 1) + if (mongoExportGlobalParams.jsonArray && num != 1) out << ','; out << obj.jsonString(); - if (!jsonArray) + if (!mongoExportGlobalParams.jsonArray) out << endl; } } - if (jsonArray) + if (mongoExportGlobalParams.jsonArray) out << ']' << endl; - cerr << "exported " << num << " records" << endl; + toolInfoOutput() << "exported " << num << " records" << endl; return 0; } }; -int main( int argc , char ** argv, char** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - Export e; - return e.main( argc , argv ); -} +REGISTER_MONGO_TOOL(Export); diff --git a/src/mongo/tools/files.cpp b/src/mongo/tools/files.cpp index 7170f57f764..6ae94fea8df 100644 --- a/src/mongo/tools/files.cpp +++ b/src/mongo/tools/files.cpp @@ -1,5 +1,3 @@ -// files.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,53 +12,40 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" -#include "client/gridfs.h" -#include "mongo/base/initializer.h" -#include "mongo/client/dbclientcursor.h" - -#include "tool.h" -#include "pcrecpp.h" +#include "mongo/pch.h" #include <fstream> #include <iostream> +#include <pcrecpp.h> -#include <boost/program_options.hpp> +#include "mongo/client/dbclientcursor.h" +#include "mongo/client/gridfs.h" +#include "mongo/tools/mongofiles_options.h" +#include "mongo/tools/tool.h" +#include "mongo/util/options_parser/option_section.h" using namespace mongo; -namespace po = boost::program_options; - class Files : public Tool { public: - Files() : Tool( "files" ) { - add_options() - ( "local,l", po::value<string>(), "local filename for put|get (default is to use the same name as 'gridfs filename')") - ( "type,t", po::value<string>(), "MIME type for put (default is to omit)") - ( "replace,r", "Remove other files with same name after PUT") - ; - add_hidden_options() - ( "command" , po::value<string>() , "command (list|search|put|get)" ) - ( "file" , po::value<string>() , "filename for get|put" ) - ; - addPositionArg( "command" , 1 ); - addPositionArg( "file" , 2 ); - } + Files() : Tool() { } - virtual void printExtraHelp( ostream & out ) { - out << "Browse and modify a GridFS filesystem.\n" << endl; - out << "usage: " << _name << " [options] command [gridfs filename]" << endl; - out << "command:" << endl; - out << " one of (list|search|put|get)" << endl; - out << " list - list all files. 'gridfs filename' is an optional prefix " << endl; - out << " which listed filenames must begin with." << endl; - out << " search - search all files. 'gridfs filename' is a substring " << endl; - out << " which listed filenames must contain." << endl; - out << " put - add a file with filename 'gridfs filename'" << endl; - out << " get - get a file with filename 'gridfs filename'" << endl; - out << " delete - delete all files with filename 'gridfs filename'" << endl; + virtual void printHelp( ostream & out ) { + printMongoFilesHelp(&out); } void display( GridFS * grid , BSONObj obj ) { @@ -75,95 +60,91 @@ public: } int run() { - string cmd = getParam( "command" ); - if ( cmd.size() == 0 ) { + if (mongoFilesGlobalParams.command.size() == 0) { cerr << "ERROR: need command" << endl << endl; printHelp(cout); return -1; } - GridFS g( conn() , _db ); - - string filename = getParam( "file" ); + GridFS g(conn(), toolGlobalParams.db); - if ( cmd == "list" ) { + if (mongoFilesGlobalParams.command == "list") { BSONObjBuilder b; - if ( filename.size() ) { + if (mongoFilesGlobalParams.gridFSFilename.size()) { b.appendRegex( "filename" , (string)"^" + - pcrecpp::RE::QuoteMeta( filename ) ); + pcrecpp::RE::QuoteMeta(mongoFilesGlobalParams.gridFSFilename) ); } display( &g , b.obj() ); return 0; } - if ( filename.size() == 0 ) { + if (mongoFilesGlobalParams.gridFSFilename.size() == 0) { cerr << "ERROR: need a filename" << endl << endl; printHelp(cout); return -1; } - if ( cmd == "search" ) { + if (mongoFilesGlobalParams.command == "search") { BSONObjBuilder b; - b.appendRegex( "filename" , filename ); + b.appendRegex("filename", mongoFilesGlobalParams.gridFSFilename); display( &g , b.obj() ); return 0; } - if ( cmd == "get" ) { - GridFile f = g.findFile( filename ); + if (mongoFilesGlobalParams.command == "get") { + GridFile f = g.findFile(mongoFilesGlobalParams.gridFSFilename); if ( ! f.exists() ) { cerr << "ERROR: file not found" << endl; return -2; } - string out = getParam("local", f.getFilename()); - f.write( out ); + f.write(mongoFilesGlobalParams.localFile); - if (out != "-") - cout << "done write to: " << out << endl; + if (mongoFilesGlobalParams.localFile != "-") { + toolInfoOutput() << "done write to: " << mongoFilesGlobalParams.localFile + << std::endl; + } return 0; } - if ( cmd == "put" ) { - const string& infile = getParam("local", filename); - const string& type = getParam("type", ""); - - BSONObj file = g.storeFile(infile, filename, type); - cout << "added file: " << file << endl; - - if (hasParam("replace")) { - auto_ptr<DBClientCursor> cursor = conn().query(_db+".fs.files", BSON("filename" << filename << "_id" << NE << file["_id"] )); + if (mongoFilesGlobalParams.command == "put") { + BSONObj file = g.storeFile(mongoFilesGlobalParams.localFile, + mongoFilesGlobalParams.gridFSFilename, + mongoFilesGlobalParams.contentType); + toolInfoOutput() << "added file: " << file << std::endl; + + if (mongoFilesGlobalParams.replace) { + auto_ptr<DBClientCursor> cursor = + conn().query(toolGlobalParams.db + ".fs.files", + BSON("filename" << mongoFilesGlobalParams.gridFSFilename + << "_id" << NE << file["_id"] )); while (cursor->more()) { BSONObj o = cursor->nextSafe(); - conn().remove(_db+".fs.files", BSON("_id" << o["_id"])); - conn().remove(_db+".fs.chunks", BSON("_id" << o["_id"])); - cout << "removed file: " << o << endl; + conn().remove(toolGlobalParams.db + ".fs.files", BSON("_id" << o["_id"])); + conn().remove(toolGlobalParams.db + ".fs.chunks", BSON("_id" << o["_id"])); + toolInfoOutput() << "removed file: " << o << std::endl; } } conn().getLastError(); - cout << "done!" << endl; + toolInfoOutput() << "done!" << std::endl; return 0; } - if ( cmd == "delete" ) { - g.removeFile(filename); + if (mongoFilesGlobalParams.command == "delete") { + g.removeFile(mongoFilesGlobalParams.gridFSFilename); conn().getLastError(); - cout << "done!" << endl; + toolInfoOutput() << "done!" << std::endl; return 0; } - cerr << "ERROR: unknown command '" << cmd << "'" << endl << endl; + cerr << "ERROR: unknown command '" << mongoFilesGlobalParams.command << "'" << endl << endl; printHelp(cout); return -1; } }; -int main( int argc , char ** argv, char** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - Files f; - return f.main( argc , argv ); -} +REGISTER_MONGO_TOOL(Files); diff --git a/src/mongo/tools/import.cpp b/src/mongo/tools/import.cpp index 61e23f01878..d87fe1209e2 100644 --- a/src/mongo/tools/import.cpp +++ b/src/mongo/tools/import.cpp @@ -1,5 +1,3 @@ -// import.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,37 +12,44 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" -#include "db/json.h" -#include "tool.h" -#include "../util/text.h" -#include "mongo/base/initializer.h" -#include <fstream> -#include <iostream> -#include <boost/program_options.hpp> +#include "mongo/pch.h" + #include <boost/algorithm/string.hpp> #include <boost/filesystem/operations.hpp> +#include <fstream> +#include <iostream> + +#include "mongo/base/initializer.h" +#include "mongo/db/json.h" +#include "mongo/tools/mongoimport_options.h" +#include "mongo/tools/tool.h" +#include "mongo/util/options_parser/option_section.h" +#include "mongo/util/text.h" using namespace mongo; using std::string; using std::stringstream; -namespace po = boost::program_options; - class Import : public Tool { enum Type { JSON , CSV , TSV }; Type _type; const char * _sep; - bool _ignoreBlanks; - bool _headerLine; - bool _upsert; - bool _doimport; - bool _jsonArray; - vector<string> _upsertFields; static const int BUF_SIZE; void csvTokenizeRow(const string& row, vector<string>& tokens) { @@ -93,7 +98,7 @@ class Import : public Tool { } void _append( BSONObjBuilder& b , const string& fieldName , const string& data ) { - if ( _ignoreBlanks && data.size() == 0 ) + if (mongoImportGlobalParams.ignoreBlanks && data.size() == 0) return; if ( b.appendAsNumber( fieldName , data ) ) @@ -109,23 +114,19 @@ class Import : public Tool { * increment buf by this amount. */ int getLine(istream* in, char* buf) { - if (_jsonArray) { - in->read(buf, BUF_SIZE); - uassert(13295, "JSONArray file too large", (in->rdstate() & ios_base::eofbit)); - buf[ in->gcount() ] = '\0'; + in->getline( buf , BUF_SIZE ); + if ((in->rdstate() & ios_base::eofbit) && (in->rdstate() & ios_base::failbit)) { + // this is the last line, and it's empty (not even a newline) + buf[0] = '\0'; + return 0; } - else { - in->getline( buf , BUF_SIZE ); - if ((in->rdstate() & ios_base::eofbit) && (in->rdstate() & ios_base::failbit)) { - // this is the last line, and it's empty (not even a newline) - buf[0] = '\0'; - return 0; - } - uassert(16329, str::stream() << "read error, or input line too long (max length: " - << BUF_SIZE << ")", !(in->rdstate() & ios_base::failbit)); - LOG(1) << "got line:" << buf << endl; + uassert(16329, str::stream() << "read error, or input line too long (max length: " + << BUF_SIZE << ")", !(in->rdstate() & ios_base::failbit)); + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << "got line:" << buf << std::endl; } + uassert( 10263 , "unknown error reading file" , (!(in->rdstate() & ios_base::badbit)) && (!(in->rdstate() & ios_base::failbit) || (in->rdstate() & ios_base::eofbit)) ); @@ -141,27 +142,44 @@ class Import : public Tool { } /* - * Parses a BSON object out of a JSON array. - * Returns number of bytes processed on success and -1 on failure. + * description: + * given a pointer into a JSON array, returns the next valid JSON object + * args: + * buf - pointer into the JSON array + * o - BSONObj to fill + * numBytesRead - return parameter for how far we read + * return: + * true if an object was successfully parsed + * false if there is no object left to parse + * throws: + * exception on parsing error */ - int parseJSONArray(char* buf, BSONObj& o) { - int len = 0; - while (buf[0] != '{' && buf[0] != '\0') { - len++; + bool parseJSONArray(char* buf, BSONObj* o, int* numBytesRead) { + + // Skip extra characters since fromjson must be passed a character buffer that starts with a + // valid JSON object, and does not accept JSON arrays. + // (NOTE: this doesn't catch all invalid JSON arrays, but does fail on invalid characters) + *numBytesRead = 0; + while (buf[0] == '[' || + buf[0] == ']' || + buf[0] == ',' || + isspace(buf[0])) { + (*numBytesRead)++; buf++; } + if (buf[0] == '\0') - return -1; + return false; - int jslen; try { - o = fromjson(buf, &jslen); + int len = 0; + *o = fromjson(buf, &len); + (*numBytesRead) += len; } catch ( MsgAssertionException& e ) { - uasserted(13293, string("BSON representation of supplied JSON array is too large: ") + e.what()); + uasserted(13293, string("Invalid JSON passed to mongoimport: ") + e.what()); } - len += jslen; - return len; + return true; } /* @@ -218,7 +236,7 @@ class Import : public Tool { line += num; numBytesRead += num; - uassert (15854, "CSV file ends while inside quoted field", line[0] != '\0'); + uassert(15854, "CSV file ends while inside quoted field", line[0] != '\0'); numBytesRead += strlen( line ); } else { break; @@ -241,13 +259,13 @@ class Import : public Tool { unsigned int pos=0; for (vector<string>::iterator it = tokens.begin(); it != tokens.end(); ++it) { string token = *it; - if ( _headerLine ) { - _fields.push_back(token); + if (mongoImportGlobalParams.headerLine) { + toolGlobalParams.fields.push_back(token); } else { string name; - if ( pos < _fields.size() ) { - name = _fields[pos]; + if (pos < toolGlobalParams.fields.size()) { + name = toolGlobalParams.fields[pos]; } else { stringstream ss; @@ -264,36 +282,12 @@ class Import : public Tool { } public: - Import() : Tool( "import" ) { - addFieldOptions(); - add_options() - ("ignoreBlanks","if given, empty fields in csv and tsv will be ignored") - ("type",po::value<string>() , "type of file to import. default: json (json,csv,tsv)") - ("file",po::value<string>() , "file to import from; if not specified stdin is used" ) - ("drop", "drop collection first " ) - ("headerline","first line in input file is a header (CSV and TSV only)") - ("upsert", "insert or update objects that already exist" ) - ("upsertFields", po::value<string>(), "comma-separated fields for the query part of the upsert. You should make sure this is indexed" ) - ("stopOnError", "stop importing at first error rather than continuing" ) - ("jsonArray", "load a json array, not one item per line. Currently limited to 16MB." ) - ; - add_hidden_options() - ("noimport", "don't actually import. useful for benchmarking parser" ) - ; - addPositionArg( "file" , 1 ); + Import() : Tool() { _type = JSON; - _ignoreBlanks = false; - _headerLine = false; - _upsert = false; - _doimport = true; - _jsonArray = false; } - ; - virtual void printExtraHelp( ostream & out ) { - out << "Import CSV, TSV or JSON data into MongoDB.\n" << endl; - out << "When importing JSON documents, each document must be a separate line of the input file.\n"; - out << "\nExample:\n"; - out << " mongoimport --host myhost --db my_cms --collection docs < mydocfile.json\n" << endl; + + virtual void printHelp( ostream & out ) { + printMongoImportHelp(&out); } unsigned long long lastErrorFailures; @@ -305,33 +299,57 @@ public: if( str::contains(s,"uplicate") ) { // we don't want to return an error from the mongoimport process for // dup key errors - log() << s << endl; + toolInfoLog() << s << endl; } else { lastErrorFailures++; - log() << "error: " << s << endl; + toolInfoLog() << "error: " << s << endl; return false; } } return true; } + void importDocument (const std::string &ns, const BSONObj& o) { + bool doUpsert = mongoImportGlobalParams.upsert; + BSONObjBuilder b; + if (mongoImportGlobalParams.upsert) { + for (vector<string>::const_iterator it = mongoImportGlobalParams.upsertFields.begin(), + end = mongoImportGlobalParams.upsertFields.end(); it != end; ++it) { + BSONElement e = o.getFieldDotted(it->c_str()); + if (e.eoo()) { + doUpsert = false; + break; + } + b.appendAs(e, *it); + } + } + + if (doUpsert) { + conn().update(ns, Query(b.obj()), o, true); + } + else { + conn().insert(ns.c_str(), o); + } + } + int run() { - string filename = getParam( "file" ); long long fileSize = 0; int headerRows = 0; istream * in = &cin; - ifstream file( filename.c_str() , ios_base::in); + ifstream file(mongoImportGlobalParams.filename.c_str(), ios_base::in); - if ( filename.size() > 0 && filename != "-" ) { - if ( ! boost::filesystem::exists( filename ) ) { - error() << "file doesn't exist: " << filename << endl; + if (mongoImportGlobalParams.filename.size() > 0 && + mongoImportGlobalParams.filename != "-") { + if ( ! boost::filesystem::exists(mongoImportGlobalParams.filename) ) { + toolError() << "file doesn't exist: " << mongoImportGlobalParams.filename + << std::endl; return -1; } in = &file; - fileSize = boost::filesystem::file_size( filename ); + fileSize = boost::filesystem::file_size(mongoImportGlobalParams.filename); } // check if we're actually talking to a machine that can write @@ -345,156 +363,204 @@ public: ns = getNS(); } catch (...) { - printHelp(cerr); - return -1; + // The only time getNS throws is when the collection was not specified. In that case, + // check if the user specified a file name and use that as the collection name. + if (!mongoImportGlobalParams.filename.empty()) { + string oldCollName = + boost::filesystem::path(mongoImportGlobalParams.filename).leaf().string(); + oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); + cerr << "using filename '" << oldCollName << "' as collection." << endl; + ns = toolGlobalParams.db + "." + oldCollName; + } + else { + printHelp(cerr); + return -1; + } } - LOG(1) << "ns: " << ns << endl; - - if ( hasParam( "drop" ) ) { - log() << "dropping: " << ns << endl; - conn().dropCollection( ns.c_str() ); + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << "ns: " << ns << endl; } - if ( hasParam( "ignoreBlanks" ) ) { - _ignoreBlanks = true; + if (mongoImportGlobalParams.drop) { + toolInfoLog() << "dropping: " << ns << endl; + conn().dropCollection(ns.c_str()); } - if ( hasParam( "upsert" ) || hasParam( "upsertFields" )) { - _upsert = true; - - string uf = getParam("upsertFields"); - if (uf.empty()) { - _upsertFields.push_back("_id"); - } - else { - StringSplitter(uf.c_str(), ",").split(_upsertFields); - } + if (mongoImportGlobalParams.type == "json") + _type = JSON; + else if (mongoImportGlobalParams.type == "csv") { + _type = CSV; + _sep = ","; } - - if ( hasParam( "noimport" ) ) { - _doimport = false; + else if (mongoImportGlobalParams.type == "tsv") { + _type = TSV; + _sep = "\t"; } - - if ( hasParam( "type" ) ) { - string type = getParam( "type" ); - if ( type == "json" ) - _type = JSON; - else if ( type == "csv" ) { - _type = CSV; - _sep = ","; - } - else if ( type == "tsv" ) { - _type = TSV; - _sep = "\t"; - } - else { - error() << "don't know what type [" << type << "] is" << endl; - return -1; - } + else { + toolError() << "don't know what type [" << mongoImportGlobalParams.type << "] is" + << std::endl; + return -1; } - if ( _type == CSV || _type == TSV ) { - _headerLine = hasParam( "headerline" ); - if ( _headerLine ) { + if (_type == CSV || _type == TSV) { + if (mongoImportGlobalParams.headerLine) { headerRows = 1; } else { - needFields(); + if (!toolGlobalParams.fieldsSpecified) { + throw UserException(9998, "You need to specify fields or have a headerline to " + "import this file type"); + } } } - if (_type == JSON && hasParam("jsonArray")) { - _jsonArray = true; - } time_t start = time(0); - LOG(1) << "filesize: " << fileSize << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoLog() << "filesize: " << fileSize << endl; + } ProgressMeter pm( fileSize ); int num = 0; int lastNumChecked = num; int errors = 0; lastErrorFailures = 0; int len = 0; - // buffer and line are only used when parsing a jsonArray - boost::scoped_array<char> buffer(new char[BUF_SIZE+2]); - char* line = buffer.get(); - while ( _jsonArray || in->rdstate() == 0 ) { - try { - BSONObj o; - if (_jsonArray) { - int bytesProcessed = 0; - if (line == buffer.get()) { // Only read on first pass - the whole array must be on one line. - bytesProcessed = getLine(in, line); - line += bytesProcessed; - len += bytesProcessed; - } - if ((bytesProcessed = parseJSONArray(line, o)) < 0) { - len += bytesProcessed; + // We have to handle jsonArrays differently since we can't read line by line + if (_type == JSON && mongoImportGlobalParams.jsonArray) { + + // We cycle through these buffers in order to continuously read from the stream + boost::scoped_array<char> buffer1(new char[BUF_SIZE]); + boost::scoped_array<char> buffer2(new char[BUF_SIZE]); + char* current_buffer = buffer1.get(); + char* next_buffer = buffer2.get(); + char* temp_buffer; + + // buffer_base_offset is the offset into the stream where our buffer starts, while + // input_stream_offset is the total number of bytes read from the stream + uint64_t buffer_base_offset = 0; + uint64_t input_stream_offset = 0; + + // Fill our buffer + // NOTE: istream::get automatically appends '\0' at the end of what it reads + in->get(current_buffer, BUF_SIZE, '\0'); + uassert(16808, str::stream() << "read error: " << strerror(errno), !in->fail()); + + // Record how far we read into the stream. + input_stream_offset += in->gcount(); + + while (true) { + try { + + BSONObj o; + + // Try to parse (parseJSONArray) + if (!parseJSONArray(current_buffer, &o, &len)) { break; } - len += bytesProcessed; - line += bytesProcessed; - } - else { - if (!parseRow(in, o, len)) { - continue; + + // Import documents + if (mongoImportGlobalParams.doimport) { + importDocument(ns, o); + + if (num < 10) { + // we absolutely want to check the first and last op of the batch. we do + // a few more as that won't be too time expensive. + checkLastError(); + lastNumChecked = num; + } + } + + // Copy over the part of buffer that was not parsed + strcpy(next_buffer, current_buffer + len); + + // Advance our buffer base past what we've already parsed + buffer_base_offset += len; + + // Fill up the end of our next buffer only if there is something in the stream + if (!in->eof()) { + // NOTE: istream::get automatically appends '\0' at the end of what it reads + in->get(next_buffer + (input_stream_offset - buffer_base_offset), + BUF_SIZE - (input_stream_offset - buffer_base_offset), '\0'); + uassert(16809, str::stream() << "read error: " + << strerror(errno), !in->fail()); + + // Record how far we read into the stream. + input_stream_offset += in->gcount(); } + + // Swap buffer pointers + temp_buffer = current_buffer; + current_buffer = next_buffer; + next_buffer = temp_buffer; + + num++; } + catch ( const std::exception& e ) { + toolError() << "exception: " << e.what() + << ", current buffer: " << current_buffer << std::endl; + errors++; - if ( _headerLine ) { - _headerLine = false; + // Since we only support JSON arrays all on one line, we might as well stop now + // because we can't read any more documents + break; } - else if (_doimport) { - bool doUpsert = _upsert; - BSONObjBuilder b; - if (_upsert) { - for (vector<string>::const_iterator it=_upsertFields.begin(), end=_upsertFields.end(); it!=end; ++it) { - BSONElement e = o.getFieldDotted(it->c_str()); - if (e.eoo()) { - doUpsert = false; - break; - } - b.appendAs(e, *it); - } - } - if (doUpsert) { - conn().update(ns, Query(b.obj()), o, true); + if (!toolGlobalParams.quiet) { + if (pm.hit(len + 1)) { + log() << "\t\t\t" << num << "\t" << (num / (time(0) - start)) << "/second" + << std::endl; } - else { - conn().insert( ns.c_str() , o ); + } + } + } + else { + while (in->rdstate() == 0) { + try { + BSONObj o; + + if (!parseRow(in, o, len)) { + continue; } - if( num < 10 ) { - // we absolutely want to check the first and last op of the batch. we do - // a few more as that won't be too time expensive. - checkLastError(); - lastNumChecked = num; + if (mongoImportGlobalParams.headerLine) { + mongoImportGlobalParams.headerLine = false; + } + else if (mongoImportGlobalParams.doimport) { + importDocument(ns, o); + + if (num < 10) { + // we absolutely want to check the first and last op of the batch. we do + // a few more as that won't be too time expensive. + checkLastError(); + lastNumChecked = num; + } } - } - num++; - } - catch ( std::exception& e ) { - log() << "exception:" << e.what() << endl; - log() << line << endl; - errors++; + num++; + } + catch ( const std::exception& e ) { + toolError() << "exception:" << e.what() << std::endl; + errors++; - if (hasParam("stopOnError") || _jsonArray) - break; - } + if (mongoImportGlobalParams.stopOnError) + break; + } - if ( pm.hit( len + 1 ) ) { - log() << "\t\t\t" << num << "\t" << ( num / ( time(0) - start ) ) << "/second" << endl; + if (!toolGlobalParams.quiet) { + if (pm.hit(len + 1)) { + log() << "\t\t\t" << num << "\t" << (num / (time(0) - start)) << "/second" + << std::endl; + } + } } } // this is for two reasons: to wait for all operations to reach the server and be processed, and this will wait until all data reaches the server, // and secondly to check if there were an error (on the last op) if( lastNumChecked+1 != num ) { // avoid redundant log message if already reported above - log() << "check " << lastNumChecked << " " << num << endl; + toolInfoLog() << "check " << lastNumChecked << " " << num << endl; checkLastError(); } @@ -502,20 +568,19 @@ public: // the message is vague on lastErrorFailures as we don't call it on every single operation. // so if we have a lastErrorFailure there might be more than just what has been counted. - log() << (lastErrorFailures ? "tried to import " : "imported ") << ( num - headerRows ) << " objects" << endl; + toolInfoLog() << (lastErrorFailures ? "tried to import " : "imported ") + << (num - headerRows) << " objects" << std::endl; if ( !hadErrors ) return 0; - error() << "encountered " << (lastErrorFailures?"at least ":"") << lastErrorFailures+errors << " error(s)" << ( lastErrorFailures+errors == 1 ? "" : "s" ) << endl; + toolError() << "encountered " << (lastErrorFailures?"at least ":"") + << lastErrorFailures+errors << " error(s)" + << (lastErrorFailures+errors == 1 ? "" : "s") << std::endl; return -1; } }; -int main( int argc , char ** argv, char** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - Import import; - return import.main( argc , argv ); -} - const int Import::BUF_SIZE(1024 * 1024 * 16); + +REGISTER_MONGO_TOOL(Import); diff --git a/src/mongo/tools/loadgenerator.cpp b/src/mongo/tools/loadgenerator.cpp deleted file mode 100644 index 42e7ad8029e..00000000000 --- a/src/mongo/tools/loadgenerator.cpp +++ /dev/null @@ -1,417 +0,0 @@ -/** @file loadgenerator.cpp */ - -/** - * Copyright (C) 2012 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -/* - * LoadGenerator drives a certain number (# threads) of simultaneous operation into a - * specified number of databases as quickly as it can at a mongo instance, - * continuously, for some number of seconds. - * - */ - -/* - * For internal reference: - * Each document generated by the docgenerator.cpp is 176 bytes long - * Number of documents per instance size : - * small (500 MB) : 2978905 docs spread over 5 dbs (each db is 100 MB) Docs Per DB : 595781 - * medium (5 GB) : 30504030 docs spread over 5 dbs (each db is 1 GB) Docs Per DB : 6100806 - * large (25 GB) : 152520145 docs evenly spread over 5 dbs (each db is 5 GB) Docs Per DB : 30504029 - * vlarge (100 GB) : 621172954 docs evenly spread over 10 dbs (each db is 10 GB) Docs Per DB : 61008058 - * - */ - -#include <map> -#include <string> - -#include <boost/program_options.hpp> -#include <boost/scoped_ptr.hpp> - -#include "mongo/base/initializer.h" -#include "mongo/util/assert_util.h" -#include "mongo/scripting/bench.h" -#include "mongo/client/dbclientinterface.h" -#include "mongo/tools/docgenerator.h" -#include "mongo/util/map_util.h" -#include "mongo/util/md5.hpp" -#include "mongo/util/mongoutils/str.h" - -using namespace std; - -using namespace mongo; - -namespace po = boost::program_options; - -namespace { - -struct LoadGeneratorOptions { - LoadGeneratorOptions() : - hostname("localhost"), - type("query"), - instanceSize( "large" ), - numdbs( 5 ), - resultNS( "" ), - numOps( 60000 ), - durationSeconds( 60 ), - parallelThreads( 32 ), - trials( 5 ), - docsPerDB( 0 ) - { } - - string hostname; - string type; - string instanceSize; - int numdbs; - string resultNS; - int numOps; - double durationSeconds; - int parallelThreads; - int trials; - unsigned long long docsPerDB; -}; - - -struct OperationStats { - OperationStats( const unsigned long long numEvents, const unsigned long long totalTimeMicros, - const long long opcounter ) : - numEvents( numEvents ), - totalTimeMicros( totalTimeMicros ), - opcounter( opcounter ) { } - OperationStats() { } - - unsigned long long numEvents; - unsigned long long totalTimeMicros; - long long opcounter; -}; - -// ------Globals and typedefs---------- -LoadGeneratorOptions globalLoadGenOption; -typedef std::map<std::string, OperationStats> OpStatsMap; - -double randomBetweenRange(const int& min, const int& max) { - return rand() % (max - min) + min; -} - -mongo::DBClientBase *getDBConnection() { - string errmsg; - mongo::ConnectionString connectionString = mongo::ConnectionString::parse( - globalLoadGenOption.hostname, errmsg ); - mongo::fassert( 16265, connectionString.isValid() ); - mongo::DBClientBase *connection = connectionString.connect( errmsg ); - mongo::fassert( 16266, connection != NULL ); - return connection; -} - -void dropNS(const string ns) { - boost::scoped_ptr<mongo::DBClientBase> connection( getDBConnection() ); - connection->dropCollection( ns ); -} - -void dropDB(const string db) { - boost::scoped_ptr<mongo::DBClientBase> connection( getDBConnection() ); - connection->dropDatabase( db ); -} - -void writeToNS(const string ns, const mongo::BSONObj bs) { - boost::scoped_ptr<mongo::DBClientBase> connection( getDBConnection() ); - connection->insert( ns, bs ); - mongo::fassert( 16267, connection->getLastError().empty() ); -} - -// find the number of documents in a namespace -void numDocsInNS(const string ns) { - boost::scoped_ptr<mongo::DBClientBase> connection( getDBConnection() ); - globalLoadGenOption.docsPerDB = connection->count( ns ); -} - - -mongo::BSONArray generateInsertOps() { - - //create a document config object - BSONObj args = BSONObjBuilder() - .append( "blob", "MongoDB is an open source document-oriented database system." ) - .append( "md5seed", "newyork" ) - .append( "counterUp", 0 ) - .append( "counterDown", numeric_limits<long long>::max() ).obj(); - - scoped_ptr<DocumentGenerator> docGen( DocumentGenerator::makeDocumentGenerator(args) ); - mongo::BSONArrayBuilder insertOps; - for (int i = 0; i < globalLoadGenOption.numOps; ++i) { - - //insert into databases - string insertNS = mongoutils::str::stream() << globalLoadGenOption.instanceSize << "DB" << - i % globalLoadGenOption.numdbs << "I.sampledata"; - BSONObj doc = docGen->createDocument(); - insertOps.append( BSON( "ns" << insertNS << - "op" << "insert" << - "doc" << doc << - "safe" << true ) ); - } - return insertOps.arr(); -} - - -mongo::BSONArray generateFindOneOps() { - - mongo::BSONArrayBuilder queryOps; - - // query a namespace and find the number of docs in that ns. All benchmark namespaces should have - // the same number of docs. - string queryNS = mongoutils::str::stream() << globalLoadGenOption.instanceSize - << "DB0.sampledata"; - numDocsInNS( queryNS ); - - // Now fill the queryOps array. The findOne query operations will be evenly distributed - // across all databases. Thus it tries to find a random document from db1, then db2, - // and so on and so forth. - for (int i = 0; i < globalLoadGenOption.numOps; ++i) { - - queryNS = mongoutils::str::stream() << globalLoadGenOption.instanceSize - << "DB" - << i % globalLoadGenOption.numdbs - << ".sampledata"; - - // select a random document among all the documents - unsigned long long centerQueryKey = - ( randomBetweenRange(0, 100 ) * globalLoadGenOption.docsPerDB ) / 100; - - // cast to long long from unsigned long long as BSON didn't have the overloaded method - mongo::BSONObj query = - BSON( "counterUp" << static_cast<long long>( floor(centerQueryKey) ) ); - - queryOps.append( BSON( "ns" << queryNS << - "op" << "findOne" << - "query" << query) ); - } - - return queryOps.arr(); -} - - -mongo::BenchRunConfig *createBenchRunConfig() { - - BSONArray ops; - - if ( globalLoadGenOption.type == "findOne" ) - ops = generateFindOneOps(); - else if ( globalLoadGenOption.type == "insert" ) - ops = generateInsertOps(); - - return mongo::BenchRunConfig::createFromBson( - BSON( "ops" << ops << - "parallel" << globalLoadGenOption.parallelThreads << - "seconds" << globalLoadGenOption.durationSeconds << - "host"<< globalLoadGenOption.hostname ) ); -} - -/* - * The stats object from benchRun has two sub-objects : findOneCounter and opcounters - * This function creates a single map data structure to store all the info. - */ - -void collectAllStats( const BenchRunStats& stats, OpStatsMap& allStats ) { - - allStats.clear(); - allStats.insert( std::make_pair("findOne", - OperationStats(stats.findOneCounter.getNumEvents(), - stats.findOneCounter.getTotalTimeMicros(), - mapFindWithDefault(stats.opcounters, "query", 0) - )) ); - allStats.insert( std::make_pair("insert", - OperationStats(stats.insertCounter.getNumEvents(), - stats.insertCounter.getTotalTimeMicros(), - mapFindWithDefault(stats.opcounters, "insert", 0) - )) ); - -} - -// add the result of this trial to the trials array -BSONObj makeTrialDocument( const OpStatsMap& allStats ) { - - BSONObjBuilder outerBuilder; - for (OpStatsMap::const_iterator it = allStats.begin(); it != allStats.end(); ++it) { - - unsigned long long numEvents = it->second.numEvents; - unsigned long long totalTimeMicros = it->second.totalTimeMicros; - - BSONObjBuilder innerDocBuilder; - innerDocBuilder.append("numEvents", static_cast<long long>(numEvents)); - innerDocBuilder.append("totalTimeMicros", static_cast<long long>(totalTimeMicros)); - - if (numEvents) - innerDocBuilder.append("latencyMicros", static_cast<double>(totalTimeMicros/numEvents)); - - outerBuilder.append(it->first, innerDocBuilder.obj()); - } - return outerBuilder.obj(); -} - -BSONObj buildInformation() { - boost::scoped_ptr<mongo::DBClientBase> connection( getDBConnection() ); - BSONObj info; - connection->simpleCommand("admin", &info, "buildinfo"); - return info; -} - -BSONObj createResultDoc(const BSONArray& trialsArray) { - - return BSON( "name" << globalLoadGenOption.type << - "config" << BSON ( "hostname" << globalLoadGenOption.hostname << - "instanceSize" << globalLoadGenOption.instanceSize << - "durationSeconds" << globalLoadGenOption.durationSeconds << - "parallelThreads" << globalLoadGenOption.parallelThreads << - "numOps" << globalLoadGenOption.numOps << - "Date" << 10 << - "buildInfo" << buildInformation() - ) << - "trials" << trialsArray ); -} - -void runTest() { - stringstream oss; - BSONArrayBuilder trialsBuilder; - - // drop any previous dbs with the same name - for (int j=0; j < globalLoadGenOption.numdbs; ++j) { - string insertTestDB = mongoutils::str::stream() << globalLoadGenOption.instanceSize << - "DB" << j <<"I" ; - dropDB(insertTestDB); - } - - for (int i = 0; i<globalLoadGenOption.trials; ++i) { - - BenchRunner runner( createBenchRunConfig() ); - runner.start(); - sleepmillis( 1000 * globalLoadGenOption.durationSeconds ); - runner.stop(); - BenchRunStats stats; - runner.populateStats(&stats); - - // collate all the stats (and save it in a local allstats map - std::map<std::string, OperationStats> allStats; - collectAllStats(stats, allStats); - - trialsBuilder.append(makeTrialDocument(allStats)); - - // print for now -- this is temporary and will be removed - oss << allStats.find("insert")->second.totalTimeMicros / allStats.find("insert")->second.numEvents << - " " << - allStats.find("insert")->second.opcounter / globalLoadGenOption.durationSeconds << - " "; - - //clean up the newly created dbs for next trial - for (int j=0; j < globalLoadGenOption.numdbs; ++j) { - string insertTestDB = mongoutils::str::stream() << globalLoadGenOption.instanceSize << - "DB" << j <<"I" ; - dropDB(insertTestDB); - } - } - // Write the experiment document to the result NS. If the user did not pass a resultNS cmdline - // parameter then we won't write the results to the database. - // This is useful in cases where we just want to drive a constant load from a client and are - // not really interested in the statistics from it and so don't really care to save the stats - // to a db. - - string resultNS = globalLoadGenOption.resultNS; - if( !resultNS.empty() ) { - BSONObj resultDoc = createResultDoc(trialsBuilder.arr()); - writeToNS( resultNS, resultDoc ); - } - - // temp line -- will be removed - cout << oss.str() << endl; -} - - -int parseCmdLineOptions(int argc, char **argv) { - - try { - - po::options_description general_options("General options"); - - general_options.add_options() - ("help", "produce help message") - ("hostname,H", po::value<string>() , "ip address of the host where mongod is running" ) - ("type", po::value<string>() , "findOne/insert" ) - ("instanceSize,I", po::value<string>(), "DB type (small/medium/large/vlarge)" ) - ("numdbs", po::value<int>(), " number of databases in this instance" ) - ("trials", po::value<int>(), "number of trials") - ("durationSeconds,D", po::value<double>(), "how long should each trial run") - ("parallelThreads,P",po::value<int>(), "number of threads") - ("numOps", po::value<int>(), "number of ops per thread") - ("resultNS", po::value<string>(), "result NS where you would like to save the results." - "If this parameter is empty results will not be written") - ; - - po::variables_map params; - po::store(po::parse_command_line(argc, argv, general_options), params); - po::notify(params); - - // Parse the values if supplied by the user. No data sanity check is performed - // here so meaningless values (for eg. passing --numdbs 0) can crash the program. - // TODO: Perform data sanity check - - if(params.count("help")) { - cout << general_options << "\n"; - return 1; - } - if (params.count("hostname")) { - globalLoadGenOption.hostname = params["hostname"].as<string>(); - } - if (params.count("type")) { - globalLoadGenOption.type = params["type"].as<string>(); - } - if (params.count("instanceSize")) { - globalLoadGenOption.instanceSize = params["instanceSize"].as<string>(); - } - if (params.count("numdbs")) { - globalLoadGenOption.numdbs = params["numdbs"].as<int>(); - } - if (params.count("trials")) { - globalLoadGenOption.trials = params["trials"].as<int>(); - } - if (params.count("durationSeconds")) { - globalLoadGenOption.durationSeconds = params["durationSeconds"].as<double>(); - } - if (params.count("parallelThreads")) { - globalLoadGenOption.parallelThreads = params["parallelThreads"].as<int>(); - } - if (params.count("numOps")) { - globalLoadGenOption.numOps = params["numOps"].as<int>(); - } - if (params.count("resultNS")) { - globalLoadGenOption.resultNS = params["resultNS"].as<string>(); - } - } - catch(exception& e) { - cerr << "error: " << e.what() << "\n"; - return 1; - } - return 0; -} - -} // namespace - - -int main(int argc, char **argv, char** envp) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - if( parseCmdLineOptions(argc, argv) ) - return 1; - - runTest(); - return 0; -} - diff --git a/src/mongo/tools/mongobridge_options.cpp b/src/mongo/tools/mongobridge_options.cpp new file mode 100644 index 00000000000..3275f135f25 --- /dev/null +++ b/src/mongo/tools/mongobridge_options.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongobridge_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoBridgeGlobalParams mongoBridgeGlobalParams; + + Status addMongoBridgeOptions(moe::OptionSection* options) { + + options->addOptionChaining("help", "help", moe::Switch, "produce help message"); + + + options->addOptionChaining("port", "port", moe::Int, "port to listen for mongo messages"); + + + options->addOptionChaining("dest", "dest", moe::String, "uri of remote mongod instance"); + + + options->addOptionChaining("delay", "delay", moe::Int, + "transfer delay in milliseconds (default = 0)") + .setDefault(moe::Value(0)); + + + return Status::OK(); + } + + void printMongoBridgeHelp(std::ostream* out) { + *out << "Usage: mongobridge --port <port> --dest <dest> [ --delay <ms> ] [ --help ]" + << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoBridgeOptions(const moe::Environment& params) { + if (params.count("help")) { + printMongoBridgeHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoBridgeOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + + if (!params.count("port")) { + return Status(ErrorCodes::BadValue, "Missing required option: \"--port\""); + } + + if (!params.count("dest")) { + return Status(ErrorCodes::BadValue, "Missing required option: \"--dest\""); + } + + mongoBridgeGlobalParams.port = params["port"].as<int>(); + mongoBridgeGlobalParams.destUri = params["dest"].as<std::string>(); + + if (params.count("delay")) { + mongoBridgeGlobalParams.delay = params["delay"].as<int>(); + } + + return Status::OK(); + } + +} // namespace mongo diff --git a/src/mongo/tools/mongobridge_options.h b/src/mongo/tools/mongobridge_options.h new file mode 100644 index 00000000000..181bd273db4 --- /dev/null +++ b/src/mongo/tools/mongobridge_options.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" + +namespace mongo { + + namespace optionenvironment { + class OptionSection; + class Environment; + } // namespace optionenvironment + + namespace moe = mongo::optionenvironment; + + struct MongoBridgeGlobalParams { + int port; + int delay; + int connectTimeoutSec; + string destUri; + + MongoBridgeGlobalParams() : port(0), delay(0), connectTimeoutSec(15) {} + }; + + extern MongoBridgeGlobalParams mongoBridgeGlobalParams; + + Status addMongoBridgeOptions(moe::OptionSection* options); + + void printMongoBridgeHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoBridgeOptions(const moe::Environment& params); + + Status storeMongoBridgeOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongobridge_options_init.cpp b/src/mongo/tools/mongobridge_options_init.cpp new file mode 100644 index 00000000000..5c530f95f27 --- /dev/null +++ b/src/mongo/tools/mongobridge_options_init.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2013 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongobridge_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoBridgeOptions)(InitializerContext* context) { + return addMongoBridgeOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoBridgeOptions)(InitializerContext* context) { + if (!handlePreValidationMongoBridgeOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoBridgeOptions)(InitializerContext* context) { + Status ret = storeMongoBridgeOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} + diff --git a/src/mongo/tools/mongodump_options.cpp b/src/mongo/tools/mongodump_options.cpp new file mode 100644 index 00000000000..74d07c838fb --- /dev/null +++ b/src/mongo/tools/mongodump_options.cpp @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongodump_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/log.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoDumpGlobalParams mongoDumpGlobalParams; + + Status addMongoDumpOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("out", "out,o", moe::String, + "output directory or \"-\" for stdout") + .setDefault(moe::Value(std::string("dump"))); + + options->addOptionChaining("query", "query,q", moe::String, "json query"); + + options->addOptionChaining("oplog", "oplog", moe::Switch, + "Use oplog for point-in-time snapshotting"); + + options->addOptionChaining("repair", "repair", moe::Switch, + "try to recover a crashed database"); + + options->addOptionChaining("forceTableScan", "forceTableScan", moe::Switch, + "force a table scan (do not use $snapshot)"); + + options->addOptionChaining("dumpDbUsersAndRoles", "dumpDbUsersAndRoles", moe::Switch, + "Dump user and role definitions for the given database") + .requires("db").incompatibleWith("collection"); + + return Status::OK(); + } + + void printMongoDumpHelp(std::ostream* out) { + *out << "Export MongoDB data to BSON files.\n" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoDumpOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoDumpHelp(&std::cout); + return false;; + } + return true; + } + + Status storeMongoDumpOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoDumpGlobalParams.repair = hasParam("repair"); + if (mongoDumpGlobalParams.repair){ + if (!hasParam("dbpath")) { + return Status(ErrorCodes::BadValue, "repair mode only works with --dbpath"); + } + + if (!hasParam("db")) { + return Status(ErrorCodes::BadValue, + "repair mode only works on 1 db at a time right now"); + } + } + mongoDumpGlobalParams.query = getParam("query"); + mongoDumpGlobalParams.useOplog = hasParam("oplog"); + if (mongoDumpGlobalParams.useOplog) { + if (hasParam("query") || hasParam("db") || hasParam("collection")) { + return Status(ErrorCodes::BadValue, "oplog mode is only supported on full dumps"); + } + } + mongoDumpGlobalParams.outputDirectory = getParam("out"); + mongoDumpGlobalParams.snapShotQuery = false; + if (!hasParam("query") && !hasParam("dbpath") && !hasParam("forceTableScan")) { + mongoDumpGlobalParams.snapShotQuery = true; + } + + // Make the default db "" if it was not explicitly set + if (!params.count("db")) { + toolGlobalParams.db = ""; + } + + if (hasParam("dumpDbUsersAndRoles") && toolGlobalParams.db == "admin") { + return Status(ErrorCodes::BadValue, + "Cannot provide --dumpDbUsersAndRoles when dumping the admin db as " + "user and role definitions for the whole server are dumped by default " + "when dumping the admin db"); + } + + // Always dump users and roles if doing a full dump. If doing a db dump, only dump users + // and roles if --dumpDbUsersAndRoles provided or you're dumping the admin db. + mongoDumpGlobalParams.dumpUsersAndRoles = hasParam("dumpDbUsersAndRoles") || + (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty()) || + toolGlobalParams.db == "admin"; + + if (mongoDumpGlobalParams.outputDirectory == "-") { + // write output to standard error to avoid mangling output + // must happen early to avoid sending junk to stdout + toolGlobalParams.canUseStdout = false; + } + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongodump_options.h b/src/mongo/tools/mongodump_options.h new file mode 100644 index 00000000000..117178d0234 --- /dev/null +++ b/src/mongo/tools/mongodump_options.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoDumpGlobalParams { + std::string outputDirectory; + std::string query; + bool useOplog; + bool repair; + bool snapShotQuery; + bool dumpUsersAndRoles; + }; + + extern MongoDumpGlobalParams mongoDumpGlobalParams; + + Status addMongoDumpOptions(moe::OptionSection* options); + + void printMongoDumpHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoDumpOptions(const moe::Environment& params); + + Status storeMongoDumpOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongodump_options_init.cpp b/src/mongo/tools/mongodump_options_init.cpp new file mode 100644 index 00000000000..a8a9b34e270 --- /dev/null +++ b/src/mongo/tools/mongodump_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongodump_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoDumpOptions)(InitializerContext* context) { + return addMongoDumpOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoDumpOptions)(InitializerContext* context) { + if (!handlePreValidationMongoDumpOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoDumpOptions)(InitializerContext* context) { + Status ret = storeMongoDumpOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongoexport_options.cpp b/src/mongo/tools/mongoexport_options.cpp new file mode 100644 index 00000000000..71e952827f3 --- /dev/null +++ b/src/mongo/tools/mongoexport_options.cpp @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongoexport_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoExportGlobalParams mongoExportGlobalParams; + + Status addMongoExportOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addFieldOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("query", "query,q", moe::String, + "query filter, as a JSON string, e.g., '{x:{$gt:1}}'"); + + options->addOptionChaining("csv", "csv", moe::Switch, "export to csv instead of json"); + + options->addOptionChaining("out", "out,o", moe::String, + "output file; if not specified, stdout is used"); + + options->addOptionChaining("jsonArray", "jsonArray", moe::Switch, + "output to a json array rather than one object per line"); + + options->addOptionChaining("slaveOk", "slaveOk,k", moe::Bool, + "use secondaries for export if available, default true") + .setDefault(moe::Value(true)); + + options->addOptionChaining("forceTableScan", "forceTableScan", moe::Switch, + "force a table scan (do not use $snapshot)"); + + options->addOptionChaining("skip", "skip", moe::Int, "documents to skip, default 0") + .setDefault(moe::Value(0)); + + options->addOptionChaining("limit", "limit", moe::Int, + "limit the numbers of documents returned, default all") + .setDefault(moe::Value(0)); + + options->addOptionChaining("sort", "sort", moe::String, + "sort order, as a JSON string, e.g., '{x:1}'"); + + + return Status::OK(); + } + + void printMongoExportHelp(std::ostream* out) { + *out << "Export MongoDB data to CSV, TSV or JSON files.\n" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoExportOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoExportHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoExportOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + ret = storeFieldOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoExportGlobalParams.outputFile = getParam("out"); + mongoExportGlobalParams.outputFileSpecified = hasParam("out"); + mongoExportGlobalParams.csv = hasParam("csv"); + mongoExportGlobalParams.jsonArray = hasParam("jsonArray"); + mongoExportGlobalParams.query = getParam("query", ""); + mongoExportGlobalParams.snapShotQuery = false; + + // Only allow snapshot query (requires _id idx scan) if following conditions are false + if (!hasParam("query") && + !hasParam("sort") && + !hasParam("dbpath") && + !hasParam("forceTableScan")) { + mongoExportGlobalParams.snapShotQuery = true; + } + + mongoExportGlobalParams.slaveOk = params["slaveOk"].as<bool>(); + mongoExportGlobalParams.limit = getParam("limit", 0); + mongoExportGlobalParams.skip = getParam("skip", 0); + mongoExportGlobalParams.sort = getParam("sort", ""); + + // we write output to standard error by default to avoid mangling output, but we don't need + // to do this if an output file was specified + toolGlobalParams.canUseStdout = false; + if (mongoExportGlobalParams.outputFileSpecified) { + if (mongoExportGlobalParams.outputFile != "-") { + toolGlobalParams.canUseStdout = true; + } + } + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongoexport_options.h b/src/mongo/tools/mongoexport_options.h new file mode 100644 index 00000000000..1db7206b462 --- /dev/null +++ b/src/mongo/tools/mongoexport_options.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoExportGlobalParams { + std::string query; + bool csv; + std::string outputFile; + bool outputFileSpecified; + bool jsonArray; + bool slaveOk; + bool snapShotQuery; + unsigned int skip; + unsigned int limit; + std::string sort; + }; + + extern MongoExportGlobalParams mongoExportGlobalParams; + + Status addMongoExportOptions(moe::OptionSection* options); + + void printMongoExportHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoExportOptions(const moe::Environment& params); + + Status storeMongoExportOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongoexport_options_init.cpp b/src/mongo/tools/mongoexport_options_init.cpp new file mode 100644 index 00000000000..02c706a9a92 --- /dev/null +++ b/src/mongo/tools/mongoexport_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongoexport_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoExportOptions)(InitializerContext* context) { + return addMongoExportOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoExportOptions)(InitializerContext* context) { + if (!handlePreValidationMongoExportOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoExportOptions)(InitializerContext* context) { + Status ret = storeMongoExportOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongofiles_options.cpp b/src/mongo/tools/mongofiles_options.cpp new file mode 100644 index 00000000000..1e423449fe6 --- /dev/null +++ b/src/mongo/tools/mongofiles_options.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongofiles_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoFilesGlobalParams mongoFilesGlobalParams; + + Status addMongoFilesOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("local", "local,l", moe::String, + "local filename for put|get (default is to use the same name as " + "'gridfs filename')"); + + options->addOptionChaining("type", "type,t", moe::String, + "MIME type for put (default is to omit)"); + + options->addOptionChaining("replace", "replace,r", moe::Switch, + "Remove other files with same name after PUT"); + + options->addOptionChaining("command", "command", moe::String, + "gridfs command to run") + .hidden() + .setSources(moe::SourceCommandLine) + .positional(1, 1); + + options->addOptionChaining("file", "file", moe::String, + "'gridfs filename' with a special meaning for various commands") + .hidden() + .setSources(moe::SourceCommandLine) + .positional(2, 2); + + + return Status::OK(); + } + + void printMongoFilesHelp(std::ostream* out) { + *out << "Browse and modify a GridFS filesystem.\n" << std::endl; + *out << "usage: mongofiles [options] command [gridfs filename]" << std::endl; + *out << "command:" << std::endl; + *out << " one of (list|search|put|get)" << std::endl; + *out << " list - list all files. 'gridfs filename' is an optional prefix " << std::endl; + *out << " which listed filenames must begin with." << std::endl; + *out << " search - search all files. 'gridfs filename' is a substring " << std::endl; + *out << " which listed filenames must contain." << std::endl; + *out << " put - add a file with filename 'gridfs filename'" << std::endl; + *out << " get - get a file with filename 'gridfs filename'" << std::endl; + *out << " delete - delete all files with filename 'gridfs filename'" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoFilesOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoFilesHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoFilesOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoFilesGlobalParams.command = getParam("command"); + mongoFilesGlobalParams.gridFSFilename = getParam("file"); + mongoFilesGlobalParams.localFile = getParam("local", mongoFilesGlobalParams.gridFSFilename); + mongoFilesGlobalParams.contentType = getParam("type", ""); + mongoFilesGlobalParams.replace = hasParam("replace"); + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongofiles_options.h b/src/mongo/tools/mongofiles_options.h new file mode 100644 index 00000000000..52a1ac9f187 --- /dev/null +++ b/src/mongo/tools/mongofiles_options.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoFilesGlobalParams { + std::string localFile; + std::string contentType; + bool replace; + std::string command; + std::string gridFSFilename; + }; + + extern MongoFilesGlobalParams mongoFilesGlobalParams; + + Status addMongoFilesOptions(moe::OptionSection* options); + + void printMongoFilesHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoFilesOptions(const moe::Environment& params); + + Status storeMongoFilesOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongofiles_options_init.cpp b/src/mongo/tools/mongofiles_options_init.cpp new file mode 100644 index 00000000000..9116ff188b5 --- /dev/null +++ b/src/mongo/tools/mongofiles_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongofiles_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoFilesOptions)(InitializerContext* context) { + return addMongoFilesOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoFilesOptions)(InitializerContext* context) { + if (!handlePreValidationMongoFilesOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoFilesOptions)(InitializerContext* context) { + Status ret = storeMongoFilesOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongoimport_options.cpp b/src/mongo/tools/mongoimport_options.cpp new file mode 100644 index 00000000000..0c076594273 --- /dev/null +++ b/src/mongo/tools/mongoimport_options.cpp @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongoimport_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" +#include "mongo/util/text.h" + +namespace mongo { + + MongoImportGlobalParams mongoImportGlobalParams; + + Status addMongoImportOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addFieldOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("ignoreBlanks", "ignoreBlanks", moe::Switch, + "if given, empty fields in csv and tsv will be ignored"); + + options->addOptionChaining("type", "type", moe::String, + "type of file to import. default: json (json,csv,tsv)"); + + options->addOptionChaining("file", "file", moe::String, + "file to import from; if not specified stdin is used") + .positional(1, 1); + + options->addOptionChaining("drop", "drop", moe::Switch, "drop collection first "); + + options->addOptionChaining("headerline", "headerline", moe::Switch, + "first line in input file is a header (CSV and TSV only)"); + + options->addOptionChaining("upsert", "upsert", moe::Switch, + "insert or update objects that already exist"); + + options->addOptionChaining("upsertFields", "upsertFields", moe::String, + "comma-separated fields for the query part of the upsert. " + "You should make sure this is indexed"); + + options->addOptionChaining("stopOnError", "stopOnError", moe::Switch, + "stop importing at first error rather than continuing"); + + options->addOptionChaining("jsonArray", "jsonArray", moe::Switch, + "load a json array, not one item per line. Currently limited to 16MB."); + + + options->addOptionChaining("noimport", "noimport", moe::Switch, + "don't actually import. useful for benchmarking parser") + .hidden(); + + + return Status::OK(); + } + + void printMongoImportHelp(std::ostream* out) { + *out << "Import CSV, TSV or JSON data into MongoDB.\n" << std::endl; + *out << "When importing JSON documents, each document must be a separate line of the input file.\n"; + *out << "\nExample:\n"; + *out << " mongoimport --host myhost --db my_cms --collection docs < mydocfile.json\n" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoImportOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoImportHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoImportOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + ret = storeFieldOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoImportGlobalParams.filename = getParam("file"); + mongoImportGlobalParams.drop = hasParam("drop"); + mongoImportGlobalParams.ignoreBlanks = hasParam("ignoreBlanks"); + + if (hasParam("upsert") || hasParam("upsertFields")) { + mongoImportGlobalParams.upsert = true; + + string uf = getParam("upsertFields"); + if (uf.empty()) { + mongoImportGlobalParams.upsertFields.push_back("_id"); + } + else { + StringSplitter(uf.c_str(), ",").split(mongoImportGlobalParams.upsertFields); + } + } + else { + mongoImportGlobalParams.upsert = false; + } + + mongoImportGlobalParams.doimport = !hasParam("noimport"); + mongoImportGlobalParams.type = getParam("type", "json"); + mongoImportGlobalParams.jsonArray = hasParam("jsonArray"); + mongoImportGlobalParams.headerLine = hasParam("headerline"); + mongoImportGlobalParams.stopOnError = hasParam("stopOnError"); + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongoimport_options.h b/src/mongo/tools/mongoimport_options.h new file mode 100644 index 00000000000..13bbd5f05a2 --- /dev/null +++ b/src/mongo/tools/mongoimport_options.h @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoImportGlobalParams { + bool ignoreBlanks; + std::string type; + std::string filename; + bool drop; + bool headerLine; + bool upsert; + std::vector<std::string> upsertFields; + bool stopOnError; + bool jsonArray; + bool doimport; + }; + + extern MongoImportGlobalParams mongoImportGlobalParams; + + Status addMongoImportOptions(moe::OptionSection* options); + + void printMongoImportHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoImportOptions(const moe::Environment& params); + + Status storeMongoImportOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongoimport_options_init.cpp b/src/mongo/tools/mongoimport_options_init.cpp new file mode 100644 index 00000000000..c80d4ecc4b4 --- /dev/null +++ b/src/mongo/tools/mongoimport_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongoimport_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoImportOptions)(InitializerContext* context) { + return addMongoImportOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoImportOptions)(InitializerContext* context) { + if (!handlePreValidationMongoImportOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoImportOptions)(InitializerContext* context) { + Status ret = storeMongoImportOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongooplog_options.cpp b/src/mongo/tools/mongooplog_options.cpp new file mode 100644 index 00000000000..e67a33b9710 --- /dev/null +++ b/src/mongo/tools/mongooplog_options.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongooplog_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/log.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoOplogGlobalParams mongoOplogGlobalParams; + + Status addMongoOplogOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("seconds", "seconds,s", moe::Int, + "seconds to go back default:86400"); + + options->addOptionChaining("from", "from", moe::String, "host to pull from"); + + options->addOptionChaining("oplogns", "oplogns", moe::String, "ns to pull from") + .setDefault(moe::Value(std::string("local.oplog.rs"))); + + + return Status::OK(); + } + + void printMongoOplogHelp(std::ostream* out) { + *out << "Pull and replay a remote MongoDB oplog.\n" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoOplogOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoOplogHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoOplogOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + if (!hasParam("from")) { + return Status(ErrorCodes::BadValue, "need to specify --from"); + } + else { + mongoOplogGlobalParams.from = getParam("from"); + } + + mongoOplogGlobalParams.seconds = getParam("seconds", 86400); + mongoOplogGlobalParams.ns = getParam("oplogns"); + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongooplog_options.h b/src/mongo/tools/mongooplog_options.h new file mode 100644 index 00000000000..84d53f75a15 --- /dev/null +++ b/src/mongo/tools/mongooplog_options.h @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoOplogGlobalParams { + int seconds; + std::string from; + std::string ns; + }; + + extern MongoOplogGlobalParams mongoOplogGlobalParams; + + Status addMongoOplogOptions(moe::OptionSection* options); + + void printMongoOplogHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoOplogOptions(const moe::Environment& params); + + Status storeMongoOplogOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongooplog_options_init.cpp b/src/mongo/tools/mongooplog_options_init.cpp new file mode 100644 index 00000000000..2e71d57bcf8 --- /dev/null +++ b/src/mongo/tools/mongooplog_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongooplog_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoOplogOptions)(InitializerContext* context) { + return addMongoOplogOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoOplogOptions)(InitializerContext* context) { + if (!handlePreValidationMongoOplogOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoOplogOptions)(InitializerContext* context) { + Status ret = storeMongoOplogOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongorestore_options.cpp b/src/mongo/tools/mongorestore_options.cpp new file mode 100644 index 00000000000..2ee83dbcfff --- /dev/null +++ b/src/mongo/tools/mongorestore_options.cpp @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongorestore_options.h" + +#include "mongo/base/status.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoRestoreGlobalParams mongoRestoreGlobalParams; + + Status addMongoRestoreOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addLocalServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addSpecifyDBCollectionToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addBSONToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("drop", "drop", moe::Switch, + "drop each collection before import"); + + options->addOptionChaining("oplogReplay", "oplogReplay", moe::Switch, + "replay oplog for point-in-time restore"); + + options->addOptionChaining("oplogLimit", "oplogLimit", moe::String, + "include oplog entries before the provided Timestamp " + "(seconds[:ordinal]) during the oplog replay; " + "the ordinal value is optional"); + + options->addOptionChaining("keepIndexVersion", "keepIndexVersion", moe::Switch, + "don't upgrade indexes to newest version"); + + options->addOptionChaining("noOptionsRestore", "noOptionsRestore", moe::Switch, + "don't restore collection options"); + + options->addOptionChaining("noIndexRestore", "noIndexRestore", moe::Switch, + "don't restore indexes"); + + options->addOptionChaining("restoreDbUsersAndRoles", "restoreDbUsersAndRoles", moe::Switch, + "Restore user and role definitions for the given database") + .requires("db").incompatibleWith("collection"); + + options->addOptionChaining( + "tempUsersCollection", "tempUsersCollection", moe::String, + "Collection in which to temporarily store user data during the restore") + .hidden() + .setDefault(moe::Value( + AuthorizationManager::defaultTempUsersCollectionNamespace.toString())); + + options->addOptionChaining( + "tempRolesCollection", "tempRolesCollection", moe::String, + "Collection in which to temporarily store role data during the restore") + .hidden() + .setDefault(moe::Value( + AuthorizationManager::defaultTempRolesCollectionNamespace.toString())); + + options->addOptionChaining("w", "w", moe::Int, "minimum number of replicas per write") + .setDefault(moe::Value(0)); + + options->addOptionChaining("dir", "dir", moe::String, "directory to restore from") + .hidden() + .setDefault(moe::Value(std::string("dump"))) + .positional(1, 1); + + + // left in for backwards compatibility + options->addOptionChaining("indexesLast", "indexesLast", moe::Switch, + "wait to add indexes (now default)") + .hidden(); + + + return Status::OK(); + } + + void printMongoRestoreHelp(std::ostream* out) { + *out << "Import BSON files into MongoDB.\n" << std::endl; + *out << "usage: mongorestore [options] [directory or filename to restore from]" + << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoRestoreOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoRestoreHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoRestoreOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + ret = storeBSONToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoRestoreGlobalParams.restoreDirectory = getParam("dir"); + mongoRestoreGlobalParams.drop = hasParam("drop"); + mongoRestoreGlobalParams.keepIndexVersion = hasParam("keepIndexVersion"); + mongoRestoreGlobalParams.restoreOptions = !hasParam("noOptionsRestore"); + mongoRestoreGlobalParams.restoreIndexes = !hasParam("noIndexRestore"); + mongoRestoreGlobalParams.w = getParam( "w" , 0 ); + mongoRestoreGlobalParams.oplogReplay = hasParam("oplogReplay"); + mongoRestoreGlobalParams.oplogLimit = getParam("oplogLimit", ""); + mongoRestoreGlobalParams.tempUsersColl = getParam("tempUsersCollection"); + mongoRestoreGlobalParams.tempRolesColl = getParam("tempRolesCollection"); + + // Make the default db "" if it was not explicitly set + if (!params.count("db")) { + toolGlobalParams.db = ""; + } + + if (hasParam("restoreDbUsersAndRoles") && toolGlobalParams.db == "admin") { + return Status(ErrorCodes::BadValue, + "Cannot provide --restoreDbUsersAndRoles when restoring the admin db as " + "user and role definitions for the whole server are restored by " + "default (if present) when restoring the admin db"); + } + + // Always restore users and roles if doing a full restore. If doing a db restore, only + // restore users and roles if --restoreDbUsersAndRoles provided or you're restoring the + // admin db + mongoRestoreGlobalParams.restoreUsersAndRoles = hasParam("restoreDbUsersAndRoles") || + (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty()) || + (toolGlobalParams.db == "admin" && toolGlobalParams.coll.empty()); + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongorestore_options.h b/src/mongo/tools/mongorestore_options.h new file mode 100644 index 00000000000..f0208952110 --- /dev/null +++ b/src/mongo/tools/mongorestore_options.h @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoRestoreGlobalParams { + bool drop; + bool oplogReplay; + std::string oplogLimit; + bool keepIndexVersion; + bool restoreOptions; + bool restoreIndexes; + bool restoreUsersAndRoles; + int w; + std::string restoreDirectory; + std::string tempUsersColl; + std::string tempRolesColl; + }; + + extern MongoRestoreGlobalParams mongoRestoreGlobalParams; + + Status addMongoRestoreOptions(moe::OptionSection* options); + + void printMongoRestoreHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoRestoreOptions(const moe::Environment& params); + + Status storeMongoRestoreOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongorestore_options_init.cpp b/src/mongo/tools/mongorestore_options_init.cpp new file mode 100644 index 00000000000..f0a97bf390c --- /dev/null +++ b/src/mongo/tools/mongorestore_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongorestore_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoRestoreOptions)(InitializerContext* context) { + return addMongoRestoreOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoRestoreOptions)(InitializerContext* context) { + if (!handlePreValidationMongoRestoreOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoRestoreOptions)(InitializerContext* context) { + Status ret = storeMongoRestoreOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongostat_options.cpp b/src/mongo/tools/mongostat_options.cpp new file mode 100644 index 00000000000..5a6174a9c9b --- /dev/null +++ b/src/mongo/tools/mongostat_options.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongostat_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoStatGlobalParams mongoStatGlobalParams; + + Status addMongoStatOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("noheaders", "noheaders", moe::Switch, + "don't output column names"); + + options->addOptionChaining("rowcount", "rowcount,n", moe::Int, + "number of stats lines to print (0 for indefinite)") + .setDefault(moe::Value(0)); + + options->addOptionChaining("http", "http", moe::Switch, + "use http instead of raw db connection"); + + options->addOptionChaining("discover", "discover", moe::Switch, + "discover nodes and display stats for all"); + + options->addOptionChaining("all", "all", moe::Switch, "all optional fields"); + + options->addOptionChaining("sleep", "sleep", moe::Int, "seconds to sleep between samples") + .hidden() + .setSources(moe::SourceCommandLine) + .positional(1, 1); + + + return Status::OK(); + } + + void printMongoStatHelp(std::ostream* out) { + *out << "View live MongoDB performance statistics.\n" << std::endl; + *out << "usage: mongostat [options] [sleep time]" << std::endl; + *out << "sleep time: time to wait (in seconds) between calls" << std::endl; + *out << moe::startupOptions.helpString(); + *out << "\n"; + *out << " Fields\n"; + *out << " inserts \t- # of inserts per second (* means replicated op)\n"; + *out << " query \t- # of queries per second\n"; + *out << " update \t- # of updates per second\n"; + *out << " delete \t- # of deletes per second\n"; + *out << " getmore \t- # of get mores (cursor batch) per second\n"; + *out << " command \t- # of commands per second, on a slave its local|replicated\n"; + *out << " flushes \t- # of fsync flushes per second\n"; + *out << " mapped \t- amount of data mmaped (total data size) megabytes\n"; + *out << " vsize \t- virtual size of process in megabytes\n"; + *out << " res \t- resident size of process in megabytes\n"; + *out << " non-mapped \t- amount virtual memeory less mapped memory (only with --all)\n"; + *out << " faults \t- # of pages faults per sec\n"; + *out << " locked \t- name of and percent time for most locked database\n"; + *out << " idx miss \t- percent of btree page misses (sampled)\n"; + *out << " qr|qw \t- queue lengths for clients waiting (read|write)\n"; + *out << " ar|aw \t- active clients (read|write)\n"; + *out << " netIn \t- network traffic in - bytes\n"; + *out << " netOut \t- network traffic out - bytes\n"; + *out << " conn \t- number of open connections\n"; + *out << " set \t- replica set name\n"; + *out << " repl \t- replication type \n"; + *out << " \t PRI - primary (master)\n"; + *out << " \t SEC - secondary\n"; + *out << " \t REC - recovering\n"; + *out << " \t UNK - unknown\n"; + *out << " \t SLV - slave\n"; + *out << " b\t RTR - mongos process (\"router\")\n"; + *out << std::flush; + } + + bool handlePreValidationMongoStatOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoStatHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoStatOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + if (hasParam("http")) { + mongoStatGlobalParams.http = true; + toolGlobalParams.noconnection = true; + } + + if (hasParam("host") && getParam("host").find(',') != string::npos) { + toolGlobalParams.noconnection = true; + mongoStatGlobalParams.many = true; + } + + if (hasParam("discover")) { + mongoStatGlobalParams.discover = true; + mongoStatGlobalParams.many = true; + } + + mongoStatGlobalParams.showHeaders = !hasParam("noheaders"); + mongoStatGlobalParams.rowCount = getParam("rowcount", 0); + mongoStatGlobalParams.sleep = getParam("sleep", 1); + mongoStatGlobalParams.allFields = hasParam("all"); + + // Make the default db "admin" if it was not explicitly set + if (!params.count("db")) { + toolGlobalParams.db = "admin"; + } + + // end of storage / start of validation + + if (mongoStatGlobalParams.sleep <= 0) { + return Status(ErrorCodes::BadValue, + "Error parsing command line: --sleep must be greater than 0"); + } + + if (mongoStatGlobalParams.rowCount < 0) { + return Status(ErrorCodes::BadValue, + "Error parsing command line: --rowcount (-n) can't be negative"); + } + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongostat_options.h b/src/mongo/tools/mongostat_options.h new file mode 100644 index 00000000000..0787ed585d9 --- /dev/null +++ b/src/mongo/tools/mongostat_options.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoStatGlobalParams { + bool showHeaders; + int rowCount; + bool http; + bool discover; + bool many; + bool allFields; + int sleep; + std::string url; + }; + + extern MongoStatGlobalParams mongoStatGlobalParams; + + Status addMongoStatOptions(moe::OptionSection* options); + + void printMongoStatHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoStatOptions(const moe::Environment& params); + + Status storeMongoStatOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongostat_options_init.cpp b/src/mongo/tools/mongostat_options_init.cpp new file mode 100644 index 00000000000..6f1eba04c82 --- /dev/null +++ b/src/mongo/tools/mongostat_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongostat_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoStatOptions)(InitializerContext* context) { + return addMongoStatOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoStatOptions)(InitializerContext* context) { + if (!handlePreValidationMongoStatOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoStatOptions)(InitializerContext* context) { + Status ret = storeMongoStatOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/mongotop_options.cpp b/src/mongo/tools/mongotop_options.cpp new file mode 100644 index 00000000000..4c128364d2f --- /dev/null +++ b/src/mongo/tools/mongotop_options.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongotop_options.h" + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + + MongoTopGlobalParams mongoTopGlobalParams; + + Status addMongoTopOptions(moe::OptionSection* options) { + Status ret = addGeneralToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + ret = addRemoteServerToolOptions(options); + if (!ret.isOK()) { + return ret; + } + + options->addOptionChaining("locks", "locks", moe::Switch, + "use db lock info instead of top"); + + options->addOptionChaining("sleep", "sleep", moe::Int, "seconds to sleep between samples") + .hidden() + .setSources(moe::SourceCommandLine) + .positional(1, 1); + + + return Status::OK(); + } + + void printMongoTopHelp(std::ostream* out) { + *out << "View live MongoDB collection statistics.\n" << std::endl; + *out << moe::startupOptions.helpString(); + *out << std::flush; + } + + bool handlePreValidationMongoTopOptions(const moe::Environment& params) { + if (!handlePreValidationGeneralToolOptions(params)) { + return false; + } + if (params.count("help")) { + printMongoTopHelp(&std::cout); + return false; + } + return true; + } + + Status storeMongoTopOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + Status ret = storeGeneralToolOptions(params, args); + if (!ret.isOK()) { + return ret; + } + + mongoTopGlobalParams.sleep = getParam("sleep", 1); + mongoTopGlobalParams.useLocks = hasParam("locks"); + + // Make the default db "admin" if it was not explicitly set + if (!params.count("db")) { + toolGlobalParams.db = "admin"; + } + + return Status::OK(); + } + +} diff --git a/src/mongo/tools/mongotop_options.h b/src/mongo/tools/mongotop_options.h new file mode 100644 index 00000000000..5f9e6b1fb4f --- /dev/null +++ b/src/mongo/tools/mongotop_options.h @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" +#include "mongo/tools/tool_options.h" + +namespace mongo { + + struct MongoTopGlobalParams { + bool useLocks; + int sleep; + }; + + extern MongoTopGlobalParams mongoTopGlobalParams; + + Status addMongoTopOptions(moe::OptionSection* options); + + void printMongoTopHelp(std::ostream* out); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationMongoTopOptions(const moe::Environment& params); + + Status storeMongoTopOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/mongotop_options_init.cpp b/src/mongo/tools/mongotop_options_init.cpp new file mode 100644 index 00000000000..965090029b7 --- /dev/null +++ b/src/mongo/tools/mongotop_options_init.cpp @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/mongotop_options.h" + +#include "mongo/util/options_parser/startup_option_init.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace mongo { + MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoTopOptions)(InitializerContext* context) { + return addMongoTopOptions(&moe::startupOptions); + } + + MONGO_STARTUP_OPTIONS_VALIDATE(MongoTopOptions)(InitializerContext* context) { + if (!handlePreValidationMongoTopOptions(moe::startupOptionsParsed)) { + ::_exit(EXIT_SUCCESS); + } + Status ret = moe::startupOptionsParsed.validate(); + if (!ret.isOK()) { + return ret; + } + return Status::OK(); + } + + MONGO_STARTUP_OPTIONS_STORE(MongoTopOptions)(InitializerContext* context) { + Status ret = storeMongoTopOptions(moe::startupOptionsParsed, context->args()); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" + << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); + } +} diff --git a/src/mongo/tools/oplog.cpp b/src/mongo/tools/oplog.cpp index ea7f2af5b13..d8eedb42f2e 100644 --- a/src/mongo/tools/oplog.cpp +++ b/src/mongo/tools/oplog.cpp @@ -1,5 +1,3 @@ -// oplog.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,76 +12,76 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" -#include "mongo/base/initializer.h" -#include "db/json.h" -#include "db/oplogreader.h" - -#include "tool.h" +#include "mongo/pch.h" #include <fstream> #include <iostream> -#include <boost/program_options.hpp> +#include "mongo/db/json.h" +#include "mongo/db/repl/oplogreader.h" +#include "mongo/tools/mongooplog_options.h" +#include "mongo/tools/tool.h" +#include "mongo/util/options_parser/option_section.h" using namespace mongo; -namespace po = boost::program_options; - class OplogTool : public Tool { public: - OplogTool() : Tool( "oplog" ) { - add_options() - ("seconds,s" , po::value<int>() , "seconds to go back default:86400" ) - ("from", po::value<string>() , "host to pull from" ) - ("oplogns", po::value<string>()->default_value( "local.oplog.rs" ) , "ns to pull from" ) - ; - } + OplogTool() : Tool() { } - virtual void printExtraHelp(ostream& out) { - out << "Pull and replay a remote MongoDB oplog.\n" << endl; + virtual void printHelp( ostream & out ) { + printMongoOplogHelp(&out); } int run() { - if ( ! hasParam( "from" ) ) { - log() << "need to specify --from" << endl; - return -1; - } - Client::initThread( "oplogreplay" ); - log() << "going to connect" << endl; + toolInfoLog() << "going to connect" << std::endl; - OplogReader r(false); + OplogReader r; r.setTailingQueryOptions( QueryOption_SlaveOk | QueryOption_AwaitData ); - r.connect( getParam( "from" ) ); + r.connect(mongoOplogGlobalParams.from); - log() << "connected" << endl; + toolInfoLog() << "connected" << std::endl; - OpTime start( time(0) - getParam( "seconds" , 86400 ) , 0 ); - log() << "starting from " << start.toStringPretty() << endl; + OpTime start(time(0) - mongoOplogGlobalParams.seconds, 0); + toolInfoLog() << "starting from " << start.toStringPretty() << std::endl; - string ns = getParam( "oplogns" ); - r.tailingQueryGTE( ns.c_str() , start ); + r.tailingQueryGTE(mongoOplogGlobalParams.ns.c_str(), start); int num = 0; while ( r.more() ) { BSONObj o = r.next(); - LOG(2) << o << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2))) { + toolInfoLog() << o << std::endl; + } if ( o["$err"].type() ) { - log() << "error getting oplog" << endl; - log() << o << endl; + toolError() << "error getting oplog" << std::endl; + toolError() << o << std::endl; return -1; } bool print = ++num % 100000 == 0; - if ( print ) - cout << num << "\t" << o << endl; + if (print) { + toolInfoLog() << num << "\t" << o << std::endl; + } if ( o["op"].String() == "n" ) continue; @@ -97,16 +95,15 @@ public: BSONObj res; bool ok = conn().runCommand( "admin" , c , res ); - if ( print || ! ok ) - log() << res << endl; + if (!ok) { + toolError() << res << std::endl; + } else if (print) { + toolInfoLog() << res << std::endl; + } } return 0; } }; -int main( int argc , char** argv, char** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - OplogTool t; - return t.main( argc , argv ); -} +REGISTER_MONGO_TOOL(OplogTool); diff --git a/src/mongo/tools/restore.cpp b/src/mongo/tools/restore.cpp index abd7f868afe..27cd6a2c785 100644 --- a/src/mongo/tools/restore.cpp +++ b/src/mongo/tools/restore.cpp @@ -1,5 +1,3 @@ -// @file restore.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,150 +12,230 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" #include <boost/filesystem/convenience.hpp> #include <boost/filesystem/operations.hpp> -#include <boost/program_options.hpp> -#include <boost/scoped_ptr.hpp> #include <boost/lexical_cast.hpp> +#include <boost/scoped_ptr.hpp> #include <fcntl.h> #include <fstream> #include <set> -#include "mongo/base/initializer.h" -#include "mongo/db/namespacestring.h" +#include "mongo/base/init.h" +#include "mongo/bson/util/bson_extract.h" +#include "mongo/client/auth_helpers.h" +#include "mongo/client/dbclientcursor.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authz_manager_external_state_d.h" +#include "mongo/db/auth/user_name.h" +#include "mongo/db/auth/role_name.h" +#include "mongo/db/json.h" +#include "mongo/db/namespace_string.h" +#include "mongo/tools/mongorestore_options.h" #include "mongo/tools/tool.h" #include "mongo/util/mmap.h" +#include "mongo/util/options_parser/option_section.h" #include "mongo/util/stringutils.h" -#include "mongo/db/json.h" -#include "mongo/client/dbclientcursor.h" using namespace mongo; -namespace po = boost::program_options; - namespace { const char* OPLOG_SENTINEL = "$oplog"; // compare by ptr not strcmp } +MONGO_INITIALIZER_WITH_PREREQUISITES(RestoreAuthExternalState, ("ToolAuthExternalState"))( + InitializerContext* context) { + // Give restore the mongod implementation of AuthorizationManager so that it can run + // the _mergeAuthzCollections command directly against the data files + clearGlobalAuthorizationManager(); + setGlobalAuthorizationManager(new AuthorizationManager( + new AuthzManagerExternalStateMongod())); + + return Status::OK(); +} + class Restore : public BSONTool { public: - bool _drop; - bool _keepIndexVersion; - bool _restoreOptions; - bool _restoreIndexes; - int _w; string _curns; string _curdb; string _curcoll; - set<string> _users; // For restoring users with --drop + string _serverBinVersion; // Version identifier of the server we're restoring to + set<UserName> _users; // Holds users that are already in the cluster when restoring with --drop + set<RoleName> _roles; // Holds roles that are already in the cluster when restoring with --drop scoped_ptr<Matcher> _opmatcher; // For oplog replay scoped_ptr<OpTime> _oplogLimitTS; // for oplog replay (limit) int _oplogEntrySkips; // oplog entries skipped int _oplogEntryApplies; // oplog entries applied - Restore() : BSONTool( "restore" ) , _drop(false) { - // Default values set here will show up in help text, but will supercede any default value - // used when calling getParam below. - add_options() - ("drop" , "drop each collection before import" ) - ("oplogReplay", "replay oplog for point-in-time restore") - ("oplogLimit", po::value<string>(), "include oplog entries before the provided Timestamp " - "(seconds[:ordinal]) during the oplog replay; the ordinal value is optional") - ("keepIndexVersion" , "don't upgrade indexes to newest version") - ("noOptionsRestore" , "don't restore collection options") - ("noIndexRestore" , "don't restore indexes") - ("w" , po::value<int>()->default_value(0) , "minimum number of replicas per write" ) - ; - add_hidden_options() - ("dir", po::value<string>()->default_value("dump"), "directory to restore from") - ("indexesLast" , "wait to add indexes (now default)") // left in for backwards compatibility - ; - addPositionArg("dir", 1); + int _serverAuthzVersion; // authSchemaVersion of the cluster being restored into. + int _dumpFileAuthzVersion; // version extracted from admin.system.version file in dump. + bool _serverAuthzVersionDocExists; // Whether the remote cluster has an admin.system.version doc + Restore() : BSONTool(), _oplogEntrySkips(0), _oplogEntryApplies(0), _serverAuthzVersion(0), + _dumpFileAuthzVersion(0), _serverAuthzVersionDocExists(false) { } + + virtual void printHelp(ostream& out) { + printMongoRestoreHelp(&out); } - virtual void printExtraHelp(ostream& out) { - out << "Import BSON files into MongoDB.\n" << endl; - out << "usage: " << _name << " [options] [directory or filename to restore from]" << endl; + void storeRemoteAuthzVersion() { + Status status = auth::getRemoteStoredAuthorizationVersion(&conn(), + &_serverAuthzVersion); + uassertStatusOK(status); + uassert(17370, + mongoutils::str::stream() << "Restoring users and roles is only supported for " + "clusters with auth schema versions " << + AuthorizationManager::schemaVersion24 << " or " << + AuthorizationManager::schemaVersion26Final << ", found: " << + _serverAuthzVersion, + _serverAuthzVersion == AuthorizationManager::schemaVersion24 || + _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); + + _serverAuthzVersionDocExists = !conn().findOne( + AuthorizationManager::versionCollectionNamespace, + AuthorizationManager::versionDocumentQuery).isEmpty(); } virtual int doRun() { - boost::filesystem::path root = getParam("dir"); + boost::filesystem::path root = mongoRestoreGlobalParams.restoreDirectory; // check if we're actually talking to a machine that can write if (!isMaster()) { return -1; } - if (isMongos() && _db == "" && exists(root / "config")) { - log() << "Cannot do a full restore on a sharded system" << endl; + if (isMongos() && toolGlobalParams.db == "" && exists(root / "config")) { + toolError() << "Cannot do a full restore on a sharded system" << std::endl; return -1; } - _drop = hasParam( "drop" ); - _keepIndexVersion = hasParam("keepIndexVersion"); - _restoreOptions = !hasParam("noOptionsRestore"); - _restoreIndexes = !hasParam("noIndexRestore"); - // Make sure default value set here stays in sync with the one set in the constructor above. - _w = getParam( "w" , 0 ); + { + // Store server's version + BSONObj out; + if (! conn().simpleCommand("admin", &out, "buildinfo")) { + toolError() << "buildinfo command failed: " + << out["errmsg"].String() << std::endl; + return -1; + } + + _serverBinVersion = out["version"].String(); + } + + if (mongoRestoreGlobalParams.restoreUsersAndRoles) { + storeRemoteAuthzVersion(); // populate _serverAuthzVersion + + if (_serverAuthzVersion == AuthorizationManager::schemaVersion26Final) { + uassert(17410, + mongoutils::str::stream() << mongoRestoreGlobalParams.tempUsersColl << + " collection already exists, but is needed to restore user data. " + "Drop this collection or specify a different collection (via " + "--tempUsersColl) to use to temporarily hold user data during the " + "restore process", + !conn().exists(mongoRestoreGlobalParams.tempUsersColl)); + uassert(17411, + mongoutils::str::stream() << mongoRestoreGlobalParams.tempRolesColl << + " collection already exists, but is needed to restore role data. " + "Drop this collection or specify a different collection (via " + "--tempRolesColl) to use to temporarily hold role data during the " + "restore process", + !conn().exists(mongoRestoreGlobalParams.tempRolesColl)); + } - bool doOplog = hasParam( "oplogReplay" ); + if (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty() && + exists(root / "admin" / "system.version.bson")) { + // Will populate _dumpFileAuthzVersion + processFileAndMetadata(root / "admin" / "system.version.bson", + "admin.system.version"); + } else if (!toolGlobalParams.db.empty()) { + // DB-specific restore + if (exists(root / "$admin.system.users.bson")) { + uassert(17372, + mongoutils::str::stream() << "$admin.system.users.bson file found, " + "which implies that the dump was taken from a system with " + "schema version " << AuthorizationManager::schemaVersion26Final + << " users, but server has authorization schema version " + << _serverAuthzVersion, + _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); + toolInfoLog() << "Restoring users for the " << toolGlobalParams.db << + " database to admin.system.users" << endl; + processFileAndMetadata(root / "$admin.system.users.bson", "admin.system.users"); + } + if (exists(root / "$admin.system.roles.bson")) { + uassert(17373, + mongoutils::str::stream() << "$admin.system.roles.bson file found, " + "which implies that the dump was taken from a system with " + "schema version " << AuthorizationManager::schemaVersion26Final + << " authorization data, but server has authorization schema " + "version " << _serverAuthzVersion, + _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); + toolInfoLog() << "Restoring roles for the " << toolGlobalParams.db << + " database to admin.system.roles" << endl; + processFileAndMetadata(root / "$admin.system.roles.bson", "admin.system.roles"); + } + } + } - if (doOplog) { + if (mongoRestoreGlobalParams.oplogReplay) { // fail early if errors - if (_db != "") { - log() << "Can only replay oplog on full restore" << endl; + if (toolGlobalParams.db != "") { + toolError() << "Can only replay oplog on full restore" << std::endl; return -1; } if ( ! exists(root / "oplog.bson") ) { - log() << "No oplog file to replay. Make sure you run mongodump with --oplog." << endl; - return -1; - } - - - BSONObj out; - if (! conn().simpleCommand("admin", &out, "buildinfo")) { - log() << "buildinfo command failed: " << out["errmsg"].String() << endl; + toolError() << "No oplog file to replay. Make sure you run mongodump with --oplog." + << std::endl; return -1; } - StringData version = out["version"].valuestr(); - if (versionCmp(version, "1.7.4-pre-") < 0) { - log() << "Can only replay oplog to server version >= 1.7.4" << endl; + if (versionCmp(_serverBinVersion, "1.7.4-pre-") < 0) { + toolError() << "Can only replay oplog to server version >= 1.7.4" << std::endl; return -1; } - string oplogLimit = getParam( "oplogLimit", "" ); string oplogInc = "0"; - if(!oplogLimit.empty()) { - size_t i = oplogLimit.find_first_of(':'); + if(!mongoRestoreGlobalParams.oplogLimit.empty()) { + size_t i = mongoRestoreGlobalParams.oplogLimit.find_first_of(':'); if ( i != string::npos ) { - if ( i + 1 < oplogLimit.length() ) { - oplogInc = oplogLimit.substr(i + 1); + if (i + 1 < mongoRestoreGlobalParams.oplogLimit.length()) { + oplogInc = mongoRestoreGlobalParams.oplogLimit.substr(i + 1); } - oplogLimit = oplogLimit.substr(0, i); + mongoRestoreGlobalParams.oplogLimit = + mongoRestoreGlobalParams.oplogLimit.substr(0, i); } try { _oplogLimitTS.reset(new OpTime( - boost::lexical_cast<unsigned long>(oplogLimit.c_str()), + boost::lexical_cast<unsigned long>( + mongoRestoreGlobalParams.oplogLimit.c_str()), boost::lexical_cast<unsigned long>(oplogInc.c_str()))); - } catch( const boost::bad_lexical_cast& error) { - log() << "Could not parse oplogLimit into Timestamp from values ( " - << oplogLimit << " , " << oplogInc << " )" - << endl; + } catch( const boost::bad_lexical_cast& ) { + toolError() << "Could not parse oplogLimit into Timestamp from values ( " + << mongoRestoreGlobalParams.oplogLimit << " , " << oplogInc << " )" + << std::endl; return -1; } - if (!oplogLimit.empty()) { + if (!mongoRestoreGlobalParams.oplogLimit.empty()) { // Only for a replica set as master will have no-op entries so we would need to // skip them all to find the real op scoped_ptr<DBClientCursor> cursor( @@ -168,9 +246,9 @@ public: if (cursor->more()) { tsOptime = cursor->next().getField("ts")._opTime(); if (tsOptime > *_oplogLimitTS.get()) { - log() << "The oplogLimit is not newer than" - << " the last oplog entry on the server." - << endl; + toolError() << "The oplogLimit is not newer than" + << " the last oplog entry on the server." + << std::endl; return -1; } } @@ -183,17 +261,17 @@ public: BSONObj query = BSON("ts" << tsRestrictBldr.obj()); if (!tsOptime.isNull()) { - log() << "Latest oplog entry on the server is " << tsOptime.getSecs() - << ":" << tsOptime.getInc() << endl; - log() << "Only applying oplog entries matching this criteria: " - << query.jsonString() << endl; + toolInfoLog() << "Latest oplog entry on the server is " << tsOptime.getSecs() + << ":" << tsOptime.getInc() << std::endl; + toolInfoLog() << "Only applying oplog entries matching this criteria: " + << query.jsonString() << std::endl; } _opmatcher.reset(new Matcher(query)); } } } - /* If _db is not "" then the user specified a db name to restore as. + /* If toolGlobalParams.db is not "" then the user specified a db name to restore as. * * In that case we better be given either a root directory that * contains only .bson files or a single .bson file (a db). @@ -202,21 +280,22 @@ public: * given either a root directory that contains only a single * .bson file, or a single .bson file itself (a collection). */ - drillDown(root, _db != "", _coll != "", !(_oplogLimitTS.get() == NULL), true); + drillDown(root, toolGlobalParams.db != "", toolGlobalParams.coll != "", + !(_oplogLimitTS.get() == NULL), true); // should this happen for oplog replay as well? - string err = conn().getLastError(_db == "" ? "admin" : _db); + string err = conn().getLastError(toolGlobalParams.db == "" ? "admin" : toolGlobalParams.db); if (!err.empty()) { - error() << err; + toolError() << err << std::endl; } - if (doOplog) { - log() << "\t Replaying oplog" << endl; + if (mongoRestoreGlobalParams.oplogReplay) { + toolInfoLog() << "\t Replaying oplog" << std::endl; _curns = OPLOG_SENTINEL; processFile( root / "oplog.bson" ); - log() << "Applied " << _oplogEntryApplies << " oplog entries out of " - << _oplogEntryApplies + _oplogEntrySkips << " (" << _oplogEntrySkips - << " skipped)." << endl; + toolInfoLog() << "Applied " << _oplogEntryApplies << " oplog entries out of " + << _oplogEntryApplies + _oplogEntrySkips << " (" << _oplogEntrySkips + << " skipped)." << std::endl; } return EXIT_CLEAN; @@ -228,7 +307,9 @@ public: bool oplogReplayLimit, bool top_level=false) { bool json_metadata = false; - LOG(2) << "drillDown: " << root.string() << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2))) { + toolInfoLog() << "drillDown: " << root.string() << std::endl; + } // skip hidden files and directories if (root.leaf().string()[0] == '.' && root.leaf().string() != ".") @@ -244,18 +325,23 @@ public: if (use_db) { if (boost::filesystem::is_directory(p)) { - error() << "ERROR: root directory must be a dump of a single database" << endl; - error() << " when specifying a db name with --db" << endl; - printHelp(cout); + toolError() << "ERROR: root directory must be a dump of a single database" + << std::endl; + toolError() << " when specifying a db name with --db" << std::endl; + toolError() << " use the --help option for more information" + << std::endl; return; } } if (use_coll) { if (boost::filesystem::is_directory(p) || i != end) { - error() << "ERROR: root directory must be a dump of a single collection" << endl; - error() << " when specifying a collection name with --collection" << endl; - printHelp(cout); + toolError() << "ERROR: root directory must be a dump of a single collection" + << std::endl; + toolError() << " when specifying a collection name with --collection" + << std::endl; + toolError() << " use the --help option for more information" + << std::endl; return; } } @@ -283,27 +369,16 @@ public: return; } - if ( endsWith( root.string().c_str() , ".metadata.json" ) ) { - // Metadata files are handled when the corresponding .bson file is handled - return; - } - - if ( ! ( endsWith( root.string().c_str() , ".bson" ) || - endsWith( root.string().c_str() , ".bin" ) ) ) { - error() << "don't know what to do with file [" << root.string() << "]" << endl; - return; - } - - log() << root.string() << endl; - - if ( root.leaf() == "system.profile.bson" ) { - log() << "\t skipping" << endl; - return; + if (oplogReplayLimit) { + toolError() << "The oplogLimit option cannot be used if " + << "normal databases/collections exist in the dump directory." + << std::endl; + exit(EXIT_FAILURE); } string ns; if (use_db) { - ns += _db; + ns += toolGlobalParams.db; } else { ns = root.parent_path().filename().string(); @@ -316,82 +391,224 @@ public: string oldCollName = root.leaf().string(); // Name of the collection that was dumped from oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); if (use_coll) { - ns += "." + _coll; + ns += "." + toolGlobalParams.coll; } else { ns += "." + oldCollName; } - if (oplogReplayLimit) { - error() << "The oplogLimit option cannot be used if " - << "normal databases/collections exist in the dump directory." - << endl; - exit(EXIT_FAILURE); + if ( endsWith( root.string().c_str() , ".metadata.json" ) ) { + // Metadata files are handled when the corresponding .bson file is handled + return; } - log() << "\tgoing into namespace [" << ns << "]" << endl; + if ((root.leaf() == "system.version.bson" && toolGlobalParams.db.empty()) || + root.leaf() == "$admin.system.users.bson" || + root.leaf() == "$admin.system.roles.bson") { + // These files were already explicitly handled at the beginning of the restore. + return; + } - if ( _drop ) { - if (root.leaf() != "system.users.bson" ) { - log() << "\t dropping" << endl; - conn().dropCollection( ns ); - } else { - // Create map of the users currently in the DB - BSONObj fields = BSON("user" << 1); - scoped_ptr<DBClientCursor> cursor(conn().query(ns, Query(), 0, 0, &fields)); - while (cursor->more()) { - BSONObj user = cursor->next(); - _users.insert(user["user"].String()); + if ( ! ( endsWith( root.string().c_str() , ".bson" ) || + endsWith( root.string().c_str() , ".bin" ) ) ) { + toolError() << "don't know what to do with file [" << root.string() << "]" << std::endl; + return; + } + + toolInfoLog() << root.string() << std::endl; + + if ( root.leaf() == "system.profile.bson" ) { + toolInfoLog() << "\t skipping system.profile.bson" << std::endl; + return; + } + + processFileAndMetadata(root, ns); + } + + static std::string getMessageAboutBrokenDBUserRestore(const StringData& serverBinVersion) { + return str::stream() << "Running mongorestore with --drop, --db, and " + "--restoreDbUsersAndRoles flags has erroneous behavior when the target server " + "version is between 2.6.0 and 2.6.3 (see SERVER-14212). Detected server version " << + serverBinVersion << ". Aborting."; + } + + /** + * 1) Drop collection if --drop was specified. For system.users or system.roles collections, + * however, you don't want to remove all the users/roles up front as some of them may be needed + * by the restore. Instead, keep a set of all the users/roles originally in the server, then + * after restoring the users/roles from the dump, remove any users roles that were present in + * the system originally but aren't in the dump. + * + * 2) Parse metadata file (if present) and if the collection doesn't exist (or was just dropped + * b/c we're using --drop), create the collection with the options from the metadata file + * + * 3) Restore the data from the dump file for this collection + * + * 4) If the user asked to drop this collection, then at this point the _users and _roles sets + * will contain users and roles that were in the collection but not in the dump we are + * restoring. Iterate these sets and delete any users and roles that are there. + * + * 5) Restore indexes based on index definitions from the metadata file. + */ + void processFileAndMetadata(const boost::filesystem::path& root, const std::string& ns) { + + _curns = ns; + _curdb = nsToDatabase(_curns); + _curcoll = nsToCollectionSubstring(_curns).toString(); + + toolInfoLog() << "\tgoing into namespace [" << _curns << "]" << std::endl; + + // 1) Drop collection if needed. Save user and role data if this is a system.users or + // system.roles collection + if (mongoRestoreGlobalParams.drop) { + if (_curcoll == "system.users") { + if (_serverAuthzVersion == AuthorizationManager::schemaVersion24 || + _curdb != "admin") { + // Restoring 2.4-style user docs so can't use the _mergeAuthzCollections command + // Create map of the users currently in the DB so the ones that don't show up in + // the dump file can be removed later. + BSONObj fields = BSON("user" << 1 << "userSource" << 1); + scoped_ptr<DBClientCursor> cursor(conn().query(_curns, Query(), 0, 0, &fields)); + while (cursor->more()) { + BSONObj user = cursor->next(); + string userDB; + uassertStatusOK(bsonExtractStringFieldWithDefault(user, + "userSource", + _curdb, + &userDB)); + _users.insert(UserName(user["user"].String(), userDB)); + } } } + else if (!startsWith(_curcoll, "system.")) { // Can't drop system collections + toolInfoLog() << "\t dropping" << std::endl; + conn().dropCollection( ns ); + } + } else { + // If drop is not used, warn if the collection exists. + scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", + Query(BSON("name" << ns)))); + if (cursor->more()) { + // collection already exists show warning + toolError() << "Restoring to " << ns << " without dropping. Restored data " + << "will be inserted without raising errors; check your server log" + << std::endl; + } } + // 2) Create collection with options from metadata file if present BSONObj metadataObject; - if (_restoreOptions || _restoreIndexes) { + if (mongoRestoreGlobalParams.restoreOptions || mongoRestoreGlobalParams.restoreIndexes) { + string oldCollName = root.leaf().string(); // Name of collection that was dumped from + oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); boost::filesystem::path metadataFile = (root.branch_path() / (oldCollName + ".metadata.json")); if (!boost::filesystem::exists(metadataFile.string())) { // This is fine because dumps from before 2.1 won't have a metadata file, just print a warning. // System collections shouldn't have metadata so don't warn if that file is missing. if (!startsWith(metadataFile.leaf().string(), "system.")) { - log() << metadataFile.string() << " not found. Skipping." << endl; + toolInfoLog() << metadataFile.string() << " not found. Skipping." << std::endl; } } else { metadataObject = parseMetadataFile(metadataFile.string()); } } - _curns = ns.c_str(); - _curdb = NamespaceString(_curns).db; - _curcoll = NamespaceString(_curns).coll; - - // If drop is not used, warn if the collection exists. - if (!_drop) { - scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", - Query(BSON("name" << ns)))); - if (cursor->more()) { - // collection already exists show warning - warning() << "Restoring to " << ns << " without dropping. Restored data " - "will be inserted without raising errors; check your server log" - << endl; - } - } - - if (_restoreOptions && metadataObject.hasField("options")) { + if (mongoRestoreGlobalParams.restoreOptions && metadataObject.hasField("options")) { // Try to create collection with given options createCollectionWithOptions(metadataObject["options"].Obj()); } + // 3) Actually restore the BSONObjs inside the dump file processFile( root ); - if (_drop && root.leaf() == "system.users.bson") { - // Delete any users that used to exist but weren't in the dump file - for (set<string>::iterator it = _users.begin(); it != _users.end(); ++it) { - BSONObj userMatch = BSON("user" << *it); - conn().remove(ns, Query(userMatch)); + + // 4) If running with --drop, remove any users/roles that were in the system at the + // beginning of the restore but weren't found in the dump file + if (_curcoll == "system.users") { + if ((_serverAuthzVersion == AuthorizationManager::schemaVersion24 || + _curdb != "admin")) { + // Restoring 2.4 style user docs so don't use the _mergeAuthzCollections command + if (mongoRestoreGlobalParams.drop) { + // Delete any users that used to exist but weren't in the dump file + for (set<UserName>::iterator it = _users.begin(); it != _users.end(); ++it) { + const UserName& name = *it; + BSONObjBuilder queryBuilder; + queryBuilder << "user" << name.getUser(); + if (name.getDB() == _curdb) { + // userSource field won't be present for v1 users docs in the same db as + // the user is defined on. + queryBuilder << "userSource" << BSONNULL; + } else { + queryBuilder << "userSource" << name.getDB(); + } + conn().remove(_curns, Query(queryBuilder.done())); + } + _users.clear(); + } + } else { + // Use _mergeAuthzCollections command to move into admin.system.users the user + // docs that were restored into the temp user collection + BSONObjBuilder cmdBuilder; + cmdBuilder.append("_mergeAuthzCollections", 1); + cmdBuilder.append("tempUsersCollection", mongoRestoreGlobalParams.tempUsersColl); + cmdBuilder.append("drop", mongoRestoreGlobalParams.drop); + cmdBuilder.append("writeConcern", BSON("w" << mongoRestoreGlobalParams.w)); + if (versionCmp(_serverBinVersion, "2.6.4-pre-") < 0) { + uassert(18528, + getMessageAboutBrokenDBUserRestore(_serverBinVersion), + !mongoRestoreGlobalParams.drop || + toolGlobalParams.db.empty() || + toolGlobalParams.db == "admin"); + } else { + // If we're doing a db-specific restore of the "admin" db, we want user data for + // *all* databases restored, not just the admin db, so we pass "" as the "db" + // param to _mergeAuthzCollections + cmdBuilder.append("db", + toolGlobalParams.db == "admin" ? "" : toolGlobalParams.db); + } + + BSONObj res; + conn().runCommand("admin", cmdBuilder.done(), res); + uassert(17412, + mongoutils::str::stream() << "Cannot restore users because the " + "_mergeAuthzCollections command failed: " << res.toString(), + res["ok"].trueValue()); + + conn().dropCollection(mongoRestoreGlobalParams.tempUsersColl); } - _users.clear(); + } + if (_curns == "admin.system.roles") { + // Use _mergeAuthzCollections command to move into admin.system.roles the role + // docs that were restored into the temp roles collection + BSONObjBuilder cmdBuilder; + cmdBuilder.append("_mergeAuthzCollections", 1); + cmdBuilder.append("tempRolesCollection", mongoRestoreGlobalParams.tempRolesColl); + cmdBuilder.append("drop", mongoRestoreGlobalParams.drop); + cmdBuilder.append("writeConcern", BSON("w" << mongoRestoreGlobalParams.w)); + if (versionCmp(_serverBinVersion, "2.6.4-pre-") < 0) { + uassert(18529, + getMessageAboutBrokenDBUserRestore(_serverBinVersion), + !mongoRestoreGlobalParams.drop || + toolGlobalParams.db.empty() || + toolGlobalParams.db == "admin"); + } else { + // If we're doing a db-specific restore of the "admin" db, we want role data for + // *all* databases restored, not just the admin db, so we pass "" as the "db" + // param to _mergeAuthzCollections + cmdBuilder.append("db", toolGlobalParams.db == "admin" ? "" : toolGlobalParams.db); + } + + BSONObj res; + conn().runCommand("admin", cmdBuilder.done(), res); + uassert(17413, + mongoutils::str::stream() << "Cannot restore roles because the " + "_mergeAuthzCollections command failed: " << res.toString(), + res["ok"].trueValue()); + + conn().dropCollection(mongoRestoreGlobalParams.tempRolesColl); } - if (_restoreIndexes && metadataObject.hasField("indexes")) { + // 5) Restore indexes + if (mongoRestoreGlobalParams.restoreIndexes && metadataObject.hasField("indexes")) { vector<BSONElement> indexes = metadataObject["indexes"].Array(); for (vector<BSONElement>::iterator it = indexes.begin(); it != indexes.end(); ++it) { createIndex((*it).Obj(), false); @@ -419,33 +636,129 @@ public: _oplogEntryApplies++; // wait for ops to propagate to "w" nodes (doesn't warn if w used without replset) - if ( _w > 0 ) { - string err = conn().getLastError(db, false, false, _w); + if (mongoRestoreGlobalParams.w > 0) { + string err = conn().getLastError(db, false, false, mongoRestoreGlobalParams.w); if (!err.empty()) { - error() << "Error while replaying oplog: " << err; + toolError() << "Error while replaying oplog: " << err << std::endl; } } + return; } - else if (NamespaceString(_curns).coll == "system.indexes") { + + if (nsToCollectionSubstring(_curns) == "system.indexes") { createIndex(obj, true); } - else if (_drop && endsWith(_curns.c_str(), ".system.users") && _users.count(obj["user"].String())) { - // Since system collections can't be dropped, we have to manually - // replace the contents of the system.users collection - BSONObj userMatch = BSON("user" << obj["user"].String()); - conn().update(_curns, Query(userMatch), obj); - _users.erase(obj["user"].String()); - } else { - conn().insert( _curns , obj ); + else if (_curns == "admin.system.roles") { + // To prevent modifying roles when other role modifications may be going on, restore + // the roles to a temporary collection and merge them into admin.system.roles later + // using the _mergeAuthzCollections command. + conn().insert(mongoRestoreGlobalParams.tempRolesColl, obj); + } + else if (_curcoll == "system.users") { + uassert(17416, + mongoutils::str::stream() << "Cannot modify user data on a server with version " + "greater than or equal to 2.5.4 that has not yet updated the " + "authorization data to schema version " << + AuthorizationManager::schemaVersion26Final << + ". Found server version " << _serverBinVersion << " with " + "authorization schema version " << _serverAuthzVersion, + _curdb != "admin" || + versionCmp(_serverBinVersion, "2.5.4") < 0 || + _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); + + if (obj.hasField("credentials")) { + if (_serverAuthzVersion == AuthorizationManager::schemaVersion24) { + // v3 user, v1 system + uasserted(17407, + mongoutils::str::stream() + << "Server has authorization schema version " + << AuthorizationManager::schemaVersion24 + << ", but found a schema version " + << AuthorizationManager::schemaVersion26Final << " user: " + << obj.toString()); + } else { + // v3 user, v3 system + uassert(17414, + mongoutils::str::stream() << "Found a schema version " << + AuthorizationManager::schemaVersion26Final << + " user when restoring to a non-admin db system.users " + "collection: " << obj.toString(), + _curdb == "admin"); + // To prevent modifying users when other user modifications may be going on, + // restore the users to a temporary collection and merge them into + // admin.system.users later using the _mergeAuthzCollections command. + conn().insert(mongoRestoreGlobalParams.tempUsersColl, obj); + } + } else { + if (_curdb == "admin" && + _serverAuthzVersion == AuthorizationManager::schemaVersion26Final && + !_serverAuthzVersionDocExists) { + // server with schemaVersion26Final implies it is running 2.5.4 or greater. + uasserted(17415, + mongoutils::str::stream() << "Cannot restore users with schema " << + "version " << AuthorizationManager::schemaVersion24 << + " to a system with server version 2.5.4 or greater"); + } - // wait for insert to propagate to "w" nodes (doesn't warn if w used without replset) - if ( _w > 0 ) { - string err = conn().getLastError(_curdb, false, false, _w); - if (!err.empty()) { - error() << err; + if (_serverAuthzVersion == AuthorizationManager::schemaVersion24 || + _curdb != "admin") { // Restoring 2.4 schema users to non-admin dbs is OK) + // v1 user, v1 system + string userDB; + uassertStatusOK(bsonExtractStringFieldWithDefault(obj, + "userSource", + _curdb, + &userDB)); + + if (mongoRestoreGlobalParams.drop && _users.count(UserName(obj["user"].String(), + userDB))) { + // Since system collections can't be dropped, we have to manually + // replace the contents of the system.users collection + BSONObj userMatch = BSON("user" << obj["user"].String() << + "userSource" << userDB); + conn().update(_curns, Query(userMatch), obj); + _users.erase(UserName(obj["user"].String(), userDB)); + } else { + conn().insert(_curns, obj); + } + } else { + // v1 user, v3 system + // TODO(spencer): SERVER-12491 Rather than failing here, we should convert the + // v1 user to an equivalent v3 schema user + uasserted(17408, + mongoutils::str::stream() + << "Server has authorization schema version " + << AuthorizationManager::schemaVersion26Final + << ", but found a schema version " + << AuthorizationManager::schemaVersion24 << " user: " + << obj.toString()); } } } + else { + if (_curns == "admin.system.version") { + long long authVersion; + uassertStatusOK(bsonExtractIntegerField(obj, + AuthorizationManager::schemaVersionFieldName, + &authVersion)); + _dumpFileAuthzVersion = static_cast<int>(authVersion); + uassert(17371, + mongoutils::str::stream() << "Server's authorization data schema version " + "does not match that of the data in the dump file. Server's schema" + " version: " << _serverAuthzVersion << ", schema version in dump: " + << _dumpFileAuthzVersion, + _serverAuthzVersion == _dumpFileAuthzVersion); + } + conn().insert(_curns, obj); + } + + // wait for insert (or update) to propagate to "w" nodes (doesn't warn if w used + // without replset) + if (mongoRestoreGlobalParams.w > 0) { + string err = conn().getLastError(_curdb, false, false, mongoRestoreGlobalParams.w); + if (!err.empty()) { + toolError() << err << std::endl; + } + } } private: @@ -454,11 +767,13 @@ private: long long fileSize = boost::filesystem::file_size(filePath); ifstream file(filePath.c_str(), ios_base::in); - scoped_ptr<char> buf(new char[fileSize]); + boost::scoped_array<char> buf(new char[fileSize + 1]); file.read(buf.get(), fileSize); + buf[fileSize] = '\0'; + int objSize; BSONObj obj; - obj = fromjson (buf.get(), &objSize); + obj = fromjson(buf.get(), &objSize); return obj; } @@ -484,34 +799,30 @@ private: return nfields == obj2.nFields(); } - void createCollectionWithOptions(BSONObj cmdObj) { + void createCollectionWithOptions(BSONObj obj) { + BSONObjIterator i(obj); - // Create a new cmdObj to skip undefined fields and fix collection name + // Rebuild obj as a command object for the "create" command. + // - {create: <name>} comes first, where <name> is the new name for the collection + // - elements with type Undefined get skipped over BSONObjBuilder bo; - - // Add a "create" field if it doesn't exist - if (!cmdObj.hasField("create")) { - bo.append("create", _curcoll); - } - - BSONObjIterator i(cmdObj); - while ( i.more() ) { + bo.append("create", _curcoll); + while (i.more()) { BSONElement e = i.next(); - // Replace the "create" field with the name of the collection we are actually creating if (strcmp(e.fieldName(), "create") == 0) { - bo.append("create", _curcoll); + continue; } - else { - if (e.type() == Undefined) { - log() << _curns << ": skipping undefined field: " << e.fieldName() << endl; - } - else { - bo.append(e); - } + + if (e.type() == Undefined) { + toolInfoLog() << _curns << ": skipping undefined field: " << e.fieldName() + << std::endl; + continue; } + + bo.append(e); } - cmdObj = bo.obj(); + obj = bo.obj(); BSONObj fields = BSON("options" << 1); scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", Query(BSON("name" << _curns)), 0, 0, &fields)); @@ -519,9 +830,12 @@ private: bool createColl = true; if (cursor->more()) { createColl = false; - BSONObj obj = cursor->next(); - if (!obj.hasField("options") || !optionsSame(cmdObj, obj["options"].Obj())) { - log() << "WARNING: collection " << _curns << " exists with different options than are in the metadata.json file and not using --drop. Options in the metadata file will be ignored." << endl; + BSONObj nsObj = cursor->next(); + if (!nsObj.hasField("options") || !optionsSame(obj, nsObj["options"].Obj())) { + toolError() << "WARNING: collection " << _curns + << " exists with different options than are in the metadata.json file and" + << " not using --drop. Options in the metadata file will be ignored." + << std::endl; } } @@ -530,10 +844,11 @@ private: } BSONObj info; - if (!conn().runCommand(_curdb, cmdObj, info)) { + if (!conn().runCommand(_curdb, obj, info)) { uasserted(15936, "Creating collection " + _curns + " failed. Errmsg: " + info["errmsg"].String()); } else { - log() << "\tCreated collection " << _curns << " with options: " << cmdObj.jsonString() << endl; + toolInfoLog() << "\tCreated collection " << _curns << " with options: " + << obj.jsonString() << std::endl; } } @@ -547,23 +862,26 @@ private: BSONElement e = i.next(); if (strcmp(e.fieldName(), "ns") == 0) { NamespaceString n(e.String()); - string s = _curdb + "." + (keepCollName ? n.coll : _curcoll); + string s = _curdb + "." + (keepCollName ? n.coll().toString() : _curcoll); bo.append("ns", s); } - else if (strcmp(e.fieldName(), "v") != 0 || _keepIndexVersion) { // Remove index version number + // Remove index version number + else if (strcmp(e.fieldName(), "v") != 0 || mongoRestoreGlobalParams.keepIndexVersion) { bo.append(e); } } BSONObj o = bo.obj(); - LOG(0) << "\tCreating index: " << o << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(0))) { + toolInfoLog() << "\tCreating index: " << o << std::endl; + } conn().insert( _curdb + ".system.indexes" , o ); // We're stricter about errors for indexes than for regular data - BSONObj err = conn().getLastErrorDetailed(_curdb, false, false, _w); + BSONObj err = conn().getLastErrorDetailed(_curdb, false, false, mongoRestoreGlobalParams.w); if (err.hasField("err") && !err["err"].isNull()) { - if (err["err"].str() == "norepl" && _w > 1) { - error() << "Cannot specify write concern for non-replicas" << endl; + if (err["err"].str() == "norepl" && mongoRestoreGlobalParams.w > 1) { + toolError() << "Cannot specify write concern for non-replicas" << std::endl; } else { string errCode; @@ -572,8 +890,8 @@ private: errCode = str::stream() << err["code"].numberInt(); } - error() << "Error creating index " << o["ns"].String() << ": " - << errCode << " " << err["err"] << endl; + toolError() << "Error creating index " << o["ns"].String() << ": " + << errCode << " " << err["err"] << std::endl; } ::abort(); @@ -584,8 +902,4 @@ private: } }; -int main( int argc , char ** argv, char ** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - Restore restore; - return restore.main( argc , argv ); -} +REGISTER_MONGO_TOOL(Restore); diff --git a/src/mongo/tools/sniffer.cpp b/src/mongo/tools/sniffer.cpp index 5e2ac66267b..2d57ef7af47 100644 --- a/src/mongo/tools/sniffer.cpp +++ b/src/mongo/tools/sniffer.cpp @@ -1,4 +1,3 @@ -// sniffer.cpp /* * Copyright (C) 2010 10gen Inc. * @@ -13,50 +12,62 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ - /* TODO: large messages - need to track what's left and ignore - single object over packet size - can only display begging of object + single object over packet size - can only display beginning of object getmore delete killcursors */ -#include "../pch.h" -#include <pcap.h> +#include "mongo/pch.h" #ifdef _WIN32 #undef min #undef max #endif -#include "mongo/base/initializer.h" -#include "../bson/util/builder.h" -#include "../util/net/message.h" -#include "../util/mmap.h" -#include "../db/dbmessage.h" - -#include <stdio.h> -#include <string.h> -#include <stdlib.h> +#include <boost/shared_ptr.hpp> #include <ctype.h> #include <errno.h> +#include <iostream> +#include <map> +#include <pcap.h> +#include <stdio.h> +#include <stdlib.h> +#include <string> +#include <string.h> #include <sys/types.h> + #ifndef _WIN32 -#include <sys/socket.h> -#include <netinet/in.h> #include <arpa/inet.h> +#include <netinet/in.h> +#include <sys/socket.h> #endif -#include <iostream> -#include <map> -#include <string> - -#include <boost/shared_ptr.hpp> +#include "mongo/base/initializer.h" +#include "mongo/bson/util/builder.h" +#include "mongo/client/dbclientinterface.h" +#include "mongo/db/dbmessage.h" +#include "mongo/util/net/message.h" +#include "mongo/util/mmap.h" +#include "mongo/util/text.h" using namespace std; using mongo::Message; @@ -68,8 +79,6 @@ using mongo::DBClientConnection; using mongo::QueryResult; using mongo::MemoryMappedFile; -mongo::CmdLine mongo::cmdLine; - #define SNAP_LEN 65535 int captureHeaderSize; @@ -329,9 +338,7 @@ void processMessage( Connection& c , Message& m ) { break; } case mongo::dbKillCursors: { - int *x = (int *) m.singleData()->_data; - x++; // reserved - int n = *x; + int n = d.pullInt(); out() << "\tkillCursors n: " << n << endl; break; } @@ -357,7 +364,7 @@ void processMessage( Connection& c , Message& m ) { if ( m.operation() == mongo::dbGetMore ) { DbMessage d( m ); d.pullInt(); - long long &cId = d.pullInt64(); + long long cId = d.pullInt64(); cId = mapCursor[ c ][ cId ]; } Message response; @@ -425,7 +432,8 @@ void processDiagLog( const char * file ) { void usage() { cout << - "Usage: mongosniff [--help] [--forward host:port] [--source (NET <interface> | (FILE | DIAGLOG) <filename>)] [<port0> <port1> ... ]\n" + "Usage: mongosniff [--help] [--forward host:port] [--objcheck] [--source (NET <interface> | (FILE | DIAGLOG) <filename>)] [<port0> <port1> ... ]\n" + "--help Print this help message.\n" "--forward Forward all parsed request messages to mongod instance at \n" " specified host:port\n" "--source Source of traffic to sniff, either a network interface or a\n" @@ -438,11 +446,10 @@ void usage() { " when there are dropped tcp packets.\n" "<port0>... These parameters are used to filter sniffing. By default, \n" " only port 27017 is sniffed.\n" - "--help Print this help message.\n" << endl; } -int main(int argc, char **argv, char** envp) { +int toolMain(int argc, char **argv, char** envp) { mongo::runGlobalInitializersOrDie(argc, argv, envp); stringstream nullStream; @@ -453,8 +460,14 @@ int main(int argc, char **argv, char** envp) { pcap_t *handle; struct bpf_program fp; - bpf_u_int32 mask; - bpf_u_int32 net; + +// PCAP_NETMASK_UNKNOWN was introduced in pcap 1.1.0, so work around earlier versions. See +// http://anonsvn.wireshark.org/viewvc?revision=33461&view=revision for details. +#if defined(PCAP_NETMASK_UNKNOWN) + bpf_u_int32 mask = PCAP_NETMASK_UNKNOWN; +#else + bpf_u_int32 mask = 0; +#endif bool source = false; bool replay = false; @@ -523,6 +536,7 @@ int main(int argc, char **argv, char** envp) { } cout << "found device: " << dev << endl; } + bpf_u_int32 net; if (pcap_lookupnet(dev, &net, &mask, errbuf) == -1) { cerr << "can't get netmask: " << errbuf << endl; return -1; @@ -545,7 +559,7 @@ int main(int argc, char **argv, char** envp) { cerr << "don't know how to handle datalink type: " << pcap_datalink( handle ) << endl; } - verify( pcap_compile(handle, &fp, const_cast< char * >( "tcp" ) , 0, net) != -1 ); + verify( pcap_compile(handle, &fp, const_cast< char * >( "tcp" ) , 0, mask) != -1 ); verify( pcap_setfilter(handle, &fp) != -1 ); cout << "sniffing... "; @@ -561,3 +575,20 @@ int main(int argc, char **argv, char** envp) { return 0; } +#if defined(_WIN32) +// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters +// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The +// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent +// and makes them available through the argv() and envp() members. This enables toolMain() +// to process UTF-8 encoded arguments and environment variables without regard to platform. +int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { + WindowsCommandLine wcl(argc, argvW, envpW); + int exitCode = toolMain(argc, wcl.argv(), wcl.envp()); + ::_exit(exitCode); +} +#else +int main(int argc, char* argv[], char** envp) { + int exitCode = toolMain(argc, argv, envp); + ::_exit(exitCode); +} +#endif diff --git a/src/mongo/tools/stat.cpp b/src/mongo/tools/stat.cpp index 07194444671..002d8a4a5f0 100644 --- a/src/mongo/tools/stat.cpp +++ b/src/mongo/tools/stat.cpp @@ -12,148 +12,92 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" - -#include "mongo/tools/tool.h" +#include "mongo/pch.h" -#include <boost/program_options.hpp> #include <boost/thread/thread.hpp> #include <fstream> #include <iostream> -#include "mongo/base/initializer.h" +#include "mongo/base/init.h" #include "mongo/client/dbclientcursor.h" +#include "mongo/client/sasl_client_authenticate.h" #include "mongo/db/jsobjmanipulator.h" #include "mongo/db/json.h" #include "mongo/s/type_shard.h" #include "mongo/tools/stat_util.h" +#include "mongo/tools/mongostat_options.h" +#include "mongo/tools/tool.h" #include "mongo/util/net/httpclient.h" +#include "mongo/util/options_parser/option_section.h" #include "mongo/util/text.h" -namespace po = boost::program_options; - namespace mongo { class Stat : public Tool { public: - Stat() : Tool( "stat" , REMOTE_SERVER , "admin" ) { - _http = false; - _many = false; - - add_hidden_options() - ( "sleep" , po::value<int>() , "time to sleep between calls" ) - ; - add_options() - ("noheaders", "don't output column names") - ("rowcount,n", po::value<int>()->default_value(0), "number of stats lines to print (0 for indefinite)") - ("http", "use http instead of raw db connection") - ("discover" , "discover nodes and display stats for all" ) - ("all" , "all optional fields" ) - ; - - addPositionArg( "sleep" , 1 ); - + Stat() : Tool() { _autoreconnect = true; } - virtual void printExtraHelp( ostream & out ) { - out << "View live MongoDB performance statistics.\n" << endl; - out << "usage: " << _name << " [options] [sleep time]" << endl; - out << "sleep time: time to wait (in seconds) between calls" << endl; - } - - virtual void printExtraHelpAfter( ostream & out ) { - out << "\n"; - out << " Fields\n"; - out << " inserts \t- # of inserts per second (* means replicated op)\n"; - out << " query \t- # of queries per second\n"; - out << " update \t- # of updates per second\n"; - out << " delete \t- # of deletes per second\n"; - out << " getmore \t- # of get mores (cursor batch) per second\n"; - out << " command \t- # of commands per second, on a slave its local|replicated\n"; - out << " flushes \t- # of fsync flushes per second\n"; - out << " mapped \t- amount of data mmaped (total data size) megabytes\n"; - out << " vsize \t- virtual size of process in megabytes\n"; - out << " res \t- resident size of process in megabytes\n"; - out << " faults \t- # of pages faults per sec\n"; - out << " locked \t- name of and percent time for most locked database\n"; - out << " idx miss \t- percent of btree page misses (sampled)\n"; - out << " qr|qw \t- queue lengths for clients waiting (read|write)\n"; - out << " ar|aw \t- active clients (read|write)\n"; - out << " netIn \t- network traffic in - bits\n"; - out << " netOut \t- network traffic out - bits\n"; - out << " conn \t- number of open connections\n"; - out << " set \t- replica set name\n"; - out << " repl \t- replication type \n"; - out << " \t PRI - primary (master)\n"; - out << " \t SEC - secondary\n"; - out << " \t REC - recovering\n"; - out << " \t UNK - unknown\n"; - out << " \t SLV - slave\n"; - out << " \t RTR - mongos process (\"router\")\n"; + virtual void printHelp( ostream & out ) { + printMongoStatHelp(&out); } BSONObj stats() { - if ( _http ) { + if (mongoStatGlobalParams.http) { HttpClient c; HttpClient::Result r; string url; { stringstream ss; - ss << "http://" << _host; - if ( _host.find( ":" ) == string::npos ) + ss << "http://" << toolGlobalParams.connectionString; + if (toolGlobalParams.connectionString.find( ":" ) == string::npos) ss << ":28017"; ss << "/_status"; url = ss.str(); } if ( c.get( url , &r ) != 200 ) { - cout << "error (http): " << r.getEntireResponse() << endl; + toolError() << "error (http): " << r.getEntireResponse() << std::endl; return BSONObj(); } BSONObj x = fromjson( r.getBody() ); BSONElement e = x["serverStatus"]; if ( e.type() != Object ) { - cout << "BROKEN: " << x << endl; + toolError() << "BROKEN: " << x << std::endl; return BSONObj(); } return e.embeddedObjectUserCheck(); } BSONObj out; - if ( ! conn().simpleCommand( _db , &out , "serverStatus" ) ) { - cout << "error: " << out << endl; + if (!conn().simpleCommand(toolGlobalParams.db, &out, "serverStatus")) { + toolError() << "error: " << out << std::endl; return BSONObj(); } return out.getOwned(); } - - virtual void preSetup() { - if ( hasParam( "http" ) ) { - _http = true; - _noconnection = true; - } - - if ( hasParam( "host" ) && - getParam( "host" ).find( ',' ) != string::npos ) { - _noconnection = true; - _many = true; - } - - if ( hasParam( "discover" ) ) { - _many = true; - } - } - int run() { - _statUtil.setSeconds( getParam( "sleep" , 1 ) ); - _statUtil.setAll( hasParam( "all" ) ); - if ( _many ) + _statUtil.setAll(mongoStatGlobalParams.allFields); + _statUtil.setSeconds(mongoStatGlobalParams.sleep); + if (mongoStatGlobalParams.many) return runMany(); return runNormal(); } @@ -200,8 +144,6 @@ namespace mongo { } int runNormal() { - bool showHeaders = ! hasParam( "noheaders" ); - int rowCount = getParam( "rowcount" , 0 ); int rowNum = 0; BSONObj prev = stats(); @@ -209,15 +151,17 @@ namespace mongo { return -1; int maxLockedDbWidth = 0; + bool warned = false; - while ( rowCount == 0 || rowNum < rowCount ) { + while (mongoStatGlobalParams.rowCount == 0 || + rowNum < mongoStatGlobalParams.rowCount) { sleepsecs((int)ceil(_statUtil.getSeconds())); BSONObj now; try { now = stats(); } catch ( std::exception& e ) { - cout << "can't get data: " << e.what() << endl; + toolError() << "can't get data: " << e.what() << std::endl; continue; } @@ -226,11 +170,16 @@ namespace mongo { try { + if ( !warned && now["storageEngine"].type() ) { + toolError() << "warning: detected a 3.0 mongod, some columns not applicable" << endl; + warned = true; + } + BSONObj out = _statUtil.doRow( prev , now ); // adjust width up as longer 'locked db' values appear setMaxLockedDbWidth( &out, &maxLockedDbWidth ); - if ( showHeaders && rowNum % 10 == 0 ) { + if (mongoStatGlobalParams.showHeaders && rowNum % 10 == 0) { printHeaders( out ); } @@ -238,9 +187,9 @@ namespace mongo { } catch ( AssertionException& e ) { - cout << "\nerror: " << e.what() << "\n" - << now - << endl; + toolError() << "\nerror: " << e.what() << "\n" + << now + << std::endl; } prev = now; @@ -287,8 +236,9 @@ namespace mongo { static void serverThread( shared_ptr<ServerState> state , int sleepTime) { try { + bool warned = false; DBClientConnection conn( true ); - conn._logLevel = 1; + conn._logLevel = logger::LogSeverity::Debug(1); string errmsg; if ( ! conn.connect( state->host , errmsg ) ) state->error = errmsg; @@ -308,19 +258,28 @@ namespace mongo { state->now = out.getOwned(); } else { + str::stream errorStream; + errorStream << "serverStatus failed"; + BSONElement errorField = out["errmsg"]; + if (errorField.type() == String) + errorStream << ": " << errorField.str(); scoped_lock lk( state->lock ); - state->error = "serverStatus failed"; + state->error = errorStream; state->lastUpdate = time(0); } + if ( !warned && out["storageEngine"].type() ) { + toolError() << "warning: detected a 3.0 mongod, some columns not applicable" << endl; + warned = true; + } if ( out["shardCursorType"].type() == Object || - out["process"].String() == "mongos" ) { + out["process"].str() == "mongos" ) { state->mongos = true; if ( cycleNumber % 10 == 1 ) { auto_ptr<DBClientCursor> c = conn.query( ShardType::ConfigNS , BSONObj() ); vector<BSONObj> shards; while ( c->more() ) { - shards.push_back( c->next().getOwned() ); + shards.push_back( c->nextSafe().getOwned() ); } scoped_lock lk( state->lock ); state->shards = shards; @@ -338,10 +297,11 @@ namespace mongo { } catch ( std::exception& e ) { - cout << "serverThread (" << state->host << ") fatal error : " << e.what() << endl; + toolError() << "serverThread (" << state->host << ") fatal error : " << e.what() + << std::endl; } catch ( ... ) { - cout << "serverThread (" << state->host << ") fatal error" << endl; + toolError() << "serverThread (" << state->host << ") fatal error" << std::endl; } } @@ -358,10 +318,11 @@ namespace mongo { state->thr.reset( new boost::thread( boost::bind( serverThread, state, (int)ceil(_statUtil.getSeconds()) ) ) ); - state->authParams = BSON( "user" << _username << - "pwd" << _password << - "userSource" << getAuthenticationDatabase() << - "mechanism" << _authenticationMechanism ); + state->authParams = BSON(saslCommandUserFieldName << toolGlobalParams.username + << saslCommandPasswordFieldName << toolGlobalParams.password + << saslCommandUserDBFieldName << getAuthenticationDatabase() + << saslCommandMechanismFieldName + << toolGlobalParams.authenticationMechanism); return true; } @@ -401,7 +362,7 @@ namespace mongo { string errmsg; ConnectionString cs = ConnectionString::parse( x["host"].String() , errmsg ); if ( errmsg.size() ) { - cerr << errmsg << endl; + toolError() << errmsg << std::endl; continue; } @@ -420,12 +381,13 @@ namespace mongo { StateMap threads; { - string orig = getParam( "host" ); + string orig = "localhost"; bool showPorts = false; - if ( orig == "" ) - orig = "localhost"; + if (toolGlobalParams.hostSet) { + orig = toolGlobalParams.host; + } - if ( orig.find( ":" ) != string::npos || hasParam( "port" ) ) + if (orig.find(":") != string::npos || toolGlobalParams.portSet) showPorts = true; StringSplitter ss( orig.c_str() , "," ); @@ -433,10 +395,14 @@ namespace mongo { string host = ss.next(); if ( showPorts && host.find( ":" ) == string::npos) { // port supplied, but not for this host. use default. - if ( hasParam( "port" ) ) - host += ":" + _params["port"].as<string>(); - else - host += ":27017"; + StringBuilder sb; + if (toolGlobalParams.portSet) { + sb << host << ":" << toolGlobalParams.port; + } + else { + sb << host << ":27017"; + } + host = sb.str(); } _add( threads , host ); } @@ -445,10 +411,9 @@ namespace mongo { sleepsecs(1); int row = 0; - bool discover = hasParam( "discover" ); int maxLockedDbWidth = 0; - while ( 1 ) { + while (mongoStatGlobalParams.rowCount == 0 || row < mongoStatGlobalParams.rowCount) { sleepsecs( (int)ceil(_statUtil.getSeconds()) ); // collect data @@ -467,7 +432,7 @@ namespace mongo { rows.push_back( Row( i->first , out ) ); } - if ( discover && ! i->second->now.isEmpty() ) { + if (mongoStatGlobalParams.discover && ! i->second->now.isEmpty()) { if ( _discover( threads , i->first , i->second ) ) break; } @@ -528,7 +493,7 @@ namespace mongo { cout << endl; // header - if ( row++ % 5 == 0 && ! biggest.isEmpty() ) { + if (row++ % 5 == 0 && mongoStatGlobalParams.showHeaders && !biggest.isEmpty()) { cout << setw( longestHost ) << "" << "\t"; printHeaders( biggest ); } @@ -550,8 +515,6 @@ namespace mongo { } StatUtil _statUtil; - bool _http; - bool _many; struct Row { Row( string h , string e ) { @@ -572,11 +535,5 @@ namespace mongo { BSONObj data; }; }; - -} - -int main( int argc , char ** argv, char ** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - mongo::Stat stat; - return stat.main( argc , argv ); + REGISTER_MONGO_TOOL(Stat); } diff --git a/src/mongo/tools/stat_util.cpp b/src/mongo/tools/stat_util.cpp index 26d62434770..d8d7c51e74b 100644 --- a/src/mongo/tools/stat_util.cpp +++ b/src/mongo/tools/stat_util.cpp @@ -14,8 +14,22 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ +#include <iomanip> + #include "stat_util.h" #include "mongo/util/mongoutils/str.h" @@ -317,9 +331,11 @@ namespace mongo { NamespaceInfo& s = stats[e.fieldName()]; s.ns = e.fieldName(); - BSONObj temp = e.Obj()["timeLockedMicros"].Obj(); - s.read = ( temp["r"].numberLong() + temp["R"].numberLong() ) / 1000; - s.write = ( temp["w"].numberLong() + temp["W"].numberLong() ) / 1000; + if ( e.Obj()["timeLockedMicros"].isABSONObj() ){ + BSONObj temp = e.Obj()["timeLockedMicros"].Obj(); + s.read = ( temp["r"].numberLong() + temp["R"].numberLong() ) / 1000; + s.write = ( temp["w"].numberLong() + temp["W"].numberLong() ) / 1000; + } } return stats; diff --git a/src/mongo/tools/stat_util.h b/src/mongo/tools/stat_util.h index 98275ae988e..c0ba9bade37 100644 --- a/src/mongo/tools/stat_util.h +++ b/src/mongo/tools/stat_util.h @@ -14,6 +14,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ #pragma once diff --git a/src/mongo/tools/tool.cpp b/src/mongo/tools/tool.cpp index 2539e5cacb4..c5e63723bf9 100644 --- a/src/mongo/tools/tool.cpp +++ b/src/mongo/tools/tool.cpp @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ // Tool.cpp @@ -22,132 +34,48 @@ #include <fstream> #include <iostream> -#include "pcrecpp.h" - +#include "mongo/base/initializer.h" +#include "mongo/base/init.h" #include "mongo/client/dbclient_rs.h" #include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" +#include "mongo/db/auth/authz_manager_external_state_mock.h" #include "mongo/db/json.h" -#include "mongo/db/namespace_details.h" +#include "mongo/db/structure/catalog/namespace_details.h" +#include "mongo/db/storage_options.h" #include "mongo/platform/posix_fadvise.h" +#include "mongo/util/exception_filter_win32.h" #include "mongo/util/file_allocator.h" +#include "mongo/util/options_parser/option_section.h" #include "mongo/util/password.h" +#include "mongo/util/net/ssl_options.h" +#include "mongo/util/text.h" #include "mongo/util/version.h" using namespace std; using namespace mongo; -namespace po = boost::program_options; - namespace mongo { - CmdLine cmdLine; - - Tool::Tool( string name , DBAccess access , string defaultDB , - string defaultCollection , bool usesstdout ) : - _name( name ) , _db( defaultDB ) , _coll( defaultCollection ) , - _usesstdout(usesstdout), _noconnection(false), _autoreconnect(false), _conn(0), _slaveConn(0), _paired(false) { - - _options = new po::options_description( "options" ); - _options->add_options() - ("help","produce help message") - ("verbose,v", "be more verbose (include multiple times for more verbosity e.g. -vvvvv)") - ("version", "print the program's version and exit" ) - ; - - if ( access & REMOTE_SERVER ) - _options->add_options() - ("host,h",po::value<string>(), "mongo host to connect to ( <set name>/s1,s2 for sets)" ) - ("port",po::value<string>(), "server port. Can also use --host hostname:port" ) - ("ipv6", "enable IPv6 support (disabled by default)") -#ifdef MONGO_SSL - ("ssl", "use SSL for all connections") -#endif - - ("username,u",po::value<string>(), "username" ) - ("password,p", new PasswordValue( &_password ), "password" ) - ("authenticationDatabase", - po::value<string>(&_authenticationDatabase)->default_value(""), - "user source (defaults to dbname)" ) - ("authenticationMechanism", - po::value<string>(&_authenticationMechanism)->default_value("MONGODB-CR"), - "authentication mechanism") - ; - - if ( access & LOCAL_SERVER ) - _options->add_options() - ("dbpath",po::value<string>(), "directly access mongod database " - "files in the given path, instead of connecting to a mongod " - "server - needs to lock the data directory, so cannot be " - "used if a mongod is currently accessing the same path" ) - ("directoryperdb", "each db is in a separate directly (relevant only if dbpath specified)" ) - ("journal", "enable journaling (relevant only if dbpath specified)" ) - ; - - if ( access & SPECIFY_DBCOL ) - _options->add_options() - ("db,d",po::value<string>(), "database to use" ) - ("collection,c",po::value<string>(), "collection to use (some commands)" ) - ; - - _hidden_options = new po::options_description( name + " hidden options" ); - - /* support for -vv -vvvv etc. */ - for (string s = "vv"; s.length() <= 10; s.append("v")) { - _hidden_options->add_options()(s.c_str(), "verbose"); - } - } + Tool::Tool() : + _autoreconnect(false), _conn(0), _slaveConn(0) { } Tool::~Tool() { - delete( _options ); - delete( _hidden_options ); if ( _conn ) delete _conn; } - void Tool::printHelp(ostream &out) { - printExtraHelp(out); - _options->print(out); - printExtraHelpAfter(out); + MONGO_INITIALIZER(ToolAuthExternalState)(InitializerContext*) { + setGlobalAuthorizationManager(new AuthorizationManager( + new AuthzManagerExternalStateMock())); + return Status::OK(); } - void Tool::printVersion(ostream &out) { - out << _name << " version " << mongo::versionString; - if (mongo::versionString[strlen(mongo::versionString)-1] == '-') - out << " (commit " << mongo::gitVersion() << ")"; - out << endl; - } - int Tool::main( int argc , char ** argv ) { + int Tool::main( int argc , char ** argv, char ** envp ) { static StaticObserver staticObserver; - cmdLine.prealloc = false; - - // The default value may vary depending on compile options, but for tools - // we want durability to be disabled. - cmdLine.dur = false; - - _name = argv[0]; - - /* using the same style as db.cpp */ - int command_line_style = (((po::command_line_style::unix_style ^ - po::command_line_style::allow_guessing) | - po::command_line_style::allow_long_disguise) ^ - po::command_line_style::allow_sticky); - try { - po::options_description all_options("all options"); - all_options.add(*_options).add(*_hidden_options); - - po::store( po::command_line_parser( argc , argv ). - options(all_options). - positional( _positonalOptions ). - style(command_line_style).run() , _params ); - - po::notify( _params ); - } - catch (po::error &e) { - cerr << "ERROR: " << e.what() << endl << endl; - printHelp(cerr); - ::_exit(EXIT_BADOPTIONS); - } + mongo::runGlobalInitializersOrDie(argc, argv, envp); // hide password from ps output for (int i=0; i < (argc-1); ++i) { @@ -159,89 +87,46 @@ namespace mongo { } } - if ( _params.count( "help" ) ) { - printHelp(cout); - ::_exit(0); - } - - if ( _params.count( "version" ) ) { - printVersion(cout); - ::_exit(0); - } - - if ( _params.count( "verbose" ) ) { - logLevel = 1; - } - - for (string s = "vv"; s.length() <= 10; s.append("v")) { - if (_params.count(s)) { - logLevel = s.length(); - } - } - - -#ifdef MONGO_SSL - if (_params.count("ssl")) { - mongo::cmdLine.sslOnNormalPorts = true; - } -#endif - - preSetup(); - - bool useDirectClient = hasParam( "dbpath" ); - - if ( ! useDirectClient ) { - _host = "127.0.0.1"; - if ( _params.count( "host" ) ) - _host = _params["host"].as<string>(); - - if ( _params.count( "port" ) ) - _host += ':' + _params["port"].as<string>(); - - if ( _noconnection ) { + if (!toolGlobalParams.useDirectClient) { + if (toolGlobalParams.noconnection) { // do nothing } else { string errmsg; - ConnectionString cs = ConnectionString::parse( _host , errmsg ); + ConnectionString cs = ConnectionString::parse(toolGlobalParams.connectionString, + errmsg); if ( ! cs.isValid() ) { - cerr << "invalid hostname [" << _host << "] " << errmsg << endl; + toolError() << "invalid hostname [" << toolGlobalParams.connectionString << "] " + << errmsg << std::endl; ::_exit(-1); } _conn = cs.connect( errmsg ); if ( ! _conn ) { - cerr << "couldn't connect to [" << _host << "] " << errmsg << endl; + toolError() << "couldn't connect to [" << toolGlobalParams.connectionString + << "] " << errmsg << std::endl; ::_exit(-1); } - (_usesstdout ? cout : cerr ) << "connected to: " << _host << endl; + toolInfoOutput() << "connected to: " << toolGlobalParams.connectionString + << std::endl; } } else { - if ( _params.count( "directoryperdb" ) ) { - directoryperdb = true; - } verify( lastError.get( true ) ); - if (_params.count("journal")){ - cmdLine.dur = true; - } - Client::initThread("tools"); _conn = new DBDirectClient(); - _host = "DIRECT"; - static string myDbpath = getParam( "dbpath" ); - dbpath = myDbpath.c_str(); + storageGlobalParams.dbpath = toolGlobalParams.dbpath; try { acquirePathLock(); } catch ( DBException& ) { - cerr << endl << "If you are running a mongod on the same " - "path you should connect to that instead of direct data " - "file access" << endl << endl; + toolError() << std::endl << "If you are running a mongod on the same " + "path you should connect to that instead of direct data " + "file access" << std::endl << std::endl; dbexit( EXIT_FS ); ::_exit(EXIT_FAILURE); } @@ -251,31 +136,14 @@ namespace mongo { dur::startup(); } - if ( _params.count( "db" ) ) - _db = _params["db"].as<string>(); - - if ( _params.count( "collection" ) ) - _coll = _params["collection"].as<string>(); - - if ( _params.count( "username" ) ) - _username = _params["username"].as<string>(); - - if ( _params.count( "password" ) - && ( _password.empty() ) ) { - _password = askPassword(); - } - - if (_params.count("ipv6")) - enableIPv6(); - int ret = -1; try { - if (!useDirectClient && !_noconnection) + if (!toolGlobalParams.useDirectClient && !toolGlobalParams.noconnection) auth(); ret = run(); } catch ( DBException& e ) { - cerr << "assertion: " << e.toString() << endl; + toolError() << "assertion: " << e.toString() << std::endl; ret = -1; } catch(const boost::filesystem::filesystem_error &fse) { @@ -301,7 +169,7 @@ namespace mongo { printHelp(cerr); else #endif // _WIN32 - cerr << "error: " << fse.what() << endl; + toolError() << "error: " << fse.what() << std::endl; ret = -1; } @@ -309,7 +177,7 @@ namespace mongo { if ( currentClient.get() ) currentClient.get()->shutdown(); - if ( useDirectClient ) + if (toolGlobalParams.useDirectClient) dbexit( EXIT_CLEAN ); fflush(stdout); @@ -329,7 +197,7 @@ namespace mongo { } bool Tool::isMaster() { - if ( hasParam("dbpath") ) { + if (toolGlobalParams.useDirectClient) { return true; } @@ -338,8 +206,9 @@ namespace mongo { bool ok = conn().isMaster(isMaster, &info); if (ok && !isMaster) { - cerr << "ERROR: trying to write to non-master " << conn().toString() << endl; - cerr << "isMaster info: " << info << endl; + toolError() << "ERROR: trying to write to non-master " << conn().toString() + << std::endl; + toolError() << "isMaster info: " << info << std::endl; return false; } @@ -353,66 +222,13 @@ namespace mongo { return isdbgrid["isdbgrid"].trueValue(); } - void Tool::addFieldOptions() { - add_options() - ("fields,f" , po::value<string>() , "comma separated list of field names e.g. -f name,age" ) - ("fieldFile" , po::value<string>() , "file with fields names - 1 per line" ) - ; - } - - void Tool::needFields() { - - if ( hasParam( "fields" ) ) { - BSONObjBuilder b; - - string fields_arg = getParam("fields"); - pcrecpp::StringPiece input(fields_arg); - - string f; - pcrecpp::RE re("([#\\w\\.\\s\\-]+),?" ); - while ( re.Consume( &input, &f ) ) { - _fields.push_back( f ); - b.append( f , 1 ); - } - - _fieldsObj = b.obj(); - return; - } - - if ( hasParam( "fieldFile" ) ) { - string fn = getParam( "fieldFile" ); - if ( ! boost::filesystem::exists( fn ) ) - throw UserException( 9999 , ((string)"file: " + fn ) + " doesn't exist" ); - - const int BUF_SIZE = 1024; - char line[ 1024 + 128]; - ifstream file( fn.c_str() ); - - BSONObjBuilder b; - while ( file.rdstate() == ios_base::goodbit ) { - file.getline( line , BUF_SIZE ); - const char * cur = line; - while ( isspace( cur[0] ) ) cur++; - if ( cur[0] == '\0' ) - continue; - - _fields.push_back( cur ); - b.append( cur , 1 ); - } - _fieldsObj = b.obj(); - return; - } - - throw UserException( 9998 , "you need to specify fields" ); - } - std::string Tool::getAuthenticationDatabase() { - if (!_authenticationDatabase.empty()) { - return _authenticationDatabase; + if (!toolGlobalParams.authenticationDatabase.empty()) { + return toolGlobalParams.authenticationDatabase; } - if (!_db.empty()) { - return _db; + if (!toolGlobalParams.db.empty()) { + return toolGlobalParams.db; } return "admin"; @@ -423,10 +239,10 @@ namespace mongo { */ void Tool::auth() { - if ( _username.empty() ) { + if (toolGlobalParams.username.empty()) { // Make sure that we don't need authentication to connect to this db // findOne throws an AssertionException if it's not authenticated. - if (_coll.size() > 0) { + if (toolGlobalParams.coll.size() > 0) { // BSONTools don't have a collection conn().findOne(getNS(), Query("{}"), 0, QueryOption_SlaveOk); } @@ -434,49 +250,51 @@ namespace mongo { return; } - _conn->auth( BSON( saslCommandPrincipalSourceFieldName << getAuthenticationDatabase() << - saslCommandPrincipalFieldName << _username << - saslCommandPasswordFieldName << _password << - saslCommandMechanismFieldName << _authenticationMechanism ) ); - } + BSONObjBuilder authParams; + authParams << + saslCommandUserDBFieldName << getAuthenticationDatabase() << + saslCommandUserFieldName << toolGlobalParams.username << + saslCommandPasswordFieldName << toolGlobalParams.password << + saslCommandMechanismFieldName << + toolGlobalParams.authenticationMechanism; - BSONTool::BSONTool( const char * name, DBAccess access , bool objcheck ) - : Tool( name , access , "" , "" , false ) , _objcheck( objcheck ) { + if (!toolGlobalParams.gssapiServiceName.empty()) { + authParams << saslCommandServiceNameFieldName << toolGlobalParams.gssapiServiceName; + } - add_options() - ("objcheck" , "validate object before inserting (default)" ) - ("noobjcheck" , "don't validate object before inserting" ) - ("filter" , po::value<string>() , "filter to apply before inserting" ) - ; + if (!toolGlobalParams.gssapiHostName.empty()) { + authParams << saslCommandServiceHostnameFieldName << toolGlobalParams.gssapiHostName; + } + + _conn->auth(authParams.obj()); } + BSONTool::BSONTool() : Tool() { } int BSONTool::run() { - if ( hasParam( "objcheck" ) ) - _objcheck = true; - else if ( hasParam( "noobjcheck" ) ) - _objcheck = false; - if ( hasParam( "filter" ) ) - _matcher.reset( new Matcher( fromjson( getParam( "filter" ) ) ) ); + if (bsonToolGlobalParams.hasFilter) { + _matcher.reset(new Matcher(fromjson(bsonToolGlobalParams.filter))); + } return doRun(); } long long BSONTool::processFile( const boost::filesystem::path& root ) { - _fileName = root.string(); + std::string fileName = root.string(); unsigned long long fileLength = file_size( root ); if ( fileLength == 0 ) { - out() << "file " << _fileName << " empty, skipping" << endl; + toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl; return 0; } - FILE* file = fopen( _fileName.c_str() , "rb" ); + FILE* file = fopen( fileName.c_str() , "rb" ); if ( ! file ) { - log() << "error opening file: " << _fileName << " " << errnoWithDescription() << endl; + toolError() << "error opening file: " << fileName << " " << errnoWithDescription() + << std::endl; return 0; } @@ -484,7 +302,9 @@ namespace mongo { posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL); #endif - LOG(1) << "\t file size: " << fileLength << endl; + if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { + toolInfoOutput() << "\t file size: " << fileLength << std::endl; + } unsigned long long read = 0; unsigned long long num = 0; @@ -494,8 +314,10 @@ namespace mongo { boost::scoped_array<char> buf_holder(new char[BUF_SIZE]); char * buf = buf_holder.get(); - ProgressMeter m( fileLength ); - m.setUnits( "bytes" ); + ProgressMeter m(fileLength); + if (!toolGlobalParams.quiet) { + m.setUnits( "bytes" ); + } while ( read < fileLength ) { size_t amt = fread(buf, 1, 4, file); @@ -508,24 +330,27 @@ namespace mongo { verify( amt == (size_t)( size - 4 ) ); BSONObj o( buf ); - if ( _objcheck && ! o.valid() ) { - cerr << "INVALID OBJECT - going try and pring out " << endl; - cerr << "size: " << size << endl; - BSONObjIterator i(o); - while ( i.more() ) { - BSONElement e = i.next(); + if (bsonToolGlobalParams.objcheck) { + const Status status = validateBSON(buf, size); + if (!status.isOK()) { + toolError() << "INVALID OBJECT - going to try and print out " << std::endl; + toolError() << "size: " << size << std::endl; + toolError() << "error: " << status.reason() << std::endl; + + StringBuilder sb; try { - e.validate(); + o.toString(sb); // using StringBuilder version to get as much as possible + } catch (...) { + toolError() << "object up to error: " << sb.str() << endl; + throw; } - catch ( ... ) { - cerr << "\t\t NEXT ONE IS INVALID" << endl; - } - cerr << "\t name : " << e.fieldName() << " " << e.type() << endl; - cerr << "\t " << e << endl; + toolError() << "complete object: " << sb.str() << endl; + + // NOTE: continuing with object even though we know it is invalid. } } - if ( _matcher.get() == 0 || _matcher->matches( o ) ) { + if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) { gotObject( o ); processed++; } @@ -533,16 +358,39 @@ namespace mongo { read += o.objsize(); num++; - m.hit( o.objsize() ); + if (!toolGlobalParams.quiet) { + m.hit(o.objsize()); + } } fclose( file ); - uassert( 10265 , "counts don't match" , m.done() == fileLength ); - (_usesstdout ? cout : cerr ) << m.hits() << " objects found" << endl; - if ( _matcher.get() ) - (_usesstdout ? cout : cerr ) << processed << " objects processed" << endl; + uassert(10265, "counts don't match", read == fileLength); + toolInfoOutput() << num << " objects found" << std::endl; + if (bsonToolGlobalParams.hasFilter) + toolInfoOutput() << processed << " objects processed" << std::endl; return processed; } } + +#if defined(_WIN32) +// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters +// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The +// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent +// and makes them available through the argv() and envp() members. This enables toolMain() +// to process UTF-8 encoded arguments and environment variables without regard to platform. +int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { + setWindowsUnhandledExceptionFilter(); + mongo::WindowsCommandLine wcl(argc, argvW, envpW); + auto_ptr<Tool> instance = (*Tool::createInstance)(); + int exitCode = instance->main(argc, wcl.argv(), wcl.envp()); + ::_exit(exitCode); +} + +#else +int main(int argc, char* argv[], char** envp) { + auto_ptr<Tool> instance = (*Tool::createInstance)(); + ::_exit(instance->main(argc, argv, envp)); +} +#endif diff --git a/src/mongo/tools/tool.h b/src/mongo/tools/tool.h index 2e7b0823d62..e5b627845bb 100644 --- a/src/mongo/tools/tool.h +++ b/src/mongo/tools/tool.h @@ -12,6 +12,18 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. */ // Tool.h @@ -20,14 +32,15 @@ #include <string> -#include <boost/program_options.hpp> - #if defined(_WIN32) #include <io.h> #endif -#include "db/instance.h" -#include "db/matcher.h" +#include "mongo/db/instance.h" +#include "mongo/db/matcher.h" +#include "mongo/tools/tool_logger.h" +#include "mongo/tools/tool_options.h" +#include "mongo/util/options_parser/environment.h" using std::string; @@ -35,122 +48,50 @@ namespace mongo { class Tool { public: - enum DBAccess { - NONE = 0 , - REMOTE_SERVER = 1 << 1 , - LOCAL_SERVER = 1 << 2 , - SPECIFY_DBCOL = 1 << 3 , - ALL = REMOTE_SERVER | LOCAL_SERVER | SPECIFY_DBCOL - }; - - Tool( string name , DBAccess access=ALL, string defaultDB="test" , - string defaultCollection="", bool usesstdout=true); + Tool(); virtual ~Tool(); - int main( int argc , char ** argv ); - - boost::program_options::options_description_easy_init add_options() { - return _options->add_options(); - } - boost::program_options::options_description_easy_init add_hidden_options() { - return _hidden_options->add_options(); - } - void addPositionArg( const char * name , int pos ) { - _positonalOptions.add( name , pos ); - } + static auto_ptr<Tool> (*createInstance)(); - string getParam( string name , string def="" ) { - if ( _params.count( name ) ) - return _params[name.c_str()].as<string>(); - return def; - } - int getParam( string name , int def ) { - if ( _params.count( name ) ) - return _params[name.c_str()].as<int>(); - return def; - } - bool hasParam( string name ) { - return _params.count( name ); - } + int main( int argc , char ** argv, char ** envp ); string getNS() { - if ( _coll.size() == 0 ) { + if (toolGlobalParams.coll.size() == 0) { cerr << "no collection specified!" << endl; throw -1; } - return _db + "." + _coll; + return toolGlobalParams.db + "." + toolGlobalParams.coll; } string getAuthenticationDatabase(); - void useStandardOutput( bool mode ) { - _usesstdout = mode; - } - bool isMaster(); bool isMongos(); - - virtual void preSetup() {} virtual int run() = 0; - virtual void printHelp(ostream &out); - - virtual void printExtraHelp( ostream & out ) {} - virtual void printExtraHelpAfter( ostream & out ) {} - - virtual void printVersion(ostream &out); + virtual void printHelp(ostream &out) = 0; protected: mongo::DBClientBase &conn( bool slaveIfPaired = false ); - string _name; - - string _db; - string _coll; - string _fileName; - - string _username; - string _password; - string _authenticationDatabase; - string _authenticationMechanism; - - bool _usesstdout; - bool _noconnection; bool _autoreconnect; - void addFieldOptions(); - void needFields(); - - vector<string> _fields; - BSONObj _fieldsObj; - - - string _host; - protected: mongo::DBClientBase * _conn; mongo::DBClientBase * _slaveConn; - bool _paired; - - boost::program_options::options_description * _options; - boost::program_options::options_description * _hidden_options; - boost::program_options::positional_options_description _positonalOptions; - - boost::program_options::variables_map _params; private: void auth(); }; class BSONTool : public Tool { - bool _objcheck; auto_ptr<Matcher> _matcher; public: - BSONTool( const char * name , DBAccess access=ALL, bool objcheck = true ); + BSONTool(); virtual int doRun() = 0; virtual void gotObject( const BSONObj& obj ) = 0; @@ -162,3 +103,7 @@ namespace mongo { }; } + +#define REGISTER_MONGO_TOOL(TYPENAME) \ + auto_ptr<Tool> createInstanceOfThisTool() {return auto_ptr<Tool>(new TYPENAME());} \ + auto_ptr<Tool> (*Tool::createInstance)() = createInstanceOfThisTool; diff --git a/src/mongo/tools/tool_logger.cpp b/src/mongo/tools/tool_logger.cpp new file mode 100644 index 00000000000..fd906ef18d1 --- /dev/null +++ b/src/mongo/tools/tool_logger.cpp @@ -0,0 +1,144 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/tools/tool_logger.h" + +#include <iostream> + +#include "mongo/base/init.h" +#include "mongo/logger/console_appender.h" +#include "mongo/logger/log_manager.h" +#include "mongo/logger/logger.h" +#include "mongo/logger/message_event.h" +#include "mongo/logger/message_event_utf8_encoder.h" +#include "mongo/tools/tool_options.h" +#include "mongo/util/log.h" + +namespace mongo { +namespace { + + /* + * Theory of operation: + * + * At process start, the loader initializes "consoleMutex" to NULL. At some point during static + * initialization, the static initialization process, running in the one and only extant thread, + * allocates a new boost::mutex on the heap and assigns consoleMutex to point to it. While + * consoleMutex is still NULL, we know that there is only one thread extant, so it is safe to + * skip locking the consoleMutex in the ErrorConsole constructor. Once the mutex is initialized, + * users of ErrorConsole can start acquiring it. + */ + + boost::mutex *consoleMutex = new boost::mutex; + +} // namespace + + ErrorConsole::ErrorConsole() : _consoleLock() { + if (consoleMutex) { + boost::unique_lock<boost::mutex> lk(*consoleMutex); + lk.swap(_consoleLock); + } + } + + std::ostream& ErrorConsole::out() { return std::cerr; } + +namespace { + + logger::MessageLogDomain* toolErrorOutput = NULL; + logger::MessageLogDomain* toolNonErrorOutput = NULL; + logger::MessageLogDomain* toolNonErrorDecoratedOutput = NULL; + +} // namespace + +MONGO_INITIALIZER_GENERAL(ToolLogRedirection, + ("GlobalLogManager", "EndStartupOptionHandling"), + ("default"))(InitializerContext*) { + + using logger::MessageEventEphemeral; + using logger::MessageEventDetailsEncoder; + using logger::MessageLogDomain; + using logger::ConsoleAppender; + + toolErrorOutput = logger::globalLogManager()->getNamedDomain("toolErrorOutput"); + toolNonErrorOutput = logger::globalLogManager()->getNamedDomain("toolNonErrorOutput"); + toolNonErrorDecoratedOutput = + logger::globalLogManager()->getNamedDomain("toolNonErrorDecoratedOutput"); + + // Errors in the tools always go to stderr + toolErrorOutput->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( + new logger::MessageEventUnadornedEncoder))); + + // If we are outputting data to stdout, we may need to redirect all logging to stderr + if (!toolGlobalParams.canUseStdout) { + logger::globalLogDomain()->clearAppenders(); + logger::globalLogDomain()->attachAppender(MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( + new MessageEventDetailsEncoder))); + } + + // Only put an appender on our informational messages if we did not use --quiet + if (!toolGlobalParams.quiet) { + if (toolGlobalParams.canUseStdout) { + // If we can use stdout, we can use the ConsoleAppender with the default console + toolNonErrorOutput->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral>( + new logger::MessageEventUnadornedEncoder))); + + toolNonErrorDecoratedOutput->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral>( + new logger::MessageEventDetailsEncoder))); + } + else { + // If we cannot use stdout, we have to use ErrorConsole to redirect informational + // messages to stderr + toolNonErrorOutput->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( + new logger::MessageEventUnadornedEncoder))); + + toolNonErrorDecoratedOutput->attachAppender( + MessageLogDomain::AppenderAutoPtr( + new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( + new logger::MessageEventDetailsEncoder))); + } + } + + return Status::OK(); +} + + LogstreamBuilder toolInfoLog() { + return LogstreamBuilder(toolNonErrorDecoratedOutput, + "", + logger::LogSeverity::Log()); + } + + LogstreamBuilder toolInfoOutput() { + return LogstreamBuilder(toolNonErrorOutput, + "", + logger::LogSeverity::Log()); + } + + LogstreamBuilder toolError() { + return LogstreamBuilder(toolErrorOutput, + "", + logger::LogSeverity::Log()); + } + +} // namespace mongo diff --git a/src/mongo/tools/tool_logger.h b/src/mongo/tools/tool_logger.h new file mode 100644 index 00000000000..6aac6217281 --- /dev/null +++ b/src/mongo/tools/tool_logger.h @@ -0,0 +1,58 @@ +/* Copyright 2013 10gen Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <boost/thread/mutex.hpp> +#include <iosfwd> + +#include "mongo/logger/logstream_builder.h" + +namespace mongo { + + /** + * This is a version of the Console class that uses stderr for output instead of stdout. See + * the description of the Console class for other details about how this class should operate + */ + class ErrorConsole { + public: + ErrorConsole(); + + std::ostream& out(); + + private: + boost::unique_lock<boost::mutex> _consoleLock; + }; + + using logger::LogstreamBuilder; + + /* + * Informational messages. Messages sent here will go to stdout normally, stderr if data is + * being sent to stdout, and be silenced if the user specifies --quiet. + */ + LogstreamBuilder toolInfoOutput(); + /* + * Informational messages. Messages sent here will go to stdout normally, stderr if data is + * being sent to stdout, and be silenced if the user specifies --quiet. Incudes extra log + * decoration. + */ + LogstreamBuilder toolInfoLog(); + /* + * Error messages. Messages sent here should always go to stderr and not be silenced by + * --quiet. + */ + LogstreamBuilder toolError(); + +} // namespace mongo diff --git a/src/mongo/tools/tool_options.cpp b/src/mongo/tools/tool_options.cpp new file mode 100644 index 00000000000..9aae245339e --- /dev/null +++ b/src/mongo/tools/tool_options.cpp @@ -0,0 +1,400 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/tools/tool_options.h" + +#include <boost/filesystem/operations.hpp> +#include <fstream> +#include "pcrecpp.h" + +#include "mongo/base/status.h" +#include "mongo/client/sasl_client_authenticate.h" +#include "mongo/db/storage_options.h" +#include "mongo/util/log.h" +#include "mongo/util/net/sock.h" +#include "mongo/util/net/ssl_manager.h" +#include "mongo/util/net/ssl_options.h" +#include "mongo/util/options_parser/startup_options.h" +#include "mongo/util/password.h" +#include "mongo/util/text.h" +#include "mongo/util/version.h" + +namespace mongo { + + ToolGlobalParams toolGlobalParams; + BSONToolGlobalParams bsonToolGlobalParams; + + Status addGeneralToolOptions(moe::OptionSection* options) { + options->addOptionChaining("help", "help", moe::Switch, "produce help message"); + + options->addOptionChaining("verbose", "verbose,v", moe::Switch, + "be more verbose (include multiple times " + "for more verbosity e.g. -vvvvv)"); + + options->addOptionChaining("quiet", "quiet", moe::Switch, + "silence all non error diagnostic messages"); + + options->addOptionChaining("version", "version", moe::Switch, + "print the program's version and exit"); + + + /* support for -vv -vvvv etc. */ + for (string s = "vv"; s.length() <= 10; s.append("v")) { + options->addOptionChaining(s.c_str(), s.c_str(), moe::Switch, "verbose") + .hidden(); + } + + return Status::OK(); + } + + Status addRemoteServerToolOptions(moe::OptionSection* options) { + options->addOptionChaining("host", "host,h", moe::String, + "mongo host to connect to ( <set name>/s1,s2 for sets)"); + + options->addOptionChaining("port", "port", moe::Int, + "server port. Can also use --host hostname:port") + .validRange(0, 65535); + + options->addOptionChaining("ipv6", "ipv6", moe::Switch, + "enable IPv6 support (disabled by default)"); + + +#ifdef MONGO_SSL + Status ret = addSSLClientOptions(options); + if (!ret.isOK()) { + return ret; + } +#endif + + options->addOptionChaining("username", "username,u", moe::String, + "username"); + + // We ask a user for a password if they pass in an empty string or pass --password with no + // argument. This must be handled when the password value is checked. + // + // Desired behavior: + // --username test // Continue with username "test" and no password + // --username test --password test // Continue with username "test" and password "test" + // --username test --password // Continue with username "test" and prompt for password + // --username test --password "" // Continue with username "test" and prompt for password + // + // To do this we pass moe::Value(std::string("")) as the "implicit value" of this option + options->addOptionChaining("password", "password,p", moe::String, "password") + .setImplicit(moe::Value(std::string(""))); + + options->addOptionChaining("authenticationDatabase", "authenticationDatabase", moe::String, + "user source (defaults to dbname)") + .setDefault(moe::Value(std::string(""))); + + options->addOptionChaining("authenticationMechanism", "authenticationMechanism", + moe::String, "authentication mechanism") + .setDefault(moe::Value(std::string("MONGODB-CR"))); + + options->addOptionChaining("gssapiServiceName", "gssapiServiceName", + moe::String, + "Service name to use when authenticating using GSSAPI/Kerberos") + .setDefault(moe::Value(std::string(saslDefaultServiceName))); + + options->addOptionChaining("gssapiHostName", "gssapiHostName", moe::String, + "Remote host name to use for purpose of GSSAPI/Kerberos authentication"); + + return Status::OK(); + } + + Status addLocalServerToolOptions(moe::OptionSection* options) { + options->addOptionChaining("dbpath", "dbpath", moe::String, + "directly access mongod database files in the given path, instead of " + "connecting to a mongod server - needs to lock the data directory, " + "so cannot be used if a mongod is currently accessing the same path"); + + options->addOptionChaining("directoryperdb", "directoryperdb", moe::Switch, + "each db is in a separate directory (relevant only if dbpath specified)"); + + options->addOptionChaining("journal", "journal", moe::Switch, + "enable journaling (relevant only if dbpath specified)"); + + + return Status::OK(); + } + Status addSpecifyDBCollectionToolOptions(moe::OptionSection* options) { + options->addOptionChaining("db", "db,d", moe::String, "database to use"); + + options->addOptionChaining("collection", "collection,c", moe::String, + "collection to use (some commands)"); + + + return Status::OK(); + } + + Status addFieldOptions(moe::OptionSection* options) { + options->addOptionChaining("fields", "fields,f", moe::String, + "comma separated list of field names e.g. -f name,age"); + + options->addOptionChaining("fieldFile", "fieldFile", moe::String, + "file with field names - 1 per line"); + + + return Status::OK(); + } + + Status addBSONToolOptions(moe::OptionSection* options) { + options->addOptionChaining("objcheck", "objcheck", moe::Switch, + "validate object before inserting (default)"); + + options->addOptionChaining("noobjcheck", "noobjcheck", moe::Switch, + "don't validate object before inserting"); + + options->addOptionChaining("filter", "filter", moe::String, + "filter to apply before inserting"); + + + return Status::OK(); + } + + std::string getParam(std::string name, std::string def) { + if (moe::startupOptionsParsed.count(name)) { + return moe::startupOptionsParsed[name.c_str()].as<string>(); + } + return def; + } + int getParam(std::string name, int def) { + if (moe::startupOptionsParsed.count(name)) { + return moe::startupOptionsParsed[name.c_str()].as<int>(); + } + return def; + } + bool hasParam(std::string name) { + return moe::startupOptionsParsed.count(name); + } + + void printToolVersionString(std::ostream &out) { + out << toolGlobalParams.name << " version " << mongo::versionString; + if (mongo::versionString[strlen(mongo::versionString)-1] == '-') + out << " (commit " << mongo::gitVersion() << ")"; + out << std::endl; + } + + bool handlePreValidationGeneralToolOptions(const moe::Environment& params) { + if (moe::startupOptionsParsed.count("version")) { + printToolVersionString(std::cout); + return false; + } + return true; + } + + extern bool directoryperdb; + + Status storeGeneralToolOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + + toolGlobalParams.name = args[0]; + + storageGlobalParams.prealloc = false; + + // The default value may vary depending on compile options, but for tools + // we want durability to be disabled. + storageGlobalParams.dur = false; + + // Set authentication parameters + if (params.count("authenticationDatabase")) { + toolGlobalParams.authenticationDatabase = + params["authenticationDatabase"].as<string>(); + } + + if (params.count("authenticationMechanism")) { + toolGlobalParams.authenticationMechanism = + params["authenticationMechanism"].as<string>(); + } + + if (params.count("gssapiServiceName")) { + toolGlobalParams.gssapiServiceName = params["gssapiServiceName"].as<string>(); + } + + if (params.count("gssapiHostName")) { + toolGlobalParams.gssapiHostName = params["gssapiHostName"].as<string>(); + } + if (params.count("verbose")) { + logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(1)); + } + + for (string s = "vv"; s.length() <= 12; s.append("v")) { + if (params.count(s)) { + logger::globalLogDomain()->setMinimumLoggedSeverity( + logger::LogSeverity::Debug(s.length())); + } + } + + toolGlobalParams.quiet = params.count("quiet"); + +#ifdef MONGO_SSL + Status ret = storeSSLClientOptions(params); + if (!ret.isOK()) { + return ret; + } +#endif + + if (args.empty()) { + return Status(ErrorCodes::InternalError, "Cannot get binary name: argv array is empty"); + } + + // setup binary name + toolGlobalParams.name = args[0]; + size_t i = toolGlobalParams.name.rfind('/'); + if (i != string::npos) { + toolGlobalParams.name = toolGlobalParams.name.substr(i + 1); + } + toolGlobalParams.db = "test"; + toolGlobalParams.coll = ""; + toolGlobalParams.noconnection = false; + + if (params.count("db")) + toolGlobalParams.db = params["db"].as<string>(); + + if (params.count("collection")) + toolGlobalParams.coll = params["collection"].as<string>(); + + if (params.count("username")) + toolGlobalParams.username = params["username"].as<string>(); + + if (params.count("password")) { + toolGlobalParams.password = params["password"].as<string>(); + if (toolGlobalParams.password.empty()) { + toolGlobalParams.password = askPassword(); + } + } + + if (params.count("ipv6")) { + enableIPv6(); + } + + toolGlobalParams.dbpath = getParam("dbpath"); + toolGlobalParams.useDirectClient = hasParam("dbpath"); + if (toolGlobalParams.useDirectClient && params.count("journal")) { + storageGlobalParams.dur = true; + } + + if (!toolGlobalParams.useDirectClient) { + toolGlobalParams.connectionString = "127.0.0.1"; + if (params.count("host")) { + toolGlobalParams.hostSet = true; + toolGlobalParams.host = params["host"].as<string>(); + toolGlobalParams.connectionString = params["host"].as<string>(); + } + + if (params.count("port")) { + toolGlobalParams.portSet = true; + toolGlobalParams.port = params["port"].as<int>(); + StringBuilder sb; + sb << toolGlobalParams.connectionString << ':' << toolGlobalParams.port; + toolGlobalParams.connectionString = sb.str(); + } + } + else { + if (params.count("directoryperdb")) { + storageGlobalParams.directoryperdb = true; + } + + toolGlobalParams.connectionString = "DIRECT"; + } + + return Status::OK(); + } + + Status storeFieldOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + + toolGlobalParams.fieldsSpecified = false; + + if (hasParam("fields")) { + toolGlobalParams.fieldsSpecified = true; + + string fields_arg = getParam("fields"); + pcrecpp::StringPiece input(fields_arg); + + string f; + pcrecpp::RE re("([#\\w\\.\\s\\-]+),?" ); + while ( re.Consume( &input, &f ) ) { + toolGlobalParams.fields.push_back( f ); + } + return Status::OK(); + } + + if (hasParam("fieldFile")) { + toolGlobalParams.fieldsSpecified = true; + + string fn = getParam("fieldFile"); + if (!boost::filesystem::exists(fn)) { + StringBuilder sb; + sb << "file: " << fn << " doesn't exist"; + return Status(ErrorCodes::InternalError, sb.str()); + } + + const int BUF_SIZE = 1024; + char line[1024 + 128]; + std::ifstream file(fn.c_str()); + + while (file.rdstate() == std::ios_base::goodbit) { + file.getline(line, BUF_SIZE); + const char * cur = line; + while (isspace(cur[0])) cur++; + if (cur[0] == '\0') + continue; + + toolGlobalParams.fields.push_back(cur); + } + return Status::OK(); + } + + return Status::OK(); + } + + + Status storeBSONToolOptions(const moe::Environment& params, + const std::vector<std::string>& args) { + + bsonToolGlobalParams.objcheck = true; + + if (hasParam("objcheck") && hasParam("noobjcheck")) { + return Status(ErrorCodes::BadValue, "can't have both --objcheck and --noobjcheck"); + } + + if (hasParam("objcheck")) { + bsonToolGlobalParams.objcheck = true; + } + else if (hasParam("noobjcheck")) { + bsonToolGlobalParams.objcheck = false; + } + + if (hasParam("filter")) { + bsonToolGlobalParams.hasFilter = true; + bsonToolGlobalParams.filter = getParam("filter"); + } + + return Status::OK(); + } +} diff --git a/src/mongo/tools/tool_options.h b/src/mongo/tools/tool_options.h new file mode 100644 index 00000000000..6644a0de58c --- /dev/null +++ b/src/mongo/tools/tool_options.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2010 10gen Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#pragma once + +#include <iosfwd> +#include <string> +#include <vector> + +#include "mongo/base/status.h" + +namespace mongo { + + namespace optionenvironment { + class OptionSection; + class Environment; + } // namespace optionenvironment + + namespace moe = mongo::optionenvironment; + + struct ToolGlobalParams { + + ToolGlobalParams() : canUseStdout(true), hostSet(false), portSet(false) { } + std::string name; + + std::string db; + std::string coll; + + std::string username; + std::string password; + std::string authenticationDatabase; + std::string authenticationMechanism; + std::string gssapiServiceName; + std::string gssapiHostName; + + bool quiet; + bool canUseStdout; + bool noconnection; + + std::vector<std::string> fields; + bool fieldsSpecified; + + std::string host; // --host + bool hostSet; + int port; // --port + bool portSet; + std::string connectionString; // --host and --port after processing + std::string dbpath; + bool useDirectClient; + }; + + extern ToolGlobalParams toolGlobalParams; + + struct BSONToolGlobalParams { + bool objcheck; + std::string filter; + bool hasFilter; + }; + + extern BSONToolGlobalParams bsonToolGlobalParams; + + Status addGeneralToolOptions(moe::OptionSection* options); + + Status addRemoteServerToolOptions(moe::OptionSection* options); + + Status addLocalServerToolOptions(moe::OptionSection* options); + + Status addSpecifyDBCollectionToolOptions(moe::OptionSection* options); + + Status addBSONToolOptions(moe::OptionSection* options); + + Status addFieldOptions(moe::OptionSection* options); + + // Legacy interface for getting options in tools + // TODO: Remove this when we use the new interface everywhere + std::string getParam(std::string name, string def=""); + int getParam(std::string name, int def); + bool hasParam(std::string name); + + /** + * Handle options that should come before validation, such as "help". + * + * Returns false if an option was found that implies we should prematurely exit with success. + */ + bool handlePreValidationGeneralToolOptions(const moe::Environment& params); + + Status storeGeneralToolOptions(const moe::Environment& params, + const std::vector<std::string>& args); + + Status storeFieldOptions(const moe::Environment& params, + const std::vector<std::string>& args); + + Status storeBSONToolOptions(const moe::Environment& params, + const std::vector<std::string>& args); +} diff --git a/src/mongo/tools/top.cpp b/src/mongo/tools/top.cpp index a0af090c1c6..3b5c09dcbf1 100644 --- a/src/mongo/tools/top.cpp +++ b/src/mongo/tools/top.cpp @@ -1,5 +1,3 @@ -// top.cpp - /** * Copyright (C) 2008 10gen Inc. * @@ -14,50 +12,46 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects +* for all of the code used other than as permitted herein. If you modify +* file(s) with this exception, you may extend this exception to your +* version of the file(s), but you are not obligated to do so. If you do not +* wish to do so, delete this exception statement from your version. If you +* delete this exception statement from all source files in the program, +* then also delete it in the license file. */ -#include "pch.h" +#include "mongo/pch.h" -#include "mongo/base/initializer.h" -#include "db/json.h" -#include "../util/text.h" -#include "tool.h" -#include "stat_util.h" #include <fstream> #include <iostream> -#include <boost/program_options.hpp> -namespace po = boost::program_options; +#include "mongo/db/json.h" +#include "mongo/tools/stat_util.h" +#include "mongo/tools/mongotop_options.h" +#include "mongo/tools/tool.h" +#include "mongo/util/options_parser/option_section.h" namespace mongo { class TopTool : public Tool { public: - TopTool() : Tool( "top" , REMOTE_SERVER , "admin" ) { - _sleep = 1; - - add_hidden_options() - ( "sleep" , po::value<int>() , "time to sleep between calls" ) - ; - add_options() - ( "locks" , "use db lock info instead of top" ) - ; - addPositionArg( "sleep" , 1 ); - + TopTool() : Tool() { _autoreconnect = true; } - virtual void printExtraHelp( ostream & out ) { - out << "View live MongoDB collection statistics.\n" << endl; - } - - bool useLocks() { - return hasParam( "locks" ); + virtual void printHelp( ostream & out ) { + printMongoTopHelp(&out); } NamespaceStats getData() { - if ( useLocks() ) + if (mongoTopGlobalParams.useLocks) return getDataLocks(); return getDataTop(); } @@ -65,8 +59,8 @@ namespace mongo { NamespaceStats getDataLocks() { BSONObj out; - if ( ! conn().simpleCommand( _db , &out , "serverStatus" ) ) { - cout << "error: " << out << endl; + if (!conn().simpleCommand(toolGlobalParams.db, &out, "serverStatus")) { + toolError() << "error: " << out << std::endl; return NamespaceStats(); } @@ -77,13 +71,13 @@ namespace mongo { NamespaceStats stats; BSONObj out; - if ( ! conn().simpleCommand( _db , &out , "top" ) ) { - cout << "error: " << out << endl; + if (!conn().simpleCommand(toolGlobalParams.db, &out, "top")) { + toolError() << "error: " << out << std::endl; return stats; } if ( ! out["totals"].isABSONObj() ) { - cout << "error: invalid top\n" << out << endl; + toolError() << "error: invalid top\n" << out << std::endl; return stats; } @@ -117,7 +111,7 @@ namespace mongo { for ( unsigned i=0; i < data.size(); i++ ) { const string& ns = data[i].ns; - if ( ! useLocks() && ns.find( '.' ) == string::npos ) + if (!mongoTopGlobalParams.useLocks && ns.find('.') == string::npos) continue; if ( ns.size() > longest ) @@ -127,7 +121,7 @@ namespace mongo { int numberWidth = 10; cout << "\n" - << setw(longest) << ( useLocks() ? "db" : "ns" ) + << setw(longest) << (mongoTopGlobalParams.useLocks ? "db" : "ns") << setw(numberWidth+2) << "total" << setw(numberWidth+2) << "read" << setw(numberWidth+2) << "write" @@ -135,7 +129,7 @@ namespace mongo { << endl; for ( int i=data.size()-1; i>=0 && data.size() - i < 10 ; i-- ) { - if ( ! useLocks() && data[i].ns.find( '.' ) == string::npos ) + if (!mongoTopGlobalParams.useLocks && data[i].ns.find('.') == string::npos) continue; cout << setw(longest) << data[i].ns @@ -148,24 +142,22 @@ namespace mongo { } int run() { - _sleep = getParam( "sleep" , _sleep ); - if (isMongos()) { - log() << "mongotop only works on instances of mongod." << endl; + toolError() << "mongotop only works on instances of mongod." << std::endl; return EXIT_FAILURE; } NamespaceStats prev = getData(); while ( true ) { - sleepsecs( _sleep ); + sleepsecs(mongoTopGlobalParams.sleep); NamespaceStats now; try { now = getData(); } catch ( std::exception& e ) { - cout << "can't get data: " << e.what() << endl; + toolError() << "can't get data: " << e.what() << std::endl; continue; } @@ -176,7 +168,7 @@ namespace mongo { printDiff( prev , now ); } catch ( AssertionException& e ) { - cout << "\nerror: " << e.what() << endl; + toolError() << "\nerror: " << e.what() << std::endl; } prev = now; @@ -184,16 +176,8 @@ namespace mongo { return 0; } - - private: - int _sleep; }; -} + REGISTER_MONGO_TOOL(TopTool); -int main( int argc , char ** argv, char ** envp ) { - mongo::runGlobalInitializersOrDie(argc, argv, envp); - mongo::TopTool top; - return top.main( argc , argv ); } - |