diff options
Diffstat (limited to 'src')
157 files changed, 4516 insertions, 1524 deletions
diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 66ba4054a11..de51c167ce8 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -355,6 +355,7 @@ env.Install( 'util/clock_sources', 'util/fail_point', 'util/ntservice', + 'util/net/ssl_manager_status', 'util/options_parser/options_parser_init', 'util/version_impl', ])) diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 98d41461fce..0604e9b4cf7 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -201,6 +201,7 @@ error_code("ChunkRangeCleanupPending", 200) error_code("CannotBuildIndexKeys", 201) error_code("NetworkInterfaceExceededTimeLimit", 202) error_code("TooManyLocks", 208) +error_code("KeyNotFound", 211) error_code("UpdateOperationFailed", 218) error_code("FTDCPathNotSet", 219) error_code("FTDCPathAlreadySet", 220) diff --git a/src/mongo/client/dbclient.cpp b/src/mongo/client/dbclient.cpp index 6f75a704d34..baa156c5b9c 100644 --- a/src/mongo/client/dbclient.cpp +++ b/src/mongo/client/dbclient.cpp @@ -441,7 +441,7 @@ void DBClientWithCommands::_auth(const BSONObj& params) { std::string clientName = ""; #ifdef MONGO_CONFIG_SSL if (sslManager() != nullptr) { - clientName = sslManager()->getSSLConfiguration().clientSubjectName; + clientName = sslManager()->getSSLConfiguration().clientSubjectName.toString(); } #endif diff --git a/src/mongo/client/replica_set_monitor_manager.h b/src/mongo/client/replica_set_monitor_manager.h index 2730167668f..731b5122984 100644 --- a/src/mongo/client/replica_set_monitor_manager.h +++ b/src/mongo/client/replica_set_monitor_manager.h @@ -98,11 +98,13 @@ private: // Protects access to the replica set monitors stdx::mutex _mutex; - ReplicaSetMonitorsMap _monitors; // Executor for monitoring replica sets. std::unique_ptr<executor::TaskExecutor> _taskExecutor; + // Needs to be after `_taskExecutor`, so that it will be destroyed before the `_taskExecutor`. + ReplicaSetMonitorsMap _monitors; + void _setupTaskExecutorInLock(const std::string& name); // set to true when shutdown has been called. diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index ded6480875c..6c3638a9a10 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -663,6 +663,7 @@ serveronlyLibdeps = [ "$BUILD_DIR/mongo/util/clock_sources", "$BUILD_DIR/mongo/util/elapsed_tracker", "$BUILD_DIR/mongo/util/net/network", + "$BUILD_DIR/mongo/util/net/ssl_manager_status", "$BUILD_DIR/mongo/db/storage/mmap_v1/file_allocator", "$BUILD_DIR/third_party/shim_snappy", '$BUILD_DIR/mongo/db/ttl_collection_cache', diff --git a/src/mongo/db/auth/auth_index_d.cpp b/src/mongo/db/auth/auth_index_d.cpp index 47987ac40c4..9944a582f5c 100644 --- a/src/mongo/db/auth/auth_index_d.cpp +++ b/src/mongo/db/auth/auth_index_d.cpp @@ -47,6 +47,7 @@ #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/jsobj.h" +#include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/storage/storage_options.h" #include "mongo/util/assert_util.h" #include "mongo/util/log.h" @@ -105,6 +106,15 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, return; } + // Do not try to generate any system indexes on a secondary. + auto replCoord = repl::ReplicationCoordinator::get(opCtx); + uassert(ErrorCodes::NotMaster, + "Not primary while creating authorization index", + replCoord->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet || + replCoord->canAcceptWritesForDatabase(ns.db())); + + invariant(!opCtx->lockState()->inAWriteUnitOfWork()); + try { auto indexSpecStatus = index_key_validate::validateIndexSpec( spec.toBSON(), ns, serverGlobalParams.featureCompatibility); @@ -115,8 +125,10 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, MultiIndexBlock indexer(opCtx, collection); + std::vector<BSONObj> indexInfoObjs; MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { - fassertStatusOK(40453, indexer.init(indexSpec)); + indexInfoObjs = fassertStatusOK(40453, indexer.init(indexSpec)); + invariant(indexInfoObjs.size() == 1); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(opCtx, "authorization index regeneration", ns.ns()); @@ -126,6 +138,8 @@ void generateSystemIndexForExistingCollection(OperationContext* opCtx, WriteUnitOfWork wunit(opCtx); indexer.commit(); + opCtx->getServiceContext()->getOpObserver()->onCreateIndex( + opCtx, ns.getSystemIndexesCollection(), indexInfoObjs[0], false /* fromMigrate */); wunit.commit(); } @@ -205,22 +219,23 @@ Status verifySystemIndexes(OperationContext* txn) { void createSystemIndexes(OperationContext* txn, Collection* collection) { invariant(collection); const NamespaceString& ns = collection->ns(); + BSONObj indexSpec; if (ns == AuthorizationManager::usersCollectionNamespace) { - auto indexSpec = fassertStatusOK( + indexSpec = fassertStatusOK( 40455, index_key_validate::validateIndexSpec( v3SystemUsersIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); - - fassertStatusOK( - 40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } else if (ns == AuthorizationManager::rolesCollectionNamespace) { - auto indexSpec = fassertStatusOK( + indexSpec = fassertStatusOK( 40457, index_key_validate::validateIndexSpec( v3SystemRolesIndexSpec.toBSON(), ns, serverGlobalParams.featureCompatibility)); - + } + if (!indexSpec.isEmpty()) { + txn->getServiceContext()->getOpObserver()->onCreateIndex( + txn, ns.getSystemIndexesCollection(), indexSpec, false /* fromMigrate */); fassertStatusOK( - 40458, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); + 40456, collection->getIndexCatalog()->createIndexOnEmptyCollection(txn, indexSpec)); } } diff --git a/src/mongo/db/auth/authorization_manager_test.cpp b/src/mongo/db/auth/authorization_manager_test.cpp index 50f823f014a..ea39d0834a7 100644 --- a/src/mongo/db/auth/authorization_manager_test.cpp +++ b/src/mongo/db/auth/authorization_manager_test.cpp @@ -32,6 +32,7 @@ */ #include "mongo/base/status.h" #include "mongo/bson/mutable/document.h" +#include "mongo/config.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_manager.h" @@ -55,6 +56,12 @@ namespace mongo { namespace { +// Construct a simple, structured X509 name equivalent to "CN=mongodb.com" +SSLX509Name buildX509Name() { + return SSLX509Name(std::vector<std::vector<SSLX509Name::Entry>>( + {{{kOID_CommonName.toString(), 19 /* Printable String */, "mongodb.com"}}})); +} + using std::vector; TEST(RoleParsingTest, BuildRoleBSON) { @@ -241,13 +248,14 @@ TEST_F(AuthorizationManagerTest, testAcquireV2User) { authzManager->releaseUser(v2cluster); } +#ifdef MONGO_CONFIG_SSL TEST_F(AuthorizationManagerTest, testLocalX509Authorization) { ServiceContextNoop serviceContext; transport::TransportLayerMock transportLayer{}; transport::SessionHandle session = transportLayer.createSession(); transportLayer.setX509PeerInfo( session, - SSLPeerInfo("CN=mongodb.com", {RoleName("read", "test"), RoleName("readWrite", "test")})); + SSLPeerInfo(buildX509Name(), {RoleName("read", "test"), RoleName("readWrite", "test")})); ServiceContext::UniqueClient client = serviceContext.makeClient("testClient", session); ServiceContext::UniqueOperationContext txn = client->makeOperationContext(); @@ -274,6 +282,7 @@ TEST_F(AuthorizationManagerTest, testLocalX509Authorization) { authzManager->releaseUser(x509User); } +#endif TEST_F(AuthorizationManagerTest, testLocalX509AuthorizationInvalidUser) { ServiceContextNoop serviceContext; @@ -281,7 +290,7 @@ TEST_F(AuthorizationManagerTest, testLocalX509AuthorizationInvalidUser) { transport::SessionHandle session = transportLayer.createSession(); transportLayer.setX509PeerInfo( session, - SSLPeerInfo("CN=mongodb.com", {RoleName("read", "test"), RoleName("write", "test")})); + SSLPeerInfo(buildX509Name(), {RoleName("read", "test"), RoleName("write", "test")})); ServiceContext::UniqueClient client = serviceContext.makeClient("testClient", session); ServiceContext::UniqueOperationContext txn = client->makeOperationContext(); diff --git a/src/mongo/db/auth/authorization_session.cpp b/src/mongo/db/auth/authorization_session.cpp index a91f350e24d..c4f219c15de 100644 --- a/src/mongo/db/auth/authorization_session.cpp +++ b/src/mongo/db/auth/authorization_session.cpp @@ -679,7 +679,12 @@ static int buildResourceSearchList(const ResourcePattern& target, // Some databases should not be matchable with ResourcePattern::forAnyNormalResource. // 'local' and 'config' are used to store special system collections, which user level // administrators should not be able to manipulate. - if (target.ns().db() != "local" && target.ns().db() != "config") { + // '$setFeatureCompatibilityVersion' is a virtual database that + // setFeatureCompatibilityVersion performs auth checks against. When this command was + // first written, there was a moratorium on creating new ActionTypes. SERVER-31983 + // introduced the ActionType after the moratorium expired. + if (target.ns().db() != "local" && target.ns().db() != "config" && + target.ns().db() != "$setFeatureCompatibilityVersion") { resourceSearchList[size++] = ResourcePattern::forAnyNormalResource(); } resourceSearchList[size++] = ResourcePattern::forDatabaseName(target.ns().db()); diff --git a/src/mongo/db/auth/authz_manager_external_state.cpp b/src/mongo/db/auth/authz_manager_external_state.cpp index ed5f0fe6bfd..0403af8e256 100644 --- a/src/mongo/db/auth/authz_manager_external_state.cpp +++ b/src/mongo/db/auth/authz_manager_external_state.cpp @@ -28,6 +28,7 @@ #include "mongo/platform/basic.h" +#include "mongo/config.h" #include "mongo/db/auth/authz_manager_external_state.h" #include "mongo/db/auth/user_name.h" #include "mongo/db/operation_context.h" @@ -42,10 +43,17 @@ AuthzManagerExternalState::~AuthzManagerExternalState() = default; bool AuthzManagerExternalState::shouldUseRolesFromConnection(OperationContext* txn, const UserName& userName) { - return txn && txn->getClient() && txn->getClient()->session() && - txn->getClient()->session()->getX509PeerInfo().subjectName == userName.getUser() && - userName.getDB() == "$external" && - !txn->getClient()->session()->getX509PeerInfo().roles.empty(); +#ifdef MONGO_CONFIG_SSL + if (!txn || !txn->getClient() || !txn->getClient()->session()) { + return false; + } + + auto sslPeerInfo = txn->getClient()->session()->getX509PeerInfo(); + return sslPeerInfo.subjectName.toString() == userName.getUser() && + userName.getDB() == "$external" && !sslPeerInfo.roles.empty(); +#else + return false; +#endif } diff --git a/src/mongo/db/catalog/apply_ops.cpp b/src/mongo/db/catalog/apply_ops.cpp index a7306940d68..552376301cb 100644 --- a/src/mongo/db/catalog/apply_ops.cpp +++ b/src/mongo/db/catalog/apply_ops.cpp @@ -180,39 +180,32 @@ Status _applyOps(OperationContext* opCtx, NamespaceString requestNss{ns}; if (nss.isSystemDotIndexes()) { - BSONObj indexSpec; - NamespaceString indexNss; - std::tie(indexSpec, indexNss) = - repl::prepForApplyOpsIndexInsert(fieldO, opObj, requestNss); - if (!indexSpec["collation"]) { - // If the index spec does not include a collation, explicitly - // specify the simple collation, so the index does not inherit the - // collection default collation. - auto indexVersion = indexSpec["v"]; - // The index version is populated by prepForApplyOpsIndexInsert(). - invariant(indexVersion); - if (indexVersion.isNumber() && - (indexVersion.numberInt() >= - static_cast<int>(IndexDescriptor::IndexVersion::kV2))) { - BSONObjBuilder bob; - bob.append("collation", CollationSpec::kSimpleSpec); - bob.appendElements(indexSpec); - indexSpec = bob.obj(); - } - } - BSONObjBuilder command; - command.append("createIndexes", indexNss.coll()); - { - BSONArrayBuilder indexes(command.subarrayStart("indexes")); - indexes.append(indexSpec); - indexes.doneFast(); + invariant(opCtx->lockState()->isW()); + + // Disable background index builds when inserting into system.indexes. + // This causes the TempRelease to fail within applyOperation_inlock(), + // leading to the background index being built in the foreground. + // We do not want a background index build because we need to validate + // the index spec and also to avoid issues resulting from any metadata + // changes before the background thread starts. + Lock::GlobalWrite nestedGlobalWriteLock(opCtx->lockState()); + + OldClientContext ctx(opCtx, nss.ns()); + status = + repl::applyOperation_inlock(opCtx, ctx.db(), opObj, alwaysUpsert); + + // applyOperation_inlock() builds the index but does not notify the + // OpObserver. Previously, applyOps relied on the createIndexes command + // to perform this function. The value used for the 'forMigrate' + // argument is consistent with create_indexes.cpp. + if (status.isOK()) { + WriteUnitOfWork wuow(opCtx); + auto opObserver = getGlobalServiceContext()->getOpObserver(); + invariant(opObserver); + auto indexSpec = fieldO.embeddedObject(); + opObserver->onCreateIndex(opCtx, nss.ns(), indexSpec, false); + wuow.commit(); } - const BSONObj commandObj = command.done(); - - DBDirectClient client(opCtx); - BSONObj infoObj; - client.runCommand(nsToDatabase(ns), commandObj, infoObj); - status = getStatusFromCommandResult(infoObj); } else { AutoGetCollection autoColl(opCtx, nss, MODE_IX); if (!autoColl.getCollection() && !nss.isSystemDotIndexes()) { diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp index 71f74628f01..63b46d3c3a8 100644 --- a/src/mongo/db/catalog/capped_utils.cpp +++ b/src/mongo/db/catalog/capped_utils.cpp @@ -66,13 +66,13 @@ Status emptyCapped(OperationContext* txn, const NamespaceString& collectionName) } Database* db = autoDb.getDb(); - massert(13429, "no such database", db); + uassert(ErrorCodes::NamespaceNotFound, "no such database", db); Collection* collection = db->getCollection(collectionName); uassert(ErrorCodes::CommandNotSupportedOnView, str::stream() << "emptycapped not supported on view: " << collectionName.ns(), collection || !db->getViewCatalog()->lookup(txn, collectionName.ns())); - massert(28584, "no such collection", collection); + uassert(ErrorCodes::NamespaceNotFound, "no such collection", collection); if (collectionName.isSystem() && !collectionName.isSystemDotProfile()) { return Status(ErrorCodes::IllegalOperation, @@ -268,6 +268,7 @@ Status convertToCapped(OperationContext* txn, const NamespaceString& collectionN Status status = db->dropCollection(txn, longTmpName); if (!status.isOK()) return status; + wunit.commit(); } diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index dd6a1f506f1..38e5b12d234 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -64,6 +64,8 @@ public: virtual BSONObj getIndexSpec(OperationContext* txn, StringData idxName) const = 0; + virtual void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const = 0; + /** * Returns true if the index identified by 'indexName' is multikey, and returns false otherwise. * diff --git a/src/mongo/db/catalog/database.cpp b/src/mongo/db/catalog/database.cpp index 23fdfb84e95..4b27ff0e48f 100644 --- a/src/mongo/db/catalog/database.cpp +++ b/src/mongo/db/catalog/database.cpp @@ -564,15 +564,26 @@ Collection* Database::createCollection(OperationContext* txn, : ic->getDefaultIdIndexSpec(featureCompatibilityVersion))); } } - - if (nss.isSystem()) { - authindex::createSystemIndexes(txn, collection); - } } getGlobalServiceContext()->getOpObserver()->onCreateCollection( txn, nss, options, fullIdIndexSpec); + // It is necessary to create the system index *after* running the onCreateCollection so that + // the oplog timestamp for the index creation is after the oplog timestamp for the + // collection creation. This way both primary and any secondaries will see the index created + // after the collection is created. + if (createIdIndex && nss.isSystem()) { + // We only want to create the indexes here on the primary. On secondaries, they will + // be created by the normal oplog application process. + auto coordinator = repl::ReplicationCoordinator::get(txn); + const bool canAcceptWrites = + (coordinator->getReplicationMode() != repl::ReplicationCoordinator::modeReplSet) || + coordinator->canAcceptWritesForDatabase(nss.db()) || nss.isSystemDotProfile(); + if (canAcceptWrites) { + authindex::createSystemIndexes(txn, collection); + } + } return collection; } diff --git a/src/mongo/db/catalog/index_create.cpp b/src/mongo/db/catalog/index_create.cpp index 17a3d981d5d..6a323605521 100644 --- a/src/mongo/db/catalog/index_create.cpp +++ b/src/mongo/db/catalog/index_create.cpp @@ -65,6 +65,8 @@ using std::endl; MONGO_FP_DECLARE(crashAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuild); MONGO_FP_DECLARE(hangAfterStartingIndexBuildUnlocked); +MONGO_FP_DECLARE(hangBeforeIndexBuildOf); +MONGO_FP_DECLARE(hangAfterIndexBuildOf); std::atomic<std::int32_t> maxIndexBuildMemoryUsageMegabytes(500); // NOLINT @@ -282,6 +284,16 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(const std::vector<BSONObj return indexInfoObjs; } +void failPointHangDuringBuild(FailPoint* fp, StringData where, const BSONObj& doc) { + MONGO_FAIL_POINT_BLOCK(*fp, data) { + int i = doc.getIntField("i"); + if (data.getData()["i"].numberInt() == i) { + log() << "Hanging " << where << " index build of i=" << i; + MONGO_FAIL_POINT_PAUSE_WHILE_SET((*fp)); + } + } +} + Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsOut) { const char* curopMessage = _buildInBackground ? "Index Build (background)" : "Index Build"; const auto numRecords = _collection->numRecords(_txn); @@ -307,11 +319,20 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO PlanExecutor::ExecState state; int retries = 0; // non-zero when retrying our last document. while (retries || - (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc)))) { + (PlanExecutor::ADVANCED == (state = exec->getNextSnapshotted(&objToIndex, &loc))) || + MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { try { if (_allowInterruption) _txn->checkForInterrupt(); + if (!(retries || (PlanExecutor::ADVANCED == state))) { + // The only reason we are still in the loop is hangAfterStartingIndexBuild. + log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; + invariant(_allowInterruption); + sleepmillis(1000); + continue; + } + // Make sure we are working with the latest version of the document. if (objToIndex.snapshotId() != _txn->recoveryUnit()->getSnapshotId() && !_collection->findDoc(_txn, loc, &objToIndex)) { @@ -323,6 +344,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO // Done before insert so we can retry document if it WCEs. progress->setTotalWhileRunning(_collection->numRecords(_txn)); + failPointHangDuringBuild(&hangBeforeIndexBuildOf, "before", objToIndex.value()); + WriteUnitOfWork wunit(_txn); Status ret = insert(objToIndex.value(), loc); if (_buildInBackground) @@ -340,6 +363,8 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO if (_buildInBackground) exec->restoreState(); // Handles any WCEs internally. + failPointHangDuringBuild(&hangAfterIndexBuildOf, "after", objToIndex.value()); + // Go to the next document progress->hit(); n++; @@ -362,18 +387,6 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(std::set<RecordId>* dupsO WorkingSetCommon::toStatusString(objToIndex.value()), state == PlanExecutor::IS_EOF); - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - // Need the index build to hang before the progress meter is marked as finished so we can - // reliably check that the index build has actually started in js tests. - while (MONGO_FAIL_POINT(hangAfterStartingIndexBuild)) { - log() << "Hanging index build due to 'hangAfterStartingIndexBuild' failpoint"; - sleepmillis(1000); - } - - // Check for interrupt to allow for killop prior to index build completion. - _txn->checkForInterrupt(); - } - if (MONGO_FAIL_POINT(hangAfterStartingIndexBuildUnlocked)) { // Unlock before hanging so replication recognizes we've completed. Locker::LockSnapshot lockInfo; diff --git a/src/mongo/db/commands.cpp b/src/mongo/db/commands.cpp index 42bad714800..8f9baa02321 100644 --- a/src/mongo/db/commands.cpp +++ b/src/mongo/db/commands.cpp @@ -325,9 +325,9 @@ void Command::generateErrorResponse(OperationContext* txn, const BSONObj& metadata) { LOG(1) << "assertion while executing command '" << request.getCommandName() << "' " << "on database '" << request.getDatabase() << "' " - << "with arguments '" << command->getRedactedCopyForLogging(request.getCommandArgs()) - << "' " - << "and metadata '" << request.getMetadata() << "': " << exception.toString(); + << "with arguments '" + << redact(command->getRedactedCopyForLogging(request.getCommandArgs())) << "' " + << "and metadata '" << request.getMetadata() << "': " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, metadata); } @@ -337,7 +337,7 @@ void Command::generateErrorResponse(OperationContext* txn, const DBException& exception, const rpc::RequestInterface& request) { LOG(1) << "assertion while executing command '" << request.getCommandName() << "' " - << "on database '" << request.getDatabase() << "': " << exception.toString(); + << "on database '" << request.getDatabase() << "': " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, rpc::makeEmptyMetadata()); } @@ -345,7 +345,7 @@ void Command::generateErrorResponse(OperationContext* txn, void Command::generateErrorResponse(OperationContext* txn, rpc::ReplyBuilderInterface* replyBuilder, const DBException& exception) { - LOG(1) << "assertion while executing command: " << exception.toString(); + LOG(1) << "assertion while executing command: " << redact(exception.toString()); _generateErrorResponse(txn, replyBuilder, exception, rpc::makeEmptyMetadata()); } diff --git a/src/mongo/db/commands/authentication_commands.cpp b/src/mongo/db/commands/authentication_commands.cpp index 1eb8569b2be..68a483b24f9 100644 --- a/src/mongo/db/commands/authentication_commands.cpp +++ b/src/mongo/db/commands/authentication_commands.cpp @@ -166,14 +166,14 @@ bool CmdAuthenticate::run(OperationContext* txn, if (mechanism.empty()) { mechanism = "MONGODB-CR"; } - UserName user; - if (mechanism == "MONGODB-X509" && !cmdObj.hasField("user")) { - Client* client = txn->getClient(); - auto clientName = client->session()->getX509PeerInfo().subjectName; - user = UserName(clientName, dbname); - } else { - user = UserName(cmdObj.getStringField("user"), dbname); + + UserName user(cmdObj.getStringField("user"), dbname); +#ifdef MONGO_CONFIG_SSL + if (mechanism == "MONGODB-X509" && user.getUser().empty()) { + auto sslPeerInfo = txn->getClient()->session()->getX509PeerInfo(); + user = UserName(sslPeerInfo.subjectName.toString(), dbname); } +#endif uassert(ErrorCodes::AuthenticationFailed, "No user name provided", !user.getUser().empty()); if (Command::testCommandsEnabled && user.getDB() == "admin" && @@ -331,7 +331,7 @@ Status CmdAuthenticate::_authenticateX509(OperationContext* txn, if (!getSSLManager()->getSSLConfiguration().hasCA) { return Status(ErrorCodes::AuthenticationFailed, "Unable to verify x.509 certificate, as no CA has been provided."); - } else if (user.getUser() != clientName) { + } else if (user.getUser() != clientName.toString()) { return Status(ErrorCodes::AuthenticationFailed, "There is no x.509 client certificate matching the user."); } else { diff --git a/src/mongo/db/commands/list_indexes.cpp b/src/mongo/db/commands/list_indexes.cpp index 1a581074d55..5e8e47a7dfd 100644 --- a/src/mongo/db/commands/list_indexes.cpp +++ b/src/mongo/db/commands/list_indexes.cpp @@ -152,7 +152,7 @@ public: vector<string> indexNames; MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { indexNames.clear(); - cce->getAllIndexes(txn, &indexNames); + cce->getReadyIndexes(txn, &indexNames); } MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "listIndexes", ns.ns()); diff --git a/src/mongo/db/commands/parameters.cpp b/src/mongo/db/commands/parameters.cpp index a3c252d050a..fe8f157974b 100644 --- a/src/mongo/db/commands/parameters.cpp +++ b/src/mongo/db/commands/parameters.cpp @@ -577,7 +577,7 @@ public: << saslCommandUserDBFieldName << "$external" << saslCommandUserFieldName - << getSSLManager()->getSSLConfiguration().clientSubjectName)); + << getSSLManager()->getSSLConfiguration().clientSubjectName.toString())); #endif } else if (str == "x509" && oldMode == ServerGlobalParams::ClusterAuthMode_sendX509) { serverGlobalParams.clusterAuthMode.store(ServerGlobalParams::ClusterAuthMode_x509); diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp index 3b5f71f23ce..5e1a14663dc 100644 --- a/src/mongo/db/curop.cpp +++ b/src/mongo/db/curop.cpp @@ -459,7 +459,7 @@ string OpDebug::report(Client* client, } if (!curop.getPlanSummary().empty()) { - s << " planSummary: " << redact(curop.getPlanSummary().toString()); + s << " planSummary: " << curop.getPlanSummary().toString(); } if (!updateobj.isEmpty()) { diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index a99ce97d57c..05fc368e3b6 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -647,7 +647,7 @@ ExitCode _initAndListen(int listenPort) { logMongodStartupWarnings(storageGlobalParams, serverGlobalParams); -#if MONGO_CONFIG_SSL +#ifdef MONGO_CONFIG_SSL if (sslGlobalParams.sslAllowInvalidCertificates && ((serverGlobalParams.clusterAuthMode.load() == ServerGlobalParams::ClusterAuthMode_x509) || sequenceContains(saslGlobalParams.authenticationMechanisms, "MONGODB-X509"))) { @@ -724,6 +724,9 @@ ExitCode _initAndListen(int listenPort) { log() << redact(status); if (status.code() == ErrorCodes::AuthSchemaIncompatible) { exitCleanly(EXIT_NEED_UPGRADE); + } else if (status == ErrorCodes::NotMaster) { + // Try creating the indexes if we become master. If we do not become master, + // the master will create the indexes and we will replicate them. } else { quickExit(EXIT_FAILURE); } diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index 96eb011018d..a372d42824b 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -60,6 +60,7 @@ #include "mongo/db/s/collection_metadata.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/sharding_state.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/data_protector.h" #include "mongo/db/storage/encryption_hooks.h" @@ -266,6 +267,10 @@ BSONObj Helpers::inferKeyPattern(const BSONObj& o) { return kpBuilder.obj(); } +// After completing internalQueryExecYieldIterations document deletions, the time in millis to wait +// before continuing deletions. +MONGO_EXPORT_SERVER_PARAMETER(rangeDeleterBatchDelayMS, int, 20); + long long Helpers::removeRange(OperationContext* txn, const KeyRange& range, BoundInclusion boundInclusion, @@ -323,7 +328,7 @@ long long Helpers::removeRange(OperationContext* txn, MONGO_LOG_COMPONENT(1, LogComponent::kSharding) - << "begin removal of " << min << " to " << max << " in " << ns + << "begin removal of " << redact(min) << " to " << redact(max) << " in " << ns << " with write concern: " << writeConcern.toBSON() << endl; long long numDeleted = 0; @@ -331,20 +336,25 @@ long long Helpers::removeRange(OperationContext* txn, replWaitDuration = Milliseconds::zero(); while (1) { + long long numDeletedPreviously = numDeleted; + long long iterationsBetweenSleeps = internalQueryExecYieldIterations.load(); + long long batchSize = writeConcern.shouldWaitForOtherNodes() ? 1 : iterationsBetweenSleeps; + // Scoping for write lock. { ScopedTransaction scopedXact(txn, MODE_IX); AutoGetCollection ctx(txn, NamespaceString(ns), MODE_IX, MODE_IX); Collection* collection = ctx.getCollection(); - if (!collection) + if (!collection) { break; + } IndexDescriptor* desc = collection->getIndexCatalog()->findIndexByName(txn, indexName); if (!desc) { warning(LogComponent::kSharding) << "shard key index '" << indexName << "' on '" << ns << "' was dropped"; - return -1; + break; } unique_ptr<PlanExecutor> exec( @@ -357,99 +367,153 @@ long long Helpers::removeRange(OperationContext* txn, PlanExecutor::YIELD_MANUAL, InternalPlanner::FORWARD, InternalPlanner::IXSCAN_FETCH)); - exec->setYieldPolicy(PlanExecutor::YIELD_AUTO, collection); - - RecordId rloc; - BSONObj obj; - PlanExecutor::ExecState state; - // This may yield so we cannot touch nsd after this. - state = exec->getNext(&obj, &rloc); - if (PlanExecutor::IS_EOF == state) { - break; - } - if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { - warning(LogComponent::kSharding) - << PlanExecutor::statestr(state) << " - cursor error while trying to delete " - << min << " to " << max << " in " << ns << ": " - << WorkingSetCommon::toStatusString(obj) - << ", stats: " << Explain::getWinningPlanStats(exec.get()) << endl; - break; - } - - verify(PlanExecutor::ADVANCED == state); - - if (onlyRemoveOrphanedDocs) { - // Do a final check in the write lock to make absolutely sure that our - // collection hasn't been modified in a way that invalidates our migration - // cleanup. + bool errorOccurred = false; - // We should never be able to turn off the sharding state once enabled, but - // in the future we might want to. - verify(ShardingState::get(txn)->enabled()); + while (numDeleted - numDeletedPreviously < batchSize) { - bool docIsOrphan; - - // In write lock, so will be the most up-to-date version - auto metadataNow = CollectionShardingState::get(txn, ns)->getMetadata(); - if (metadataNow) { - ShardKeyPattern kp(metadataNow->getKeyPattern()); - BSONObj key = kp.extractShardKeyFromDoc(obj); - docIsOrphan = - !metadataNow->keyBelongsToMe(key) && !metadataNow->keyIsPending(key); - } else { - docIsOrphan = false; + RecordId rloc; + BSONObj obj; + PlanExecutor::ExecState state; + // This may yield so we cannot touch nsd after this. + state = exec->getNext(&obj, &rloc); + if (PlanExecutor::IS_EOF == state) { + errorOccurred = true; + break; } - if (!docIsOrphan) { + if (PlanExecutor::FAILURE == state || PlanExecutor::DEAD == state) { warning(LogComponent::kSharding) - << "aborting migration cleanup for chunk " << min << " to " << max - << (metadataNow ? (string) " at document " + obj.toString() : "") - << ", collection " << ns << " has changed " << endl; + << PlanExecutor::statestr(state) + << " - cursor error while trying to delete " << redact(min) << " to " + << redact(max) << " in " << ns << ": " + << redact(WorkingSetCommon::toStatusString(obj)) + << ", stats: " << Explain::getWinningPlanStats(exec.get()) << endl; + errorOccurred = true; break; } - } - MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { - WriteUnitOfWork wuow(txn); - NamespaceString nss(ns); - if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(nss)) { - warning() << "stepped down from primary while deleting chunk; " - << "orphaning data in " << ns << " in range [" << redact(min) << ", " - << redact(max) << ")"; - return numDeleted; + verify(PlanExecutor::ADVANCED == state); + + if (onlyRemoveOrphanedDocs) { + // Do a final check in the write lock to make absolutely sure that our + // collection hasn't been modified in a way that invalidates our migration + // cleanup. + + // We should never be able to turn off the sharding state once enabled, but + // in the future we might want to. + verify(ShardingState::get(txn)->enabled()); + + bool docIsOrphan; + + // In write lock, so will be the most up-to-date version + auto metadataNow = CollectionShardingState::get(txn, ns)->getMetadata(); + if (metadataNow) { + ShardKeyPattern kp(metadataNow->getKeyPattern()); + BSONObj key = kp.extractShardKeyFromDoc(obj); + docIsOrphan = + !metadataNow->keyBelongsToMe(key) && !metadataNow->keyIsPending(key); + } else { + docIsOrphan = false; + } + + if (!docIsOrphan) { + warning(LogComponent::kSharding) + << "aborting migration cleanup for chunk " << redact(min) << " to " + << redact(max) + << (metadataNow ? (string) " at document " + redact(obj.toString()) + : "") + << ", collection " << ns << " has changed " << endl; + // No chance of success with a new plan, so fully abort. + errorOccurred = true; + break; + } } - if (callback) - callback->goingToDelete(obj); + exec->saveState(); + + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + WriteUnitOfWork wuow(txn); + NamespaceString nss(ns); + if (!repl::getGlobalReplicationCoordinator()->canAcceptWritesFor(nss)) { + warning() << "stepped down from primary while deleting chunk; " + << "orphaning data in " << ns << " in range [" << redact(min) + << ", " << redact(max) << ")"; + // No chance of success with a new plan, so fully abort. + errorOccurred = true; + break; + } + + if (callback) + callback->goingToDelete(obj); + + OpDebug* const nullOpDebug = nullptr; + collection->deleteDocument(txn, rloc, nullOpDebug, fromMigrate); + wuow.commit(); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "delete range", ns); + + if (!exec->restoreState()) { + MONGO_LOG_COMPONENT(1, LogComponent::kSharding) + << "unable to restore cursor state while trying to delete " << redact(min) + << " to " << redact(max) << " in " << ns + << ", stats: " << Explain::getWinningPlanStats(exec.get()) + << ", replanning"; + // Try again with a new plan. + break; + } - OpDebug* const nullOpDebug = nullptr; - collection->deleteDocument(txn, rloc, nullOpDebug, fromMigrate); - wuow.commit(); + numDeleted++; } - MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "delete range", ns); - numDeleted++; - } + if (errorOccurred) { + break; + } - // TODO remove once the yielding below that references this timer has been removed - Timer secondaryThrottleTime; - - if (writeConcern.shouldWaitForOtherNodes() && numDeleted > 0) { - repl::ReplicationCoordinator::StatusAndDuration replStatus = - repl::getGlobalReplicationCoordinator()->awaitReplication( - txn, - repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), - writeConcern); - if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || - replStatus.status.code() == ErrorCodes::WriteConcernFailed) { - warning(LogComponent::kSharding) << "replication to secondaries for removeRange at " - "least 60 seconds behind"; - } else { - uassertStatusOK(replStatus.status); + } // End scope for write lock. + + if (numDeleted > 0) { + if (writeConcern.shouldWaitForOtherNodes()) { + repl::ReplicationCoordinator::StatusAndDuration replStatus = + repl::getGlobalReplicationCoordinator()->awaitReplication( + txn, + repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), + writeConcern); + if (replStatus.status.code() == ErrorCodes::ExceededTimeLimit || + replStatus.status.code() == ErrorCodes::WriteConcernFailed) { + warning(LogComponent::kSharding) + << "replication to secondaries for removeRange at " + "least 60 seconds behind"; + } else { + uassertStatusOK(replStatus.status); + } + replWaitDuration += replStatus.duration; + } + + // The `rangeDeleterBatchDelayMS` parameter is defined as a delay every + // `internalQueryExecYieldIterations` (aka `batchSize`) document deletions. + // + // In v3.6+, this applies regardless of waiting for replication (aka + // _secondaryThrottle), because in those versions the replication waits and query + // replanning also happen after each batch of `batchSize` deletions. + // + // However, here in v3.4, when _secondaryThrottle is on it's necessary to preserve the + // semantics of waiting for replication after each document deletion. But it's also + // necessary for `rangeDeleterBatchDelayMS` - which is the only wait when + // _secondaryThrottle is off - to behave as it does in other versions (despite query + // planning still occuring every document deletion in v3.4 when _secondaryThrottle is + // on). + // + // Therefore, we sleep for `rangeDeleterBatchDelayMS` here (every `batchSize` + // iterations), even if we have also waited for replication above. This approach also + // makes the RangeDeleter behavior more consistent when enabling/disabling + // _secondaryThrottle. + if (batchSize != 1 || numDeleted % iterationsBetweenSleeps == 0) { + sleepmillis(rangeDeleterBatchDelayMS.load()); } - replWaitDuration += replStatus.duration; } + + // Loop back to get a new plan and go again. } if (writeConcern.shouldWaitForOtherNodes()) @@ -457,8 +521,8 @@ long long Helpers::removeRange(OperationContext* txn, << "Helpers::removeRangeUnlocked time spent waiting for replication: " << durationCount<Milliseconds>(replWaitDuration) << "ms" << endl; - MONGO_LOG_COMPONENT(1, LogComponent::kSharding) << "end removal of " << min << " to " << max - << " in " << ns << " (took " + MONGO_LOG_COMPONENT(1, LogComponent::kSharding) << "end removal of " << redact(min) << " to " + << redact(max) << " in " << ns << " (took " << rangeRemoveTimer.millis() << "ms)" << endl; return numDeleted; diff --git a/src/mongo/db/exec/cached_plan.cpp b/src/mongo/db/exec/cached_plan.cpp index f1867423b78..8c729927e71 100644 --- a/src/mongo/db/exec/cached_plan.cpp +++ b/src/mongo/db/exec/cached_plan.cpp @@ -140,7 +140,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Execution of cached plan failed, falling back to replan." << " query: " << redact(_canonicalQuery->toStringShort()) - << " planSummary: " << redact(Explain::getPlanSummary(child().get())) + << " planSummary: " << Explain::getPlanSummary(child().get()) << " status: " << redact(statusObj); const bool shouldCache = false; @@ -151,7 +151,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Execution of cached plan failed: PlanStage died" << ", query: " << redact(_canonicalQuery->toStringShort()) - << " planSummary: " << redact(Explain::getPlanSummary(child().get())) + << " planSummary: " << Explain::getPlanSummary(child().get()) << " status: " << redact(statusObj); return WorkingSetCommon::getMemberObjectStatus(statusObj); @@ -166,7 +166,7 @@ Status CachedPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { << " works, but was originally cached with only " << _decisionWorks << " works. Evicting cache entry and replanning query: " << redact(_canonicalQuery->toStringShort()) - << " plan summary before replan: " << redact(Explain::getPlanSummary(child().get())); + << " plan summary before replan: " << Explain::getPlanSummary(child().get()); const bool shouldCache = true; return replan(yieldPolicy, shouldCache); @@ -241,7 +241,7 @@ Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) { LOG(1) << "Replanning of query resulted in single query solution, which will not be cached. " << redact(_canonicalQuery->toStringShort()) - << " plan summary after replan: " << redact(Explain::getPlanSummary(child().get())) + << " plan summary after replan: " << Explain::getPlanSummary(child().get()) << " previous cache entry evicted: " << (shouldCache ? "yes" : "no"); return Status::OK(); } @@ -274,7 +274,7 @@ Status CachedPlanStage::replan(PlanYieldPolicy* yieldPolicy, bool shouldCache) { } LOG(1) << "Replanning " << redact(_canonicalQuery->toStringShort()) - << " resulted in plan with summary: " << redact(Explain::getPlanSummary(child().get())) + << " resulted in plan with summary: " << Explain::getPlanSummary(child().get()) << ", which " << (shouldCache ? "has" : "has not") << " been written to the cache"; return Status::OK(); } diff --git a/src/mongo/db/exec/multi_plan.cpp b/src/mongo/db/exec/multi_plan.cpp index 28c14dbcba3..10b36d49b8d 100644 --- a/src/mongo/db/exec/multi_plan.cpp +++ b/src/mongo/db/exec/multi_plan.cpp @@ -239,7 +239,7 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { const auto& bestSolution = bestCandidate.solution; LOG(5) << "Winning solution:\n" << redact(bestSolution->toString()); - LOG(2) << "Winning plan: " << redact(Explain::getPlanSummary(bestCandidate.root)); + LOG(2) << "Winning plan: " << Explain::getPlanSummary(bestCandidate.root); _backupPlanIdx = kNoSuchPlan; if (bestSolution->hasBlockingStage && (0 == alreadyProduced.size())) { @@ -276,10 +276,10 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { LOG(1) << "Winning plan tied with runner-up. Not caching." << " ns: " << _collection->ns() << " " << redact(_query->toStringShort()) - << " winner score: " << ranking->scores[0] << " winner summary: " - << redact(Explain::getPlanSummary(_candidates[winnerIdx].root)) + << " winner score: " << ranking->scores[0] + << " winner summary: " << Explain::getPlanSummary(_candidates[winnerIdx].root) << " runner-up score: " << ranking->scores[1] << " runner-up summary: " - << redact(Explain::getPlanSummary(_candidates[runnerUpIdx].root)); + << Explain::getPlanSummary(_candidates[runnerUpIdx].root); } if (alreadyProduced.empty()) { @@ -290,8 +290,8 @@ Status MultiPlanStage::pickBestPlan(PlanYieldPolicy* yieldPolicy) { size_t winnerIdx = ranking->candidateOrder[0]; LOG(1) << "Winning plan had zero results. Not caching." << " ns: " << _collection->ns() << " " << redact(_query->toStringShort()) - << " winner score: " << ranking->scores[0] << " winner summary: " - << redact(Explain::getPlanSummary(_candidates[winnerIdx].root)); + << " winner score: " << ranking->scores[0] + << " winner summary: " << Explain::getPlanSummary(_candidates[winnerIdx].root); } } diff --git a/src/mongo/db/ftdc/compressor.cpp b/src/mongo/db/ftdc/compressor.cpp index ecf9c7ece6c..5be24ceb1f7 100644 --- a/src/mongo/db/ftdc/compressor.cpp +++ b/src/mongo/db/ftdc/compressor.cpp @@ -45,7 +45,12 @@ using std::swap; StatusWith<boost::optional<std::tuple<ConstDataRange, FTDCCompressor::CompressorState, Date_t>>> FTDCCompressor::addSample(const BSONObj& sample, Date_t date) { if (_referenceDoc.isEmpty()) { - FTDCBSONUtil::extractMetricsFromDocument(sample, sample, &_metrics); + auto swMatchesReference = + FTDCBSONUtil::extractMetricsFromDocument(sample, sample, &_metrics); + if (!swMatchesReference.isOK()) { + return swMatchesReference.getStatus(); + } + _reset(sample, date); return {boost::none}; } diff --git a/src/mongo/db/ftdc/compressor_test.cpp b/src/mongo/db/ftdc/compressor_test.cpp index 0c01bd58040..10914eba102 100644 --- a/src/mongo/db/ftdc/compressor_test.cpp +++ b/src/mongo/db/ftdc/compressor_test.cpp @@ -48,17 +48,17 @@ namespace mongo { ASSERT_TRUE(st.isOK()); \ ASSERT_FALSE(st.getValue().is_initialized()); -#define ASSERT_SCHEMA_CHANGED(st) \ - ASSERT_TRUE(st.isOK()); \ - ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ - FTDCCompressor::CompressorState::kSchemaChanged); \ - ASSERT_TRUE(st.getValue().is_initialized()); - -#define ASSERT_FULL(st) \ - ASSERT_TRUE(st.isOK()); \ - ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ - FTDCCompressor::CompressorState::kCompressorFull); \ - ASSERT_TRUE(st.getValue().is_initialized()); +#define ASSERT_SCHEMA_CHANGED(st) \ + ASSERT_TRUE(st.isOK()); \ + ASSERT_TRUE(st.getValue().is_initialized()); \ + ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ + FTDCCompressor::CompressorState::kSchemaChanged); + +#define ASSERT_FULL(st) \ + ASSERT_TRUE(st.isOK()); \ + ASSERT_TRUE(st.getValue().is_initialized()); \ + ASSERT_TRUE(std::get<1>(st.getValue().get()) == \ + FTDCCompressor::CompressorState::kCompressorFull); // Sanity check TEST(FTDCCompressor, TestBasic) { @@ -125,7 +125,8 @@ TEST(FTDCCompressor, TestStrings) { */ class TestTie { public: - TestTie() : _compressor(&_config) {} + TestTie(FTDCValidationMode mode = FTDCValidationMode::kStrict) + : _compressor(&_config), _mode(mode) {} ~TestTie() { validate(boost::none); @@ -169,7 +170,7 @@ public: list = sw.getValue(); } - ValidateDocumentList(list, _docs); + ValidateDocumentList(list, _docs, _mode); } private: @@ -177,6 +178,7 @@ private: FTDCConfig _config; FTDCCompressor _compressor; FTDCDecompressor _decompressor; + FTDCValidationMode _mode; }; // Test various schema changes @@ -340,6 +342,114 @@ TEST(FTDCCompressor, TestSchemaChanges) { ASSERT_SCHEMA_CHANGED(st); } +// Test various schema changes with strings +TEST(FTDCCompressorTest, TestStringSchemaChanges) { + TestTie c(FTDCValidationMode::kWeak); + + auto st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 42)); + ASSERT_HAS_SPACE(st); + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 45)); + ASSERT_HAS_SPACE(st); + + // Add string field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "int1" + << 47)); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "int2" + << 48)); + ASSERT_SCHEMA_CHANGED(st); + + // Remove string field + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 49)); + ASSERT_HAS_SPACE(st); + + + // Add string field as last element + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 50 + << "str3" + << "bar")); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 51 + << "str3" + << "bar")); + ASSERT_SCHEMA_CHANGED(st); + + // Remove string field as last element + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 52)); + ASSERT_HAS_SPACE(st); + + + // Add 2 string fields + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "str3" + << "foo" + << "int1" + << 53)); + ASSERT_HAS_SPACE(st); + + // Reset schema by renaming a int field + st = c.addSample(BSON("str1" + << "joe" + << "str2" + << "smith" + << "str3" + << "foo" + << "int2" + << 54)); + ASSERT_SCHEMA_CHANGED(st); + + // Remove 2 string fields + st = c.addSample(BSON("str1" + << "joe" + << "int2" + << 55)); + ASSERT_HAS_SPACE(st); + + // Change string to number + st = c.addSample(BSON("str1" << 12 << "int1" << 56)); + ASSERT_SCHEMA_CHANGED(st); + + // Change number to string + st = c.addSample(BSON("str1" + << "joe" + << "int1" + << 67)); + ASSERT_SCHEMA_CHANGED(st); +} + // Ensure changing between the various number formats is considered compatible TEST(FTDCCompressor, TestNumbersCompat) { TestTie c; diff --git a/src/mongo/db/ftdc/controller_test.cpp b/src/mongo/db/ftdc/controller_test.cpp index c75a3fe44d6..94b028845d4 100644 --- a/src/mongo/db/ftdc/controller_test.cpp +++ b/src/mongo/db/ftdc/controller_test.cpp @@ -202,7 +202,7 @@ TEST(FTDCControllerTest, TestFull) { auto alog = files[0]; - ValidateDocumentList(alog, allDocs); + ValidateDocumentList(alog, allDocs, FTDCValidationMode::kStrict); } // Test we can start and stop the controller in quick succession, make sure it succeeds without @@ -274,7 +274,7 @@ TEST(FTDCControllerTest, TestStartAsDisabled) { auto alog = files[0]; - ValidateDocumentList(alog, allDocs); + ValidateDocumentList(alog, allDocs, FTDCValidationMode::kStrict); } } // namespace mongo diff --git a/src/mongo/db/ftdc/file_manager_test.cpp b/src/mongo/db/ftdc/file_manager_test.cpp index 6c2e5c220a6..8ac115df584 100644 --- a/src/mongo/db/ftdc/file_manager_test.cpp +++ b/src/mongo/db/ftdc/file_manager_test.cpp @@ -378,11 +378,11 @@ TEST(FTDCFileManagerTest, TestNormalCrashInterim) { // Validate old file std::vector<BSONObj> docs1 = {mdoc1, sdoc1, sdoc1}; - ValidateDocumentList(files[0], docs1); + ValidateDocumentList(files[0], docs1, FTDCValidationMode::kStrict); // Validate new file std::vector<BSONObj> docs2 = {sdoc2, sdoc2, sdoc2, sdoc2}; - ValidateDocumentList(files[1], docs2); + ValidateDocumentList(files[1], docs2, FTDCValidationMode::kStrict); } } // namespace mongo diff --git a/src/mongo/db/ftdc/file_writer_test.cpp b/src/mongo/db/ftdc/file_writer_test.cpp index 138d7c850f6..a182d52fdfe 100644 --- a/src/mongo/db/ftdc/file_writer_test.cpp +++ b/src/mongo/db/ftdc/file_writer_test.cpp @@ -196,7 +196,7 @@ private: _writer.close(); - ValidateDocumentList(_path, _docs); + ValidateDocumentList(_path, _docs, FTDCValidationMode::kStrict); } private: diff --git a/src/mongo/db/ftdc/ftdc_test.cpp b/src/mongo/db/ftdc/ftdc_test.cpp index 7e4020b3979..49be73bb8cc 100644 --- a/src/mongo/db/ftdc/ftdc_test.cpp +++ b/src/mongo/db/ftdc/ftdc_test.cpp @@ -48,7 +48,23 @@ namespace mongo { -void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BSONObj>& docs) { +namespace { + +BSONObj filteredFTDCCopy(const BSONObj& obj) { + BSONObjBuilder builder; + for (const auto& f : obj) { + if (FTDCBSONUtil::isFTDCType(f.type())) { + builder.append(f); + } + } + return builder.obj(); +} +} // namespace + + +void ValidateDocumentList(const boost::filesystem::path& p, + const std::vector<BSONObj>& docs, + FTDCValidationMode mode) { FTDCFileReader reader; ASSERT_OK(reader.open(p)); @@ -62,20 +78,32 @@ void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BS ASSERT_OK(sw); - ValidateDocumentList(list, docs); + ValidateDocumentList(list, docs, mode); } -void ValidateDocumentList(const std::vector<BSONObj>& docs1, const std::vector<BSONObj>& docs2) { +void ValidateDocumentList(const std::vector<BSONObj>& docs1, + const std::vector<BSONObj>& docs2, + FTDCValidationMode mode) { ASSERT_EQUALS(docs1.size(), docs2.size()); auto ai = docs1.begin(); auto bi = docs2.begin(); while (ai != docs1.end() && bi != docs2.end()) { - if (SimpleBSONObjComparator::kInstance.evaluate(*ai != *bi)) { - std::cout << *ai << " vs " << *bi << std::endl; - ASSERT_BSONOBJ_EQ(*ai, *bi); + if (mode == FTDCValidationMode::kStrict) { + if (SimpleBSONObjComparator::kInstance.evaluate(*ai != *bi)) { + std::cout << *ai << " vs " << *bi << std::endl; + ASSERT_BSONOBJ_EQ(*ai, *bi); + } + } else { + BSONObj left = filteredFTDCCopy(*ai); + BSONObj right = filteredFTDCCopy(*bi); + if (SimpleBSONObjComparator::kInstance.evaluate(left != right)) { + std::cout << left << " vs " << right << std::endl; + ASSERT_BSONOBJ_EQ(left, right); + } } + ++ai; ++bi; } diff --git a/src/mongo/db/ftdc/ftdc_test.h b/src/mongo/db/ftdc/ftdc_test.h index afbca103b4f..d3b7b52d647 100644 --- a/src/mongo/db/ftdc/ftdc_test.h +++ b/src/mongo/db/ftdc/ftdc_test.h @@ -34,18 +34,38 @@ namespace mongo { /** + * Validation mode for tests, strict by default + */ +enum class FTDCValidationMode { + /** + * Compare BSONObjs exactly. + */ + kStrict, + + /** + * Compare BSONObjs by only comparing types FTDC compares about. FTDC ignores somes changes in + * the shapes of documents and therefore no longer reconstructs the shapes of documents exactly. + */ + kWeak, +}; + +/** * Validate the documents in a file match the specified vector. * * Unit Test ASSERTs if there is mismatch. */ -void ValidateDocumentList(const boost::filesystem::path& p, const std::vector<BSONObj>& docs); +void ValidateDocumentList(const boost::filesystem::path& p, + const std::vector<BSONObj>& docs, + FTDCValidationMode mode); /** * Validate that two lists of documents are equal. * * Unit Test ASSERTs if there is mismatch. */ -void ValidateDocumentList(const std::vector<BSONObj>& docs1, const std::vector<BSONObj>& docs2); +void ValidateDocumentList(const std::vector<BSONObj>& docs1, + const std::vector<BSONObj>& docs2, + FTDCValidationMode mode); /** * Delete a file if it exists. diff --git a/src/mongo/db/ftdc/util.cpp b/src/mongo/db/ftdc/util.cpp index 243ba90b666..ecfc4db5f37 100644 --- a/src/mongo/db/ftdc/util.cpp +++ b/src/mongo/db/ftdc/util.cpp @@ -123,6 +123,47 @@ namespace FTDCBSONUtil { namespace { +/** + * Iterate a BSONObj but only return fields that have types that FTDC cares about. + */ +class FTDCBSONObjIterator { +public: + FTDCBSONObjIterator(const BSONObj& obj) : _iterator(obj) { + advance(); + } + + bool more() { + return !_current.eoo(); + } + + BSONElement next() { + auto ret = _current; + advance(); + return ret; + } + +private: + /** + * Find the next element that is a valid FTDC type. + */ + void advance() { + _current = BSONElement(); + + while (_iterator.more()) { + + auto elem = _iterator.next(); + if (isFTDCType(elem.type())) { + _current = elem; + break; + } + } + } + +private: + BSONObjIterator _iterator; + BSONElement _current; +}; + StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, const BSONObj& currentDoc, std::vector<std::uint64_t>* metrics, @@ -132,15 +173,14 @@ StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, return {ErrorCodes::BadValue, "Recursion limit reached."}; } - BSONObjIterator itCurrent(currentDoc); - BSONObjIterator itReference(referenceDoc); + FTDCBSONObjIterator itCurrent(currentDoc); + FTDCBSONObjIterator itReference(referenceDoc); while (itCurrent.more()) { // Schema mismatch if current document is longer than reference document if (matches && !itReference.more()) { LOG(4) << "full-time diagnostic data capture schema change: currrent document is " - "longer than " - "reference document"; + "longer than reference document"; matches = false; } @@ -230,6 +270,24 @@ StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, } // namespace +bool isFTDCType(BSONType type) { + switch (type) { + case NumberDouble: + case NumberInt: + case NumberLong: + case NumberDecimal: + case Bool: + case Date: + case bsonTimestamp: + case Object: + case Array: + return true; + + default: + return false; + } +} + StatusWith<bool> extractMetricsFromDocument(const BSONObj& referenceDoc, const BSONObj& currentDoc, std::vector<std::uint64_t>* metrics) { diff --git a/src/mongo/db/ftdc/util.h b/src/mongo/db/ftdc/util.h index 4816c534f96..04d453139ce 100644 --- a/src/mongo/db/ftdc/util.h +++ b/src/mongo/db/ftdc/util.h @@ -168,6 +168,11 @@ StatusWith<BSONObj> getBSONDocumentFromMetadataDoc(const BSONObj& obj); */ StatusWith<std::vector<BSONObj>> getMetricsFromMetricDoc(const BSONObj& obj, FTDCDecompressor* decompressor); + +/** + * Is this a type that FTDC find's interesting? I.e. is this a numeric or container type? + */ +bool isFTDCType(BSONType type); } // namespace FTDCBSONUtil diff --git a/src/mongo/db/initialize_server_global_state.cpp b/src/mongo/db/initialize_server_global_state.cpp index f552a238109..4993fbf7bd8 100644 --- a/src/mongo/db/initialize_server_global_state.cpp +++ b/src/mongo/db/initialize_server_global_state.cpp @@ -373,7 +373,7 @@ bool initializeServerGlobalState() { << saslCommandUserDBFieldName << "$external" << saslCommandUserFieldName - << getSSLManager()->getSSLConfiguration().clientSubjectName)); + << getSSLManager()->getSSLConfiguration().clientSubjectName.toString())); } #endif return true; diff --git a/src/mongo/db/ops/write_ops_exec.cpp b/src/mongo/db/ops/write_ops_exec.cpp index 6ea4d88293a..ad307b508bf 100644 --- a/src/mongo/db/ops/write_ops_exec.cpp +++ b/src/mongo/db/ops/write_ops_exec.cpp @@ -52,6 +52,7 @@ #include "mongo/db/ops/parsed_update.h" #include "mongo/db/ops/update_lifecycle_impl.h" #include "mongo/db/ops/update_request.h" +#include "mongo/db/ops/write_ops.h" #include "mongo/db/ops/write_ops_exec.h" #include "mongo/db/query/get_executor.h" #include "mongo/db/query/plan_summary_stats.h" @@ -301,12 +302,13 @@ static WriteResult performCreateIndexes(OperationContext* txn, const InsertOp& w static void insertDocuments(OperationContext* txn, Collection* collection, std::vector<BSONObj>::const_iterator begin, - std::vector<BSONObj>::const_iterator end) { + std::vector<BSONObj>::const_iterator end, + bool fromMigrate) { // Intentionally not using a WRITE_CONFLICT_RETRY_LOOP. That is handled by the caller so it can // react to oversized batches. WriteUnitOfWork wuow(txn); uassertStatusOK(collection->insertDocuments( - txn, begin, end, &CurOp::get(txn)->debug(), /*enforceQuota*/ true)); + txn, begin, end, &CurOp::get(txn)->debug(), /*enforceQuota*/ true, fromMigrate)); wuow.commit(); } @@ -317,7 +319,8 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, const InsertOp& wholeOp, const std::vector<BSONObj>& batch, LastOpFixer* lastOpFixer, - WriteResult* out) { + WriteResult* out, + bool fromMigrate) { if (batch.empty()) return true; @@ -350,7 +353,8 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, // First try doing it all together. If all goes well, this is all we need to do. // See Collection::_insertDocuments for why we do all capped inserts one-at-a-time. lastOpFixer->startingOp(); - insertDocuments(txn, collection->getCollection(), batch.begin(), batch.end()); + insertDocuments( + txn, collection->getCollection(), batch.begin(), batch.end(), fromMigrate); lastOpFixer->finishedOpSuccessfully(); globalOpCounters.gotInserts(batch.size()); std::fill_n( @@ -374,7 +378,7 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, if (!collection) acquireCollection(); lastOpFixer->startingOp(); - insertDocuments(txn, collection->getCollection(), it, it + 1); + insertDocuments(txn, collection->getCollection(), it, it + 1, fromMigrate); lastOpFixer->finishedOpSuccessfully(); out->results.emplace_back(WriteResult::SingleResult{1}); curOp.debug().ninserted++; @@ -396,7 +400,7 @@ static bool insertBatchAndHandleErrors(OperationContext* txn, return true; } -WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp) { +WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp, bool fromMigrate) { invariant(!txn->lockState()->inAWriteUnitOfWork()); // Does own retries. auto& curOp = *CurOp::get(txn); ON_BLOCK_EXIT([&] { @@ -453,7 +457,8 @@ WriteResult performInserts(OperationContext* txn, const InsertOp& wholeOp) { continue; // Add more to batch before inserting. } - bool canContinue = insertBatchAndHandleErrors(txn, wholeOp, batch, &lastOpFixer, &out); + bool canContinue = + insertBatchAndHandleErrors(txn, wholeOp, batch, &lastOpFixer, &out, fromMigrate); batch.clear(); // We won't need the current batch any more. bytesInBatch = 0; diff --git a/src/mongo/db/ops/write_ops_exec.h b/src/mongo/db/ops/write_ops_exec.h index 49d3d2e0cf1..362bf5a4782 100644 --- a/src/mongo/db/ops/write_ops_exec.h +++ b/src/mongo/db/ops/write_ops_exec.h @@ -75,8 +75,10 @@ struct WriteResult { * LastError is updated for failures of individual writes, but not for batch errors reported by an * exception being thrown from these functions. Callers are responsible for managing LastError in * that case. This should generally be combined with LastError handling from parse failures. + * + * 'fromMigrate' indicates whether the operation was induced by a chunk migration */ -WriteResult performInserts(OperationContext* txn, const InsertOp& op); +WriteResult performInserts(OperationContext* txn, const InsertOp& op, bool fromMigrate = false); WriteResult performUpdates(OperationContext* txn, const UpdateOp& op); WriteResult performDeletes(OperationContext* txn, const DeleteOp& op); diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index 51d76e6cf73..79a378c2a6a 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -76,7 +76,6 @@ public: const Decimal128 decimalValue; }; -#pragma pack(1) class ValueStorage { public: // Note: it is important the memory is zeroed out (by calling zero()) at the start of every @@ -311,6 +310,7 @@ public: // This data is public because this should only be used by Value which would be a friend union { +#pragma pack(1) struct { // byte 1 signed char type; @@ -354,11 +354,16 @@ public: }; }; }; +#pragma pack() // covers the whole ValueStorage long long i64[2]; + + // Forces the ValueStorage type to have at least pointer alignment. Can't use alignas on the + // type since that causes issues on MSVC. + void* forcePointerAlignment; }; }; MONGO_STATIC_ASSERT(sizeof(ValueStorage) == 16); -#pragma pack() +MONGO_STATIC_ASSERT(alignof(ValueStorage) >= alignof(void*)); } diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index d6c0cc2bd8e..93d529704c9 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -407,7 +407,7 @@ StatusWith<PrepareExecutionResult> prepareExecution(OperationContext* opCtx, root.reset(rawRoot); LOG(2) << "Using fast count: " << redact(canonicalQuery->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); querySolution.reset(solutions[i]); return PrepareExecutionResult( @@ -425,7 +425,7 @@ StatusWith<PrepareExecutionResult> prepareExecution(OperationContext* opCtx, LOG(2) << "Only one plan is available; it will be run but will not be cached. " << redact(canonicalQuery->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); querySolution.reset(solutions[0]); return PrepareExecutionResult( @@ -1536,7 +1536,7 @@ StatusWith<unique_ptr<PlanExecutor>> getExecutorDistinct(OperationContext* txn, unique_ptr<PlanStage> root(rawRoot); LOG(2) << "Using fast distinct: " << redact(cq->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); return PlanExecutor::make(txn, std::move(ws), @@ -1572,7 +1572,7 @@ StatusWith<unique_ptr<PlanExecutor>> getExecutorDistinct(OperationContext* txn, unique_ptr<PlanStage> root(rawRoot); LOG(2) << "Using fast distinct: " << redact(cq->toStringShort()) - << ", planSummary: " << redact(Explain::getPlanSummary(root.get())); + << ", planSummary: " << Explain::getPlanSummary(root.get()); return PlanExecutor::make(txn, std::move(ws), diff --git a/src/mongo/db/query/index_bounds.cpp b/src/mongo/db/query/index_bounds.cpp index ef8eeb6d8b8..b8b7e4bce74 100644 --- a/src/mongo/db/query/index_bounds.cpp +++ b/src/mongo/db/query/index_bounds.cpp @@ -32,6 +32,7 @@ #include <tuple> #include <utility> +#include "mongo/base/simple_string_data_comparator.h" #include "mongo/bson/simple_bsonobj_comparator.h" namespace mongo { @@ -163,6 +164,21 @@ BoundInclusion IndexBounds::makeBoundInclusionFromBoundBools(bool startKeyInclus } } +BoundInclusion IndexBounds::reverseBoundInclusion(BoundInclusion b) { + switch (b) { + case BoundInclusion::kIncludeStartKeyOnly: + return BoundInclusion::kIncludeEndKeyOnly; + case BoundInclusion::kIncludeEndKeyOnly: + return BoundInclusion::kIncludeStartKeyOnly; + case BoundInclusion::kIncludeBothStartAndEndKeys: + case BoundInclusion::kExcludeBothStartAndEndKeys: + // These are both symmetric. + return b; + default: + MONGO_UNREACHABLE; + } +} + bool OrderedIntervalList::operator==(const OrderedIntervalList& other) const { if (this->name != other.name) { @@ -186,6 +202,38 @@ bool OrderedIntervalList::operator!=(const OrderedIntervalList& other) const { return !(*this == other); } +void OrderedIntervalList::reverse() { + for (size_t i = 0; i < (intervals.size() + 1) / 2; i++) { + const size_t otherIdx = intervals.size() - i - 1; + intervals[i].reverse(); + if (i != otherIdx) { + intervals[otherIdx].reverse(); + std::swap(intervals[i], intervals[otherIdx]); + } + } +} + +OrderedIntervalList OrderedIntervalList::reverseClone() const { + OrderedIntervalList clone(name); + + for (auto it = intervals.rbegin(); it != intervals.rend(); ++it) { + clone.intervals.push_back(it->reverseClone()); + } + + return clone; +} + +Interval::Direction OrderedIntervalList::computeDirection() const { + for (auto&& iv : intervals) { + const auto dir = iv.getDirection(); + if (dir != Interval::Direction::kDirectionNone) { + return dir; + } + } + + return Interval::Direction::kDirectionNone; +} + // static void OrderedIntervalList::complement() { BSONObjBuilder minBob; @@ -299,6 +347,40 @@ BSONObj IndexBounds::toBSON() const { return bob.obj(); } +IndexBounds IndexBounds::forwardize() const { + IndexBounds newBounds; + newBounds.isSimpleRange = isSimpleRange; + + if (isSimpleRange) { + const int cmpRes = startKey.woCompare(endKey); + if (cmpRes <= 0) { + newBounds.startKey = startKey; + newBounds.endKey = endKey; + newBounds.boundInclusion = boundInclusion; + } else { + // Swap start and end key. + newBounds.endKey = startKey; + newBounds.startKey = endKey; + newBounds.boundInclusion = IndexBounds::reverseBoundInclusion(boundInclusion); + } + + return newBounds; + } + + newBounds.fields.reserve(fields.size()); + std::transform(fields.begin(), + fields.end(), + std::back_inserter(newBounds.fields), + [](const OrderedIntervalList& oil) -> OrderedIntervalList { + if (oil.computeDirection() == Interval::Direction::kDirectionDescending) { + return oil.reverseClone(); + } + return oil; + }); + + return newBounds; +} + // // Validity checking for bounds // diff --git a/src/mongo/db/query/index_bounds.h b/src/mongo/db/query/index_bounds.h index 2dfcc122b26..4b04f374b16 100644 --- a/src/mongo/db/query/index_bounds.h +++ b/src/mongo/db/query/index_bounds.h @@ -74,6 +74,15 @@ struct OrderedIntervalList { bool operator==(const OrderedIntervalList& other) const; bool operator!=(const OrderedIntervalList& other) const; + + void reverse(); + + /** + * Return a clone of this OIL, that is reversed. + */ + OrderedIntervalList reverseClone() const; + + Interval::Direction computeDirection() const; }; /** @@ -126,6 +135,11 @@ struct IndexBounds { static BoundInclusion makeBoundInclusionFromBoundBools(bool startKeyInclusive, bool endKeyInclusive); + /** + * Reverse the BoundInclusion. + */ + static BoundInclusion reverseBoundInclusion(BoundInclusion b); + /** * BSON format for explain. The format is an array of strings for each field. @@ -137,6 +151,12 @@ struct IndexBounds { */ BSONObj toBSON() const; + /** + * Return a copy of the index bounds, but with each of the OILs going in the ascending + * direction. + */ + IndexBounds forwardize() const; + // TODO: we use this for max/min scan. Consider migrating that. bool isSimpleRange; BSONObj startKey; diff --git a/src/mongo/db/query/index_bounds_builder.cpp b/src/mongo/db/query/index_bounds_builder.cpp index eeb1c2f1114..a326b960192 100644 --- a/src/mongo/db/query/index_bounds_builder.cpp +++ b/src/mongo/db/query/index_bounds_builder.cpp @@ -54,6 +54,23 @@ namespace mongo { namespace { +// Helper for checking that an OIL "appears" to be ascending given one interval. +void assertOILIsAscendingLocally(const vector<Interval>& intervals, size_t idx) { + // Each individual interval being examined should be ascending or none. + const auto dir = intervals[idx].getDirection(); + + // Should be either ascending, or have no direction (be a point/null/empty interval). + invariant(dir == Interval::Direction::kDirectionAscending || + dir == Interval::Direction::kDirectionNone); + + // The previous OIL's end value should be <= the next OIL's start value. + if (idx > 0) { + // Pass 'false' to avoid comparing the field names. + const int res = intervals[idx - 1].end.woCompare(intervals[idx].start, false); + invariant(res <= 0); + } +} + // Tightness rules are shared for $lt, $lte, $gt, $gte. IndexBoundsBuilder::BoundsTightness getInequalityPredicateTightness(const BSONElement& dataElt, const IndexEntry& index) { @@ -624,52 +641,57 @@ Interval IndexBoundsBuilder::makeRangeInterval(const BSONObj& obj, BoundInclusio } // static -void IndexBoundsBuilder::intersectize(const OrderedIntervalList& arg, OrderedIntervalList* oilOut) { - verify(arg.name == oilOut->name); +void IndexBoundsBuilder::intersectize(const OrderedIntervalList& oilA, OrderedIntervalList* oilB) { + invariant(oilB); + invariant(oilA.name == oilB->name); - size_t argidx = 0; - const vector<Interval>& argiv = arg.intervals; + size_t oilAIdx = 0; + const vector<Interval>& oilAIntervals = oilA.intervals; - size_t ividx = 0; - vector<Interval>& iv = oilOut->intervals; + size_t oilBIdx = 0; + vector<Interval>& oilBIntervals = oilB->intervals; vector<Interval> result; - while (argidx < argiv.size() && ividx < iv.size()) { - Interval::IntervalComparison cmp = argiv[argidx].compare(iv[ividx]); + while (oilAIdx < oilAIntervals.size() && oilBIdx < oilBIntervals.size()) { + if (kDebugBuild) { + // Ensure that both OILs are ascending. + assertOILIsAscendingLocally(oilAIntervals, oilAIdx); + assertOILIsAscendingLocally(oilBIntervals, oilBIdx); + } + Interval::IntervalComparison cmp = oilAIntervals[oilAIdx].compare(oilBIntervals[oilBIdx]); verify(Interval::INTERVAL_UNKNOWN != cmp); if (cmp == Interval::INTERVAL_PRECEDES || cmp == Interval::INTERVAL_PRECEDES_COULD_UNION) { - // argiv is before iv. move argiv forward. - ++argidx; + // oilAIntervals is before oilBIntervals. move oilAIntervals forward. + ++oilAIdx; } else if (cmp == Interval::INTERVAL_SUCCEEDS) { - // iv is before argiv. move iv forward. - ++ividx; + // oilBIntervals is before oilAIntervals. move oilBIntervals forward. + ++oilBIdx; } else { - // argiv[argidx] (cmpresults) iv[ividx] - Interval newInt = argiv[argidx]; - newInt.intersect(iv[ividx], cmp); + Interval newInt = oilAIntervals[oilAIdx]; + newInt.intersect(oilBIntervals[oilBIdx], cmp); result.push_back(newInt); if (Interval::INTERVAL_EQUALS == cmp) { - ++argidx; - ++ividx; + ++oilAIdx; + ++oilBIdx; } else if (Interval::INTERVAL_WITHIN == cmp) { - ++argidx; + ++oilAIdx; } else if (Interval::INTERVAL_CONTAINS == cmp) { - ++ividx; + ++oilBIdx; } else if (Interval::INTERVAL_OVERLAPS_BEFORE == cmp) { - ++argidx; + ++oilAIdx; } else if (Interval::INTERVAL_OVERLAPS_AFTER == cmp) { - ++ividx; + ++oilBIdx; } else { - verify(0); + MONGO_UNREACHABLE; } } } - oilOut->intervals.swap(result); + oilB->intervals.swap(result); } // static @@ -892,13 +914,7 @@ void IndexBoundsBuilder::alignBounds(IndexBounds* bounds, const BSONObj& kp, int int direction = (elt.number() >= 0) ? 1 : -1; direction *= scanDir; if (-1 == direction) { - vector<Interval>& iv = bounds->fields[oilIdx].intervals; - // Step 1: reverse the list. - std::reverse(iv.begin(), iv.end()); - // Step 2: reverse each interval. - for (size_t i = 0; i < iv.size(); ++i) { - iv[i].reverse(); - } + bounds->fields[oilIdx].reverse(); } ++oilIdx; } diff --git a/src/mongo/db/query/index_bounds_builder_test.cpp b/src/mongo/db/query/index_bounds_builder_test.cpp index a8ae3651d32..77d447094e5 100644 --- a/src/mongo/db/query/index_bounds_builder_test.cpp +++ b/src/mongo/db/query/index_bounds_builder_test.cpp @@ -2187,4 +2187,19 @@ TEST(IndexBoundsBuilderTest, CanUseCoveredMatchingForExistsTrueWithSparseIndex) ASSERT_TRUE(IndexBoundsBuilder::canUseCoveredMatching(expr.get(), testIndex)); } +TEST(IndexBoundsBuilderTest, IntersectizeBasic) { + OrderedIntervalList oil1("xyz"); + oil1.intervals = {Interval(BSON("" << 0 << "" << 5), false, false)}; + + OrderedIntervalList oil2("xyz"); + oil2.intervals = {Interval(BSON("" << 1 << "" << 6), false, false)}; + + IndexBoundsBuilder::intersectize(oil1, &oil2); + + OrderedIntervalList expectedIntersection("xyz"); + expectedIntersection.intervals = {Interval(BSON("" << 1 << "" << 5), false, false)}; + + ASSERT_TRUE(oil2 == expectedIntersection); +} + } // namespace diff --git a/src/mongo/db/query/index_bounds_test.cpp b/src/mongo/db/query/index_bounds_test.cpp index 1fca089e584..5597eaa9f6f 100644 --- a/src/mongo/db/query/index_bounds_test.cpp +++ b/src/mongo/db/query/index_bounds_test.cpp @@ -122,6 +122,58 @@ TEST(IndexBoundsTest, ValidOverlapOnlyWhenBothOpen) { ASSERT(bounds.isValidFor(BSON("foo" << 1), 1)); } +TEST(IndexBoundsCheckerTest, CheckOILReverse) { + // Check that the reverse of an empty list is empty. + OrderedIntervalList emptyList("someField"); + emptyList.reverse(); + OrderedIntervalList expectedReversedEmptyList("someField"); + ASSERT_TRUE(emptyList == expectedReversedEmptyList); + + // The reverse of a single-interval OIL is just an OIL with that interval reversed. + OrderedIntervalList singleEltList("xyz"); + singleEltList.intervals = {Interval(BSON("" << 5 << "" << 0), false, false)}; + singleEltList.reverse(); + + OrderedIntervalList expectedReversedSingleEltList("xyz"); + expectedReversedSingleEltList.intervals = {Interval(BSON("" << 0 << "" << 5), false, false)}; + ASSERT_TRUE(singleEltList == expectedReversedSingleEltList); + + // List with a few elements + OrderedIntervalList fooList("foo"); + fooList.intervals = {Interval(BSON("" << 40 << "" << 35), false, true), + Interval(BSON("" << 30 << "" << 21), true, true), + Interval(BSON("" << 20 << "" << 7), true, false)}; + fooList.reverse(); + + OrderedIntervalList expectedReverseFooList("foo"); + expectedReverseFooList.intervals = {Interval(BSON("" << 7 << "" << 20), false, true), + Interval(BSON("" << 21 << "" << 30), true, true), + Interval(BSON("" << 35 << "" << 40), true, false)}; + + ASSERT_TRUE(fooList == expectedReverseFooList); +} + +TEST(IndexBoundsTest, OILReverseClone) { + OrderedIntervalList emptyA("foo"); + OrderedIntervalList emptyB = emptyA.reverseClone(); + + ASSERT(emptyA == emptyB); + ASSERT(emptyA.computeDirection() == Interval::Direction::kDirectionNone); + ASSERT(emptyB.computeDirection() == Interval::Direction::kDirectionNone); + + OrderedIntervalList list("foo"); + + list.intervals.push_back(Interval(BSON("" << 7 << "" << 20), true, false)); + list.intervals.push_back(Interval(BSON("" << 20 << "" << 25), false, true)); + + OrderedIntervalList listClone = list.reverseClone(); + OrderedIntervalList reverseList("foo"); + reverseList.intervals = {Interval(BSON("" << 25 << "" << 20), true, false), + Interval(BSON("" << 20 << "" << 7), false, true)}; + ASSERT(reverseList == listClone); + ASSERT(listClone.computeDirection() == Interval::Direction::kDirectionDescending); +} + // // Tests for OrderedIntervalList::complement() // @@ -519,6 +571,47 @@ TEST(IndexBoundsTest, SimpleRangeBoundsNotEqualDifferentEndKeyInclusive) { ASSERT_TRUE(bounds1 != bounds2); } +TEST(IndexBoundsTest, ForwardizeSimpleRange) { + IndexBounds bounds1; + bounds1.isSimpleRange = true; + bounds1.startKey = BSON("" << 2 << "" << 4); + bounds1.endKey = BSON("" << 1 << "" << 3); + bounds1.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; + + IndexBounds expectedBounds1; + expectedBounds1.isSimpleRange = true; + expectedBounds1.startKey = bounds1.endKey; + expectedBounds1.endKey = bounds1.startKey; + expectedBounds1.boundInclusion = BoundInclusion::kIncludeEndKeyOnly; + ASSERT(bounds1.forwardize() == expectedBounds1); + + IndexBounds bounds2; + bounds1.isSimpleRange = true; + bounds1.startKey = BSON("" << 1 << "" << 3); + bounds1.endKey = BSON("" << 2 << "" << 4); + bounds1.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; + ASSERT(bounds2 == bounds2.forwardize()); +} + + +TEST(IndexBoundsTest, ForwardizeOnNonSimpleRangeShouldOnlyReverseDescendingRanges) { + OrderedIntervalList fooList("foo"); + fooList.intervals = {Interval(BSON("" << 7 << "" << 20), true, true)}; + + OrderedIntervalList barList("bar"); + barList.intervals = {Interval(BSON("" << 10 << "" << 5), false, false), + Interval(BSON("" << 4 << "" << 3), false, false)}; + + IndexBounds bounds; + bounds.fields = {fooList, barList}; + + IndexBounds forwardizedBounds = bounds.forwardize(); + + IndexBounds expectedBounds; + expectedBounds.fields = {fooList, barList.reverseClone()}; + ASSERT(expectedBounds == forwardizedBounds); +} + // // Iteration over // diff --git a/src/mongo/db/query/interval.cpp b/src/mongo/db/query/interval.cpp index df80321eb60..187d0704a20 100644 --- a/src/mongo/db/query/interval.cpp +++ b/src/mongo/db/query/interval.cpp @@ -66,6 +66,19 @@ bool Interval::isNull() const { return (!startInclusive || !endInclusive) && 0 == start.woCompare(end, false); } +Interval::Direction Interval::getDirection() const { + if (isEmpty() || isPoint() || isNull()) { + return Direction::kDirectionNone; + } + + // 'false' to not consider the field name. + const int res = start.woCompare(end, false); + + invariant(res != 0); + return res < 0 ? Direction::kDirectionAscending : Direction::kDirectionDescending; +} + + // // Comparison // @@ -93,6 +106,17 @@ bool Interval::equals(const Interval& other) const { } bool Interval::intersects(const Interval& other) const { + if (kDebugBuild) { + // This function assumes that both intervals are ascending (or are empty/point intervals). + // Determining this may be expensive, so we only do these checks when in a debug build. + const auto thisDir = getDirection(); + invariant(thisDir == Direction::kDirectionAscending || + thisDir == Direction::kDirectionNone); + const auto otherDir = other.getDirection(); + invariant(otherDir == Direction::kDirectionAscending || + otherDir == Direction::kDirectionNone); + } + int res = this->start.woCompare(other.end, false); if (res > 0) { return false; @@ -261,4 +285,15 @@ void Interval::reverse() { std::swap(startInclusive, endInclusive); } +Interval Interval::reverseClone() const { + Interval reversed; + reversed.start = end; + reversed.end = start; + reversed.startInclusive = endInclusive; + reversed.endInclusive = startInclusive; + reversed._intervalData = _intervalData; + + return reversed; +} + } // namespace mongo diff --git a/src/mongo/db/query/interval.h b/src/mongo/db/query/interval.h index 66767036f95..21b2cfce4ab 100644 --- a/src/mongo/db/query/interval.h +++ b/src/mongo/db/query/interval.h @@ -98,6 +98,18 @@ struct Interval { */ bool isNull() const; + enum class Direction { + // Point intervals, empty intervals, and null intervals have no direction. + kDirectionNone, + kDirectionAscending, + kDirectionDescending + }; + + /** + * Compute the direction. + */ + Direction getDirection() const; + // // Comparison with other intervals // @@ -169,6 +181,11 @@ struct Interval { void reverse(); /** + * Return a new Interval that's a reverse of this one. + */ + Interval reverseClone() const; + + /** * Updates 'this' with the intersection of 'this' and 'other'. If 'this' and 'other' * have been compare()d before, that result can be optionally passed in 'cmp' */ @@ -182,7 +199,7 @@ struct Interval { }; inline bool operator==(const Interval& lhs, const Interval& rhs) { - return lhs.compare(rhs) == Interval::INTERVAL_EQUALS; + return lhs.equals(rhs); } inline bool operator!=(const Interval& lhs, const Interval& rhs) { diff --git a/src/mongo/db/query/interval_test.cpp b/src/mongo/db/query/interval_test.cpp index d9e829a254b..608f7e25459 100644 --- a/src/mongo/db/query/interval_test.cpp +++ b/src/mongo/db/query/interval_test.cpp @@ -293,4 +293,41 @@ TEST(Union, Succeds) { ASSERT_EQUALS(a.compare(Interval(itv, true, true)), Interval::INTERVAL_EQUALS); } +TEST(Introspection, GetDirection) { + // Empty/uninitialized Interval. + boost::optional<Interval> i; + i.emplace(); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Empty Interval. + i.emplace(BSON("" << 10 << "" << 10), false, false); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Point bound Interval. + i.emplace(BSON("" << 10 << "" << 10), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionNone); + + // Ascending interval. + i.emplace(BSON("" << 10 << "" << 20), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionAscending); + + // Descending interval. + i.emplace(BSON("" << 11 << "" << 10), true, true); + ASSERT(i->getDirection() == Interval::Direction::kDirectionDescending); +} + +TEST(Copying, ReverseClone) { + Interval a(BSON("" << 10 << "" << 20), false, true); + ASSERT(a.reverseClone() == Interval(BSON("" << 20 << "" << 10), true, false)); + ASSERT(a.reverseClone() != a); + + Interval b(BSON("" << 10 << "" << 5), true, true); + ASSERT(b.reverseClone() == Interval(BSON("" << 5 << "" << 10), true, true)); + ASSERT(b.reverseClone() != b); + + Interval c(BSON("" << 1 << "" << 1), true, true); + ASSERT(c.reverseClone() == c); +} + + } // unnamed namespace diff --git a/src/mongo/db/query/plan_ranker.cpp b/src/mongo/db/query/plan_ranker.cpp index ce943e117a7..9377e1382fc 100644 --- a/src/mongo/db/query/plan_ranker.cpp +++ b/src/mongo/db/query/plan_ranker.cpp @@ -95,7 +95,7 @@ size_t PlanRanker::pickBestPlan(const vector<CandidatePlan>& candidates, PlanRan LOG(5) << "Scoring plan " << i << ":" << endl << redact(candidates[i].solution->toString()) << "Stats:\n" << redact(Explain::statsToBSON(*statTrees[i]).jsonString(Strict, true)); - LOG(2) << "Scoring query plan: " << redact(Explain::getPlanSummary(candidates[i].root)) + LOG(2) << "Scoring query plan: " << Explain::getPlanSummary(candidates[i].root) << " planHitEOF=" << statTrees[i]->common.isEOF; double score = scoreTree(statTrees[i]); diff --git a/src/mongo/db/query/query_planner_common.cpp b/src/mongo/db/query/query_planner_common.cpp index 337f3a045fc..1347332c077 100644 --- a/src/mongo/db/query/query_planner_common.cpp +++ b/src/mongo/db/query/query_planner_common.cpp @@ -46,27 +46,11 @@ void QueryPlannerCommon::reverseScans(QuerySolutionNode* node) { if (isn->bounds.isSimpleRange) { std::swap(isn->bounds.startKey, isn->bounds.endKey); // If only one bound is included, swap which one is included. - switch (isn->bounds.boundInclusion) { - case BoundInclusion::kIncludeStartKeyOnly: - isn->bounds.boundInclusion = BoundInclusion::kIncludeEndKeyOnly; - break; - case BoundInclusion::kIncludeEndKeyOnly: - isn->bounds.boundInclusion = BoundInclusion::kIncludeStartKeyOnly; - break; - case BoundInclusion::kIncludeBothStartAndEndKeys: - case BoundInclusion::kExcludeBothStartAndEndKeys: - // These are both symmetric so no change needed. - break; - } + isn->bounds.boundInclusion = + IndexBounds::reverseBoundInclusion(isn->bounds.boundInclusion); } else { for (size_t i = 0; i < isn->bounds.fields.size(); ++i) { - std::vector<Interval>& iv = isn->bounds.fields[i].intervals; - // Step 1: reverse the list. - std::reverse(iv.begin(), iv.end()); - // Step 2: reverse each interval. - for (size_t j = 0; j < iv.size(); ++j) { - iv[j].reverse(); - } + isn->bounds.fields[i].reverse(); } } diff --git a/src/mongo/db/query/query_solution.cpp b/src/mongo/db/query/query_solution.cpp index aa2426954c4..9ab324ab3bd 100644 --- a/src/mongo/db/query/query_solution.cpp +++ b/src/mongo/db/query/query_solution.cpp @@ -594,9 +594,13 @@ bool IndexScanNode::sortedByDiskLoc() const { } // static -std::set<StringData> IndexScanNode::getFieldsWithStringBounds(const IndexBounds& bounds, +std::set<StringData> IndexScanNode::getFieldsWithStringBounds(const IndexBounds& inputBounds, const BSONObj& indexKeyPattern) { - BSONObjIterator keyPatternIterator = indexKeyPattern.begin(); + // Produce a copy of the bounds which are all ascending, as we can only compute intersections + // of ascending bounds. + IndexBounds bounds = inputBounds.forwardize(); + + BSONObjIterator keyPatternIterator(indexKeyPattern); if (bounds.isSimpleRange) { // With a simple range, the only cases we can say for sure do not contain strings diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index fbbf1c56efd..ac49485ede0 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -427,7 +427,7 @@ void BackgroundSync::_produce() { source, NamespaceString(rsOplogName), _replCoord->getConfig(), - _replicationCoordinatorExternalState->getOplogFetcherMaxFetcherRestarts(), + _replicationCoordinatorExternalState->getOplogFetcherSteadyStateMaxFetcherRestarts(), syncSourceResp.rbid, true /* requireFresherSyncSource */, &dataReplicatorExternalState, diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 76df271ed78..13769c5334d 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -658,12 +658,16 @@ std::map<std::string, ApplyOpMetadata> opsMap = { return applyOps(txn, nsToDatabase(ns), cmd, &resultWeDontCareAbout); }, {ErrorCodes::UnknownError}}}, - {"convertToCapped", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { - return convertToCapped(txn, parseNs(ns, cmd), cmd["size"].number()); - }}}, - {"emptycapped", {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { - return emptyCapped(txn, parseNs(ns, cmd)); - }}}, + {"convertToCapped", + {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { + return convertToCapped(txn, parseNs(ns, cmd), cmd["size"].number()); + }, + {ErrorCodes::NamespaceNotFound}}}, + {"emptycapped", + {[](OperationContext* txn, const char* ns, BSONObj& cmd) -> Status { + return emptyCapped(txn, parseNs(ns, cmd)); + }, + {ErrorCodes::NamespaceNotFound}}}, }; } // namespace diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp index c9b96ab77f7..36a6e237e73 100644 --- a/src/mongo/db/repl/oplog_fetcher.cpp +++ b/src/mongo/db/repl/oplog_fetcher.cpp @@ -58,8 +58,15 @@ MONGO_FP_DECLARE(stopReplProducer); namespace { // Number of seconds for the `maxTimeMS` on the initial `find` command. +// +// For the initial 'find' request, we provide a generous timeout, to account for the potentially +// slow process of a sync source finding the lastApplied optime provided in a node's query in its +// oplog. MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60); +// Number of seconds for the `maxTimeMS` on any retried `find` commands. +MONGO_EXPORT_SERVER_PARAMETER(oplogRetriedFindMaxSeconds, int, 2); + // Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network // timeout for the requests. const Milliseconds kNetworkTimeoutBufferMS{5000}; @@ -373,7 +380,7 @@ OplogFetcher::OplogFetcher(executor::TaskExecutor* executor, uassert(ErrorCodes::BadValue, "null onShutdownCallback function", onShutdownCallbackFn); auto currentTerm = dataReplicatorExternalState->getCurrentTermAndLastCommittedOpTime().value; - _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime); + _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime, _getInitialFindMaxTime()); } OplogFetcher::~OplogFetcher() { @@ -463,10 +470,14 @@ Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const { return _getGetMoreMaxTime(); } -Milliseconds OplogFetcher::_getFindMaxTime() const { +Milliseconds OplogFetcher::_getInitialFindMaxTime() const { return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000); } +Milliseconds OplogFetcher::_getRetriedFindMaxTime() const { + return Milliseconds(oplogRetriedFindMaxSeconds.load() * 1000); +} + Milliseconds OplogFetcher::_getGetMoreMaxTime() const { return _awaitDataTimeout; } @@ -511,7 +522,7 @@ void OplogFetcher::_callback(const Fetcher::QueryResponseStatus& result, // Move the old fetcher into the shutting down instance. _shuttingDownFetcher.swap(_fetcher); // Create and start fetcher with current term and new starting optime. - _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime); + _fetcher = _makeFetcher(currentTerm, _lastFetched.opTime, _getRetriedFindMaxTime()); auto scheduleStatus = _scheduleFetcher_inlock(); if (scheduleStatus.isOK()) { log() << "Scheduled new oplog query " << _fetcher->toString(); @@ -704,15 +715,16 @@ void OplogFetcher::_finishCallback(Status status, OpTimeWithHash opTimeWithHash) } std::unique_ptr<Fetcher> OplogFetcher::_makeFetcher(long long currentTerm, - OpTime lastFetchedOpTime) { + OpTime lastFetchedOpTime, + Milliseconds findMaxTime) { return stdx::make_unique<Fetcher>( _executor, _source, _nss.db().toString(), - makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime, _getFindMaxTime()), + makeFindCommandObject(_nss, currentTerm, lastFetchedOpTime, findMaxTime), stdx::bind(&OplogFetcher::_callback, this, stdx::placeholders::_1, stdx::placeholders::_3), _metadataObject, - _getFindMaxTime() + kNetworkTimeoutBufferMS, + findMaxTime + kNetworkTimeoutBufferMS, _getGetMoreMaxTime() + kNetworkTimeoutBufferMS); } diff --git a/src/mongo/db/repl/oplog_fetcher.h b/src/mongo/db/repl/oplog_fetcher.h index 54bfbabbf8d..c567598a66f 100644 --- a/src/mongo/db/repl/oplog_fetcher.h +++ b/src/mongo/db/repl/oplog_fetcher.h @@ -237,7 +237,16 @@ private: /** * Returns how long the `find` command should wait before timing out. */ - virtual Milliseconds _getFindMaxTime() const; + virtual Milliseconds _getInitialFindMaxTime() const; + + /** + * Returns how long the `find` command should wait before timing out, if we are retrying the + * 'find' due to an error. This timeout should be considerably smaller than our initial oplog + * find time, since a communication failure with an upstream node may indicate it is + * unreachable. + */ + virtual Milliseconds _getRetriedFindMaxTime() const; + /** * Returns how long the `getMore` command should wait before timing out. @@ -247,7 +256,9 @@ private: /** * Creates a new instance of the fetcher to tail the remote oplog starting at the given optime. */ - std::unique_ptr<Fetcher> _makeFetcher(long long currentTerm, OpTime lastFetchedOpTime); + std::unique_ptr<Fetcher> _makeFetcher(long long currentTerm, + OpTime lastFetchedOpTime, + Milliseconds findMaxTime); /** * Returns whether the oplog fetcher is in shutdown. diff --git a/src/mongo/db/repl/oplog_fetcher_test.cpp b/src/mongo/db/repl/oplog_fetcher_test.cpp index 9170c203695..2b25d0bbe02 100644 --- a/src/mongo/db/repl/oplog_fetcher_test.cpp +++ b/src/mongo/db/repl/oplog_fetcher_test.cpp @@ -204,6 +204,11 @@ BSONObj OplogFetcherTest::makeOplogQueryMetadataObject(OpTime lastAppliedOpTime, HostAndPort source("localhost:12345"); NamespaceString nss("local.oplog.rs"); +// For testing, set these network timeouts to match the defaults in the OplogFetcher. +const Milliseconds kNetworkTimeoutBufferMS{5000}; +const Milliseconds initialFindMaxTime = Milliseconds(60000); +const Milliseconds retriedFindMaxTime = Milliseconds(2000); + ReplSetConfig _createConfig(bool isV1ElectionProtocol) { BSONObjBuilder bob; bob.append("_id", "myset"); @@ -1478,6 +1483,99 @@ TEST_F(OplogFetcherTest, OplogFetcherAbortsWithOriginalResponseErrorOnFailureToS ASSERT_EQUALS(_getOpTimeWithHash(ops[2]), shutdownState->getLastFetched()); } +TEST_F(OplogFetcherTest, OplogFetcherTimesOutCorrectlyOnInitialFindRequests) { + auto ops = _generateOplogEntries(2U); + std::size_t maxFetcherRestarts = 0U; + auto shutdownState = stdx::make_unique<ShutdownState>(); + OplogFetcher oplogFetcher(&getExecutor(), + _getOpTimeWithHash(ops[0]), + source, + nss, + _createConfig(true), + maxFetcherRestarts, + rbid, + true, + dataReplicatorExternalState.get(), + enqueueDocumentsFn, + stdx::ref(*shutdownState)); + + ON_BLOCK_EXIT([this] { getExecutor().shutdown(); }); + + ASSERT_OK(oplogFetcher.startup()); + ASSERT_TRUE(oplogFetcher.isActive()); + + auto net = getNet(); + + // Schedule a response at a time that would exceed the initial find request network timeout. + net->enterNetwork(); + auto when = net->now() + initialFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + auto noi = getNet()->getNextReadyRequest(); + RemoteCommandResponse response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + auto request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + oplogFetcher.join(); + + // The fetcher should have shut down after its last request timed out. + ASSERT_EQUALS(ErrorCodes::NetworkTimeout, shutdownState->getStatus()); +} + +TEST_F(OplogFetcherTest, OplogFetcherTimesOutCorrectlyOnRetriedFindRequests) { + auto ops = _generateOplogEntries(2U); + std::size_t maxFetcherRestarts = 1U; + auto shutdownState = stdx::make_unique<ShutdownState>(); + OplogFetcher oplogFetcher(&getExecutor(), + _getOpTimeWithHash(ops[0]), + source, + nss, + _createConfig(true), + maxFetcherRestarts, + rbid, + true, + dataReplicatorExternalState.get(), + enqueueDocumentsFn, + stdx::ref(*shutdownState)); + + + ON_BLOCK_EXIT([this] { getExecutor().shutdown(); }); + + ASSERT_OK(oplogFetcher.startup()); + ASSERT_TRUE(oplogFetcher.isActive()); + + auto net = getNet(); + + // Schedule a response at a time that would exceed the initial find request network timeout. + net->enterNetwork(); + auto when = net->now() + initialFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + auto noi = getNet()->getNextReadyRequest(); + RemoteCommandResponse response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + auto request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + // Schedule a response at a time that would exceed the retried find request network timeout. + net->enterNetwork(); + when = net->now() + retriedFindMaxTime + kNetworkTimeoutBufferMS + Milliseconds(10); + noi = getNet()->getNextReadyRequest(); + response = { + {makeCursorResponse(1, {ops[0], ops[1]})}, rpc::makeEmptyMetadata(), Milliseconds(0)}; + request = net->scheduleSuccessfulResponse(noi, when, response); + net->runUntil(when); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + oplogFetcher.join(); + + // The fetcher should have shut down after its last request timed out. + ASSERT_EQUALS(ErrorCodes::NetworkTimeout, shutdownState->getStatus()); +} + + bool sharedCallbackStateDestroyed = false; class SharedCallbackState { MONGO_DISALLOW_COPYING(SharedCallbackState); diff --git a/src/mongo/db/repl/repl_set_request_votes.cpp b/src/mongo/db/repl/repl_set_request_votes.cpp index 02eb5311cb3..8c1fe8adece 100644 --- a/src/mongo/db/repl/repl_set_request_votes.cpp +++ b/src/mongo/db/repl/repl_set_request_votes.cpp @@ -83,8 +83,10 @@ private: ReplSetRequestVotesResponse response; status = getGlobalReplicationCoordinator()->processReplSetRequestVotes( txn, parsedArgs, &response); + uassertStatusOK(status); + response.addToBSON(&result); - return appendCommandStatus(result, status); + return true; } } cmdReplSetRequestVotes; diff --git a/src/mongo/db/repl/replication_coordinator_external_state.h b/src/mongo/db/repl/replication_coordinator_external_state.h index 8776bfa8330..dadaaff2d2d 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state.h +++ b/src/mongo/db/repl/replication_coordinator_external_state.h @@ -348,9 +348,15 @@ public: /** * Returns maximum number of times that the oplog fetcher will consecutively restart the oplog - * tailing query on non-cancellation errors. + * tailing query on non-cancellation errors during steady state replication. */ - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const = 0; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const = 0; + + /** + * Returns maximum number of times that the oplog fetcher will consecutively restart the oplog + * tailing query on non-cancellation errors during initial sync. + */ + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const = 0; /* * Creates noop writer instance. Setting the _noopWriter member is not protected by a guard, diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 05d070c08fe..597c9b47799 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -37,6 +37,9 @@ #include "mongo/base/init.h" #include "mongo/base/status_with.h" #include "mongo/bson/oid.h" +#include "mongo/db/auth/auth_index_d.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_holder.h" #include "mongo/db/client.h" @@ -128,29 +131,59 @@ MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBuffer, // Set this to specify size of read ahead buffer in the OplogBufferCollection. MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBufferPeekCacheSize, int, 10000); -// Set this to specify maximum number of times the oplog fetcher will consecutively restart the -// oplog tailing query on non-cancellation errors. +// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the +// oplog tailing query on non-cancellation errors during steady state replication. server_parameter_storage_type<int, ServerParameterType::kStartupAndRuntime>::value_type - oplogFetcherMaxFetcherRestarts(3); -class ExportedOplogFetcherMaxFetcherRestartsServerParameter + oplogFetcherSteadyStateMaxFetcherRestarts(1); +class ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter : public ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime> { public: - ExportedOplogFetcherMaxFetcherRestartsServerParameter(); + ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter(); Status validate(const int& potentialNewValue) override; -} _exportedOplogFetcherMaxFetcherRestartsServerParameter; +} _exportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter; -ExportedOplogFetcherMaxFetcherRestartsServerParameter:: - ExportedOplogFetcherMaxFetcherRestartsServerParameter() +ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter:: + ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter() : ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime>( ServerParameterSet::getGlobal(), - "oplogFetcherMaxFetcherRestarts", - &oplogFetcherMaxFetcherRestarts) {} + "oplogFetcherSteadyStateMaxFetcherRestarts", + &oplogFetcherSteadyStateMaxFetcherRestarts) {} -Status ExportedOplogFetcherMaxFetcherRestartsServerParameter::validate( +Status ExportedOplogFetcherSteadyStateMaxFetcherRestartsServerParameter::validate( const int& potentialNewValue) { if (potentialNewValue < 0) { - return Status(ErrorCodes::BadValue, - "oplogFetcherMaxFetcherRestarts must be greater than or equal to 0"); + return Status( + ErrorCodes::BadValue, + "oplogFetcherSteadyStateMaxFetcherRestarts must be greater than or equal to 0"); + } + return Status::OK(); +} + +// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the +// oplog tailing query on non-cancellation errors during initial sync. By default we provide a +// generous amount of restarts to avoid potentially restarting an entire initial sync from scratch. +server_parameter_storage_type<int, ServerParameterType::kStartupAndRuntime>::value_type + oplogFetcherInitialSyncMaxFetcherRestarts(10); +class ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter + : public ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime> { +public: + ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter(); + Status validate(const int& potentialNewValue) override; +} _exportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter; + +ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter:: + ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter() + : ExportedServerParameter<int, ServerParameterType::kStartupAndRuntime>( + ServerParameterSet::getGlobal(), + "oplogFetcherInitialSyncMaxFetcherRestarts", + &oplogFetcherInitialSyncMaxFetcherRestarts) {} + +Status ExportedOplogFetcherInitialSyncMaxFetcherRestartsServerParameter::validate( + const int& potentialNewValue) { + if (potentialNewValue < 0) { + return Status( + ErrorCodes::BadValue, + "oplogFetcherInitialSyncMaxFetcherRestarts must be greater than or equal to 0"); } return Status::OK(); } @@ -454,6 +487,16 @@ OpTime ReplicationCoordinatorExternalStateImpl::onTransitionToPrimary(OperationC _shardingOnTransitionToPrimaryHook(txn); _dropAllTempCollections(txn); + // It is only necessary to check the system indexes on the first transition to master. + // On subsequent transitions to master the indexes will have already been created. + static std::once_flag verifySystemIndexesOnce; + std::call_once(verifySystemIndexesOnce, [txn] { + const auto globalAuthzManager = AuthorizationManager::get(txn->getServiceContext()); + if (globalAuthzManager->shouldValidateAuthSchemaOnStartup()) { + fassert(65536, authindex::verifySystemIndexes(txn)); + } + }); + serverGlobalParams.featureCompatibility.validateFeaturesAsMaster.store(true); return opTimeToReturn; @@ -944,8 +987,14 @@ bool ReplicationCoordinatorExternalStateImpl::shouldUseDataReplicatorInitialSync return !use3dot2InitialSync; } -std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherMaxFetcherRestarts() const { - return oplogFetcherMaxFetcherRestarts; +std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherSteadyStateMaxFetcherRestarts() + const { + return oplogFetcherSteadyStateMaxFetcherRestarts.load(); +} + +std::size_t ReplicationCoordinatorExternalStateImpl::getOplogFetcherInitialSyncMaxFetcherRestarts() + const { + return oplogFetcherInitialSyncMaxFetcherRestarts.load(); } JournalListener::Token ReplicationCoordinatorExternalStateImpl::getToken() { diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.h b/src/mongo/db/repl/replication_coordinator_external_state_impl.h index 8926f378829..ff2fa102982 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.h @@ -116,7 +116,8 @@ public: virtual std::unique_ptr<OplogBuffer> makeSteadyStateOplogBuffer( OperationContext* txn) const override; virtual bool shouldUseDataReplicatorInitialSync() const override; - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const override; // Methods from JournalListener. virtual JournalListener::Token getToken(); diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp index 6b832728f24..cef211451a6 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp @@ -288,7 +288,13 @@ bool ReplicationCoordinatorExternalStateMock::shouldUseDataReplicatorInitialSync return true; } -std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherMaxFetcherRestarts() const { +std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherSteadyStateMaxFetcherRestarts() + const { + return 0; +} + +std::size_t ReplicationCoordinatorExternalStateMock::getOplogFetcherInitialSyncMaxFetcherRestarts() + const { return 0; } diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h index c22e053bf35..7120433bd73 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h @@ -109,7 +109,8 @@ public: virtual std::unique_ptr<OplogBuffer> makeSteadyStateOplogBuffer( OperationContext* txn) const override; virtual bool shouldUseDataReplicatorInitialSync() const override; - virtual std::size_t getOplogFetcherMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherSteadyStateMaxFetcherRestarts() const override; + virtual std::size_t getOplogFetcherInitialSyncMaxFetcherRestarts() const override; /** * Adds "host" to the list of hosts that this mock will match when responding to "isSelf" diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 10b60b7a205..eec1b8989ae 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -289,7 +289,8 @@ InitialSyncerOptions createInitialSyncerOptions( options.getSlaveDelay = [replCoord]() { return replCoord->getSlaveDelaySecs(); }; options.syncSourceSelector = replCoord; options.replBatchLimitBytes = dur::UncommittedBytesLimit; - options.oplogFetcherMaxFetcherRestarts = externalState->getOplogFetcherMaxFetcherRestarts(); + options.oplogFetcherMaxFetcherRestarts = + externalState->getOplogFetcherInitialSyncMaxFetcherRestarts(); return options; } } // namespace diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 09953633f7e..f9fea88bf0b 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -1530,4 +1530,38 @@ TEST_F(IdempotencyTest, ResyncOnRenameCollection) { ASSERT_EQUALS(runOp(op), ErrorCodes::OplogOperationUnsupported); } +TEST_F(IdempotencyTest, EmptyCappedNamespaceNotFound) { + // Create a BSON "emptycapped" command. + auto emptyCappedCmd = BSON("emptycapped" << nss.coll()); + + // Create an "emptycapped" oplog entry. + auto emptyCappedOp = makeCommandOplogEntry(nextOpTime(), nss, emptyCappedCmd); + + // Ensure that NamespaceNotFound is acceptable. + ASSERT_OK(runOps({emptyCappedOp})); + + AutoGetCollectionForRead autoColl(_opCtx.get(), nss); + + // Ensure that autoColl.getCollection() and autoColl.getDb() are both null. + ASSERT_FALSE(autoColl.getCollection()); + ASSERT_FALSE(autoColl.getDb()); +} + +TEST_F(IdempotencyTest, ConvertToCappedNamespaceNotFound) { + // Create a BSON "convertToCapped" command. + auto convertToCappedCmd = BSON("convertToCapped" << nss.coll()); + + // Create a "convertToCapped" oplog entry. + auto convertToCappedOp = makeCommandOplogEntry(nextOpTime(), nss, convertToCappedCmd); + + // Ensure that NamespaceNotFound is acceptable. + ASSERT_OK(runOps({convertToCappedOp})); + + AutoGetCollectionForRead autoColl(_opCtx.get(), nss); + + // Ensure that autoColl.getCollection() and autoColl.getDb() are both null. + ASSERT_FALSE(autoColl.getCollection()); + ASSERT_FALSE(autoColl.getDb()); +} + } // namespace diff --git a/src/mongo/db/repl/task_runner.cpp b/src/mongo/db/repl/task_runner.cpp index 210718bba3e..134160bbdf1 100644 --- a/src/mongo/db/repl/task_runner.cpp +++ b/src/mongo/db/repl/task_runner.cpp @@ -131,20 +131,17 @@ void TaskRunner::join() { } void TaskRunner::_runTasks() { - Client* client = nullptr; + // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() to be equal to the + // client used to create the operation context. + Client::initThreadIfNotAlready(); + Client* client = &cc(); + if (AuthorizationManager::get(client->getServiceContext())->isAuthEnabled()) { + AuthorizationSession::get(client)->grantInternalAuthorization(); + } ServiceContext::UniqueOperationContext txn; while (Task task = _waitForNextTask()) { if (!txn) { - if (!client) { - // We initialize cc() because ServiceContextMongoD::_newOpCtx() expects cc() - // to be equal to the client used to create the operation context. - Client::initThreadIfNotAlready(); - client = &cc(); - if (getGlobalAuthorizationManager()->isAuthEnabled()) { - AuthorizationSession::get(client)->grantInternalAuthorization(); - } - } txn = client->makeOperationContext(); } diff --git a/src/mongo/db/run_commands.cpp b/src/mongo/db/run_commands.cpp index 41adbdfb507..9068b8c9adb 100644 --- a/src/mongo/db/run_commands.cpp +++ b/src/mongo/db/run_commands.cpp @@ -62,7 +62,7 @@ void runCommands(OperationContext* txn, } LOG(2) << "run command " << request.getDatabase() << ".$cmd" << ' ' - << c->getRedactedCopyForLogging(request.getCommandArgs()); + << redact(c->getRedactedCopyForLogging(request.getCommandArgs())); { // Try to set this as early as possible, as soon as we have figured out the command. diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp index 0bba1ff478e..afda09358f2 100644 --- a/src/mongo/db/s/metadata_manager.cpp +++ b/src/mongo/db/s/metadata_manager.cpp @@ -192,8 +192,6 @@ void MetadataManager::beginReceive(const ChunkRange& range) { auto itRecv = _receivingChunks.find(overlapChunkMin.first); invariant(itRecv != _receivingChunks.end()); - const ChunkRange receivingRange(itRecv->first, itRecv->second.getMaxKey()); - _receivingChunks.erase(itRecv); } diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index 8dae501c57e..3c62381a1f8 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -47,6 +47,8 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/ops/delete.h" +#include "mongo/db/ops/write_ops.h" +#include "mongo/db/ops/write_ops_exec.h" #include "mongo/db/range_deleter_service.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/replication_coordinator_global.h" @@ -56,6 +58,7 @@ #include "mongo/db/s/move_timing_helper.h" #include "mongo/db/s/sharded_connection_info.h" #include "mongo/db/s/sharding_state.h" +#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/logger/ramlog.h" #include "mongo/s/catalog/type_chunk.h" @@ -349,7 +352,7 @@ Status MigrationDestinationManager::start(const NamespaceString& nss, void MigrationDestinationManager::cloneDocumentsFromDonor( OperationContext* txn, - stdx::function<void(OperationContext*, BSONObjIterator)> insertBatchFn, + stdx::function<void(OperationContext*, BSONObj)> insertBatchFn, stdx::function<BSONObj(OperationContext*)> fetchBatchFn) { ProducerConsumerQueue<BSONObj> batches(1); @@ -364,7 +367,7 @@ void MigrationDestinationManager::cloneDocumentsFromDonor( if (arr.isEmpty()) { return; } - insertBatchFn(inserterTxn.get(), BSONObjIterator(arr)); + insertBatchFn(inserterTxn.get(), arr); } } catch (...) { stdx::lock_guard<Client> lk(*txn->getClient()); @@ -506,6 +509,17 @@ void MigrationDestinationManager::_migrateThread(BSONObj min, _isActiveCV.notify_all(); } +// The maximum number of documents to insert in a single batch during migration clone. +// secondaryThrottle and migrateCloneInsertionBatchDelayMS apply between each batch. +// 0 or negative values (the default) means no limit to batch size. +// 1 corresponds to 3.4.16 (and earlier) behavior. +MONGO_EXPORT_SERVER_PARAMETER(migrateCloneInsertionBatchSize, int, 0); + +// Time in milliseconds between batches of insertions during migration clone. +// This is in addition to any time spent waiting for replication (secondaryThrottle). +// Defaults to 0. +MONGO_EXPORT_SERVER_PARAMETER(migrateCloneInsertionBatchDelayMS, int, 0); + void MigrationDestinationManager::_migrateDriver(OperationContext* txn, const BSONObj& min, const BSONObj& max, @@ -665,6 +679,12 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, wunit.commit(); } + Status status = _notePending(txn, _nss, min, max, epoch); + if (!status.isOK()) { + setState(FAIL); + return; + } + timing.done(1); MONGO_FAIL_POINT_PAUSE_WHILE_SET(migrateThreadHangAtStep1); } @@ -680,7 +700,12 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, // results deleterOptions.waitForOpenCursors = false; deleterOptions.fromMigrate = true; - deleterOptions.onlyRemoveOrphanedDocs = true; + + // There is no need to perform checking for orphaned docs as part of the range deletion, + // because the call above (to _notePending) will ensure that the chunk which is coming in is + // not currently owned by this shard and the scopedRegisterReceiveChunk would prevent this + // chunk from getting received while range deletion is running. + deleterOptions.onlyRemoveOrphanedDocs = false; deleterOptions.removeSaverReason = "preCleanup"; if (!getDeleter()->deleteNow(txn, deleterOptions, &errmsg)) { @@ -689,12 +714,6 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, return; } - Status status = _notePending(txn, _nss, min, max, epoch); - if (!status.isOK()) { - setState(FAIL); - return; - } - timing.done(2); MONGO_FAIL_POINT_PAUSE_WHILE_SET(migrateThreadHangAtStep2); } @@ -705,57 +724,61 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* txn, const BSONObj migrateCloneRequest = createMigrateCloneRequest(_nss, *_sessionId); - auto insertBatchFn = [&](OperationContext* txn, BSONObjIterator docs) { - while (docs.more()) { - txn->checkForInterrupt(); + auto assertNotAborted = [&](OperationContext* opCtx) { + opCtx->checkForInterrupt(); + uassert(40655, "Migration aborted while copying documents", getState() != ABORT); + }; - if (getState() == ABORT) { - auto message = "Migration aborted while copying documents"; - log() << message << migrateLog; - uasserted(40655, message); + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj arr) { + auto it = arr.begin(); + while (it != arr.end()) { + int batchNumCloned = 0; + int batchClonedBytes = 0; + int batchMaxCloned = migrateCloneInsertionBatchSize.load(); + + assertNotAborted(opCtx); + + std::vector<BSONObj> toInsert; + while (it != arr.end() && + (batchMaxCloned <= 0 || batchNumCloned < batchMaxCloned)) { + const auto& doc = *it; + BSONObj docToClone = doc.Obj(); + toInsert.push_back(docToClone); + batchNumCloned++; + batchClonedBytes += docToClone.objsize(); + it++; } + InsertOp insertOp; + insertOp.ns = _nss; + insertOp.documents = toInsert; - BSONObj docToClone = docs.next().Obj(); - { - OldClientWriteContext cx(txn, _nss.ns()); - BSONObj localDoc; - if (willOverrideLocalId(txn, - _nss.ns(), - min, - max, - shardKeyPattern, - cx.db(), - docToClone, - &localDoc)) { - const std::string errMsg = str::stream() - << "cannot migrate chunk, local document " << redact(localDoc) - << " has same _id as cloned " - << "remote document " << redact(docToClone); - warning() << errMsg; - - // Exception will abort migration cleanly - uasserted(16976, errMsg); - } - Helpers::upsert(txn, _nss.ns(), docToClone, true); + const WriteResult reply = performInserts(opCtx, insertOp, true); + + for (unsigned long i = 0; i < reply.results.size(); ++i) { + uassertStatusOK(reply.results[i]); } + { stdx::lock_guard<stdx::mutex> statsLock(_mutex); - _numCloned++; - _clonedBytes += docToClone.objsize(); + _numCloned += batchNumCloned; + _clonedBytes += batchClonedBytes; } + if (writeConcern.shouldWaitForOtherNodes()) { repl::ReplicationCoordinator::StatusAndDuration replStatus = - repl::ReplicationCoordinator::get(txn)->awaitReplication( - txn, - repl::ReplClientInfo::forClient(txn->getClient()).getLastOp(), + repl::ReplicationCoordinator::get(opCtx)->awaitReplication( + opCtx, + repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(), writeConcern); if (replStatus.status.code() == ErrorCodes::WriteConcernFailed) { warning() << "secondaryThrottle on, but doc insert timed out; " "continuing"; } else { - massertStatusOK(replStatus.status); + uassertStatusOK(replStatus.status); } } + + sleepmillis(migrateCloneInsertionBatchDelayMS.load()); } }; diff --git a/src/mongo/db/s/migration_destination_manager.h b/src/mongo/db/s/migration_destination_manager.h index 836761530d4..791aa393116 100644 --- a/src/mongo/db/s/migration_destination_manager.h +++ b/src/mongo/db/s/migration_destination_manager.h @@ -105,7 +105,7 @@ public: */ static void cloneDocumentsFromDonor( OperationContext* txn, - stdx::function<void(OperationContext*, BSONObjIterator)> insertBatchFn, + stdx::function<void(OperationContext*, BSONObj)> insertBatchFn, stdx::function<BSONObj(OperationContext*)> fetchBatchFn); /** diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp index ac9752fd96d..4c28de35b44 100644 --- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp +++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp @@ -111,11 +111,16 @@ public: } } - const NamespaceString nss(cmdObj.firstElement().String()); - const auto chunkRange = uassertStatusOK(ChunkRange::fromBSON(cmdObj)); + const MigrationSessionId migrationSessionId( + uassertStatusOK(MigrationSessionId::extractFromBSON(cmdObj))); + + // Ensure this shard is not currently receiving or donating any chunks. + auto scopedRegisterReceiveChunk( + uassertStatusOK(shardingState->registerReceiveChunk(nss, chunkRange, fromShard))); + // Refresh our collection manager from the config server, we need a collection manager to // start registering pending chunks. We force the remote refresh here to make the behavior // consistent and predictable, generally we'd refresh anyway, and to be paranoid. @@ -147,13 +152,6 @@ public: return false; } - const MigrationSessionId migrationSessionId( - uassertStatusOK(MigrationSessionId::extractFromBSON(cmdObj))); - - // Ensure this shard is not currently receiving or donating any chunks. - auto scopedRegisterReceiveChunk( - uassertStatusOK(shardingState->registerReceiveChunk(nss, chunkRange, fromShard))); - // Even if this shard is not currently donating any chunks, it may still have pending // deletes from a previous migration, particularly if there are still open cursors on the // range pending deletion. diff --git a/src/mongo/db/s/migration_destination_manager_test.cpp b/src/mongo/db/s/migration_destination_manager_test.cpp index f1ce14d3147..b9daad4ad9f 100644 --- a/src/mongo/db/s/migration_destination_manager_test.cpp +++ b/src/mongo/db/s/migration_destination_manager_test.cpp @@ -84,9 +84,9 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsFromDonorWorksCorrectly) { std::vector<BSONObj> resultDocs; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) { - while (docs.more()) { - resultDocs.push_back(docs.next().Obj().getOwned()); + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) { + for (auto&& docToClone : docs) { + resultDocs.push_back(docToClone.Obj().getOwned()); } }; @@ -122,7 +122,7 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsThrowsFetchErrors) { return fetchBatchResultBuilder.obj(); }; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) {}; + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) {}; ASSERT_THROWS_CODE_AND_WHAT(MigrationDestinationManager::cloneDocumentsFromDonor( operationContext(), insertBatchFn, fetchBatchFn), @@ -140,7 +140,7 @@ TEST_F(MigrationDestinationManagerTest, CloneDocumentsCatchesInsertErrors) { return fetchBatchResultBuilder.obj(); }; - auto insertBatchFn = [&](OperationContext* opCtx, BSONObjIterator docs) { + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj docs) { uasserted(ErrorCodes::FailedToParse, "insertion error"); }; diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 035d21cbb82..6d522bcbd26 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -393,17 +393,40 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn << redact(status)}); } - // Do a best effort attempt to incrementally refresh the metadata. If this fails, just clear it - // up so that subsequent requests will try to do a full refresh. - ChunkVersion unusedShardVersion; - Status refreshStatus = - ShardingState::get(txn)->refreshMetadataNow(txn, getNss(), &unusedShardVersion); + // Because the CatalogCache's WithRefresh methods (on which forceShardFilteringMetadataRefresh + // depends) are not causally consistent, we need to perform up to two refresh rounds if refresh + // returns that the shard still owns the chunk + ChunkVersion collectionVersionAfterRefresh; + + for (int retriesLeft = 1;; --retriesLeft) { + ChunkVersion unusedShardVersion; + Status refreshStatus = + ShardingState::get(txn)->refreshMetadataNow(txn, getNss(), &unusedShardVersion); + + // If the refresh fails, there is no way to confirm whether the migration commit actually + // went through or not. Because of that, the collection's metadata is reset to UNSHARDED so + // that subsequent versioned requests will get StaleShardVersion and will retry the refresh. + if (!refreshStatus.isOK()) { + ScopedTransaction scopedXact(txn, MODE_IX); + AutoGetCollection autoColl(txn, getNss(), MODE_IX, MODE_X); + + CollectionShardingState::get(txn, getNss())->refreshMetadata(txn, nullptr); + + log() + << "Failed to refresh metadata after a failed commit attempt. Metadata was cleared " + "so it will get a full refresh when accessed again" + << causedBy(redact(refreshStatus)); - if (refreshStatus.isOK()) { - ScopedTransaction scopedXact(txn, MODE_IS); - AutoGetCollection autoColl(txn, getNss(), MODE_IS); + return {migrationCommitStatus.code(), + str::stream() << "Failed to refresh metadata after migration commit due to " + << refreshStatus.toString()}; + } - auto refreshedMetadata = CollectionShardingState::get(txn, getNss())->getMetadata(); + auto refreshedMetadata = [&] { + ScopedTransaction scopedXact(txn, MODE_IS); + AutoGetCollection autoColl(txn, getNss(), MODE_IS); + return CollectionShardingState::get(txn, getNss())->getMetadata(); + }(); if (!refreshedMetadata) { return {ErrorCodes::NamespaceNotSharded, @@ -412,32 +435,43 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn << migrationCommitStatus.toString()}; } - if (refreshedMetadata->keyBelongsToMe(_args.getMinKey())) { - // The chunk modification was not applied, so report the original error - return {migrationCommitStatus.code(), - str::stream() << "Chunk move was not successful due to " - << migrationCommitStatus.reason()}; + // If after a successful refresh the metadata indicates that the node still owns the chunk, + // we must do one more refresh in order to ensure that the previous refresh round didn't + // join an already active catalog cache refresh and missed its own commit + if (!refreshedMetadata->keyBelongsToMe(_args.getMinKey())) { + collectionVersionAfterRefresh = refreshedMetadata->getCollVersion(); + break; } - // Migration succeeded - log() << "Migration succeeded and updated collection version to " - << refreshedMetadata->getCollVersion(); - } else { - ScopedTransaction scopedXact(txn, MODE_IX); - AutoGetCollection autoColl(txn, getNss(), MODE_IX, MODE_X); - - CollectionShardingState::get(txn, getNss())->refreshMetadata(txn, nullptr); + if (retriesLeft) + continue; - log() << "Failed to refresh metadata after a failed commit attempt. Metadata was cleared " - "so it will get a full refresh when accessed again" - << causedBy(redact(refreshStatus)); + // This condition may only happen if the migration commit has failed for any reason + if (migrationCommitStatus.isOK()) { + severe() << "The migration commit succeeded, but the new chunk placement was not " + "reflected after metadata refresh, which is an indication of an " + "afterOpTime bug."; + severe() << "The current config server opTime is " << grid.configOpTime(); + severe() << "The commit response contained:"; + severe() << " metadata: " + << redact(commitChunkMigrationResponse.getValue().metadata.toString()); + severe() << " response: " + << redact(commitChunkMigrationResponse.getValue().response.toString()); + + fassertFailed(50878); + } - // We don't know whether migration succeeded or failed return {migrationCommitStatus.code(), - str::stream() << "Failed to refresh metadata after migration commit due to " - << refreshStatus.toString()}; + str::stream() << "Chunk move was not successful due to " + << migrationCommitStatus.reason()}; } + invariant(collectionVersionAfterRefresh.isSet()); + + // Migration succeeded + log() << "Migration succeeded and updated collection version to " + << collectionVersionAfterRefresh; + MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeLeavingCriticalSection); scopedGuard.Dismiss(); diff --git a/src/mongo/db/s/operation_sharding_state.cpp b/src/mongo/db/s/operation_sharding_state.cpp index 0f92bbd5492..2f0911bc9ac 100644 --- a/src/mongo/db/s/operation_sharding_state.cpp +++ b/src/mongo/db/s/operation_sharding_state.cpp @@ -40,7 +40,7 @@ const OperationContext::Decoration<OperationShardingState> shardingMetadataDecor OperationContext::declareDecoration<OperationShardingState>(); // Max time to wait for the migration critical section to complete -const Microseconds kMaxWaitForMigrationCriticalSection = Minutes(5); +const Milliseconds kMaxWaitForMigrationCriticalSection = Minutes(5); } // namespace @@ -109,7 +109,7 @@ bool OperationShardingState::waitForMigrationCriticalSectionSignal(OperationCont _migrationCriticalSectionSignal->waitFor( txn, txn->hasDeadline() - ? std::min(txn->getRemainingMaxTimeMicros(), kMaxWaitForMigrationCriticalSection) + ? std::min(txn->getRemainingMaxTimeMillis(), kMaxWaitForMigrationCriticalSection) : kMaxWaitForMigrationCriticalSection); _migrationCriticalSectionSignal = nullptr; return true; diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.cpp b/src/mongo/db/storage/bson_collection_catalog_entry.cpp index 7837e898b56..f0e5589a946 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/bson_collection_catalog_entry.cpp @@ -143,6 +143,16 @@ void BSONCollectionCatalogEntry::getAllIndexes(OperationContext* txn, } } +void BSONCollectionCatalogEntry::getReadyIndexes(OperationContext* txn, + std::vector<std::string>* names) const { + MetaData md = _getMetaData(txn); + + for (unsigned i = 0; i < md.indexes.size(); i++) { + if (md.indexes[i].ready) + names->push_back(md.indexes[i].spec["name"].String()); + } +} + bool BSONCollectionCatalogEntry::isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const { diff --git a/src/mongo/db/storage/bson_collection_catalog_entry.h b/src/mongo/db/storage/bson_collection_catalog_entry.h index 83c2238fc17..c42908f5f8c 100644 --- a/src/mongo/db/storage/bson_collection_catalog_entry.h +++ b/src/mongo/db/storage/bson_collection_catalog_entry.h @@ -58,6 +58,8 @@ public: virtual void getAllIndexes(OperationContext* txn, std::vector<std::string>* names) const; + virtual void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const; + virtual bool isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const; diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index a2421962853..abf43946697 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -106,6 +106,19 @@ void NamespaceDetailsCollectionCatalogEntry::getAllIndexes(OperationContext* txn } } +void NamespaceDetailsCollectionCatalogEntry::getReadyIndexes( + OperationContext* txn, std::vector<std::string>* names) const { + NamespaceDetails::IndexIterator i = _details->ii(true); + while (i.more()) { + const IndexDetails& id = i.next(); + const BSONObj obj = _indexRecordStore->dataFor(txn, id.info.toRecordId()).toBson(); + const char* idxName = obj.getStringField("name"); + if (isIndexReady(txn, StringData(idxName))) { + names->push_back(idxName); + } + } +} + bool NamespaceDetailsCollectionCatalogEntry::isIndexMultikey(OperationContext* txn, StringData idxName, MultikeyPaths* multikeyPaths) const { diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h index 0f8940be756..8d57825d0c3 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h @@ -67,6 +67,8 @@ public: BSONObj getIndexSpec(OperationContext* txn, StringData idxName) const final; + void getReadyIndexes(OperationContext* txn, std::vector<std::string>* names) const final; + bool isIndexMultikey(OperationContext* txn, StringData indexName, MultikeyPaths* multikeyPaths) const final; diff --git a/src/mongo/db/update_index_data.cpp b/src/mongo/db/update_index_data.cpp index 2b1144d40a3..ade76f0c50d 100644 --- a/src/mongo/db/update_index_data.cpp +++ b/src/mongo/db/update_index_data.cpp @@ -149,12 +149,35 @@ bool getCanonicalIndexField(StringData fullName, string* out) { while (j + 1 < fullName.size() && isdigit(fullName[j + 1])) j++; - if (j + 1 == fullName.size() || fullName[j + 1] == '.') { + if (j + 1 == fullName.size()) { // only digits found, skip forward i = j; modified = true; continue; } + + // Check for consecutive digits separated by a period. + if (fullName[j + 1] == '.') { + // Peek ahead to see if the next set of characters are also numeric. + size_t k = j + 2; + while (k < fullName.size() && isdigit(fullName[k])) { + k++; + } + + // The second set of digits may end at the end of the path or a '.'. + if (k == fullName.size() || fullName[k] == '.') { + // Found consecutive numerical path components. Since this implies a numeric + // field name, return the prefix as the canonical index field. This is meant to + // fix SERVER-37058. + modified = true; + break; + } + + // Only one numerical path component, skip forward. + i = j; + modified = true; + continue; + } } buf << c; diff --git a/src/mongo/db/update_index_data_test.cpp b/src/mongo/db/update_index_data_test.cpp index 167cb632ab1..f01f495ed1f 100644 --- a/src/mongo/db/update_index_data_test.cpp +++ b/src/mongo/db/update_index_data_test.cpp @@ -126,4 +126,26 @@ TEST(UpdateIndexDataTest, getCanonicalIndexField1) { ASSERT_FALSE(getCanonicalIndexField("a.", &x)); } + +TEST(UpdateIndexDataTest, CanonicalIndexFieldForConsecutiveDigits) { + std::string indexField; + + ASSERT_TRUE(getCanonicalIndexField("a.0.0", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.55.01", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.0.b.1", &indexField)); + ASSERT_EQ(indexField, "a"); + + ASSERT_TRUE(getCanonicalIndexField("a.0b.1", &indexField)); + ASSERT_EQ(indexField, "a.0b"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.b.1.2", &indexField)); + ASSERT_EQ(indexField, "a.b"); + + ASSERT_TRUE(getCanonicalIndexField("a.0.11b", &indexField)); + ASSERT_EQ(indexField, "a.11b"); +} } diff --git a/src/mongo/executor/network_interface_asio_auth.cpp b/src/mongo/executor/network_interface_asio_auth.cpp index 571a1be1f81..f5f9ec19b68 100644 --- a/src/mongo/executor/network_interface_asio_auth.cpp +++ b/src/mongo/executor/network_interface_asio_auth.cpp @@ -184,7 +184,7 @@ void NetworkInterfaceASIO::_authenticate(AsyncOp* op) { std::string clientName; #ifdef MONGO_CONFIG_SSL if (getSSLManager()) { - clientName = getSSLManager()->getSSLConfiguration().clientSubjectName; + clientName = getSSLManager()->getSSLConfiguration().clientSubjectName.toString(); } #endif diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml index ee3c741a010..44c64b6870d 100644 --- a/src/mongo/gotools/common.yml +++ b/src/mongo/gotools/common.yml @@ -1429,21 +1429,21 @@ buildvariants: ####################################### - name: amazonlinux64 - display_name: Amazon Linux 64 (Go 1.8) + display_name: Amazon Linux 64 (Go 1.10) run_on: - linux-64-amzn-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist - name: amazon2 - display_name: Amazon Linux 64 v2 (Go 1.8) + display_name: Amazon Linux 64 v2 (Go 1.10) run_on: - amazon2-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1463,11 +1463,11 @@ buildvariants: - name: dist - name: debian81 - display_name: Debian 8.1 (Go 1.8) + display_name: Debian 8.1 (Go 1.10) run_on: - debian81-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1477,7 +1477,7 @@ buildvariants: ####################################### - name: macOS-1012 - display_name: MacOS 10.12 (Go 1.8) + display_name: MacOS 10.12 (Go 1.10) run_on: - macos-1012 expansions: @@ -1486,11 +1486,11 @@ buildvariants: mongo_os: "osx" arch: "osx/x86_64" excludes: requires_many_files - gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' tasks: *macos_1012_tasks - name: macOS-1012-ssl - display_name: MacOS 10.12 SSL (Go 1.8) + display_name: MacOS 10.12 SSL (Go 1.10) run_on: - macos-1012 expansions: @@ -1501,7 +1501,7 @@ buildvariants: arch: "osx/x86_64" build_tags: "ssl openssl_pre_1.0" excludes: requires_many_files - gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' tasks: *macos_1012_ssl_tasks ####################################### @@ -1509,21 +1509,21 @@ buildvariants: ####################################### - name: rhel62 - display_name: RHEL 6.2 (Go 1.8) + display_name: RHEL 6.2 (Go 1.10) run_on: - rhel62-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist - name: rhel70 - display_name: RHEL 7.0 (Go 1.8) + display_name: RHEL 7.0 (Go 1.10) run_on: - rhel70 expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1533,11 +1533,11 @@ buildvariants: ####################################### - name: suse12 - display_name: SUSE 12 (Go 1.8) + display_name: SUSE 12 (Go 1.10) run_on: - suse12-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1547,7 +1547,7 @@ buildvariants: ####################################### - name: ubuntu1404 - display_name: Ubuntu 14.04 (Go 1.8) + display_name: Ubuntu 14.04 (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1555,15 +1555,15 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "targeted" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - build_tags: "ssl" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' + build_tags: "sasl ssl" arch: "linux/x86_64" integration_test_args: integration resmoke_args: --jobs $(grep -c ^processor /proc/cpuinfo) tasks: *ubuntu1404_tasks - name: ubuntu1404-ssl - display_name: Ubuntu 14.04 SSL (Go 1.8) + display_name: Ubuntu 14.04 SSL (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1571,7 +1571,7 @@ buildvariants: <<: *mongo_ssl_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" edition: ssl arch: "linux/x86_64" @@ -1590,7 +1590,7 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "ssl sasl" smoke_use_ssl: --use-ssl resmoke_use_ssl: _ssl @@ -1602,11 +1602,11 @@ buildvariants: tasks: *ubuntu1404_enterprise_tasks - name: ubuntu1604 - display_name: Ubuntu 16.04 (Go 1.8) + display_name: Ubuntu 16.04 (Go 1.10) run_on: - ubuntu1604-test expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' build_tags: "sasl ssl" tasks: - name: dist @@ -1616,7 +1616,7 @@ buildvariants: ####################################### - name: windows-64 - display_name: Windows 64-bit (Go 1.8) + display_name: Windows 64-bit (Go 1.10) run_on: - windows-64-vs2013-test expansions: @@ -1629,11 +1629,11 @@ buildvariants: arch: "win32/x86_64" preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' tasks: *windows_64_tasks - name: windows-64-ssl - display_name: Windows 64-bit SSL (Go 1.8) + display_name: Windows 64-bit SSL (Go 1.10) run_on: - windows-64-vs2013-compile expansions: @@ -1649,13 +1649,13 @@ buildvariants: multiversion_override: "2.6" extension: .exe arch: "win32/x86_64" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration,ssl" tasks: *windows_64_ssl_tasks - name: windows-64-enterprise - display_name: Windows 64-bit Enterprise (Go 1.8) + display_name: Windows 64-bit Enterprise (Go 1.10) run_on: - windows-64-vs2013-compile expansions: @@ -1672,7 +1672,7 @@ buildvariants: edition: enterprise extension: .exe arch: "win32/x86_64" - gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + gorootvars: 'PATH="/cygdrive/c/golang/go1.10/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/golang/go1.10"' preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' " integration_test_args: "integration" tasks: *windows_64_enterprise_tasks @@ -1682,7 +1682,7 @@ buildvariants: ####################################### - name: ubuntu1604-arm64 - display_name: ZAP ARM64 Ubuntu 16.04 SSL (gccgo 1.4) + display_name: ZAP ARM64 Ubuntu 16.04 SSL (Go 1.10) run_on: - ubuntu1604-arm64-small stepback: false @@ -1693,10 +1693,9 @@ buildvariants: mongo_os: "ubuntu1604" mongo_edition: "targeted" mongo_arch: "arm64" - args: -gccgoflags "$(pkg-config --libs --cflags libcrypto libssl)" build_tags: "ssl" resmoke_use_ssl: _ssl - gorootvars: PATH="/opt/mongodbtoolchain/v2/bin/:$PATH" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/aarch64-mongodb-linux-gcc' excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" @@ -1710,7 +1709,7 @@ buildvariants: ####################################### - name: rhel71-ppc64le-enterprise - display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.8) + display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.10) run_on: - rhel71-power8-test stepback: false @@ -1725,7 +1724,7 @@ buildvariants: #args: ... libsasl2; build_tags "sasl ssl" build_tags: 'ssl' resmoke_use_ssl: _ssl - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' resmoke_args: -j 4 excludes: requires_mmap_available,requires_large_ram,requires_mongo_24,requires_mongo_26,requires_mongo_30 multiversion_override: "skip" @@ -1736,14 +1735,14 @@ buildvariants: tasks: *rhel71_enterprise_tasks - name: ubuntu1604-ppc64le-enterprise - display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.8) + display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.10) run_on: - ubuntu1604-power8-test stepback: false batchtime: 10080 # weekly expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' - build_tags: 'ssl' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: 'ssl sasl' tasks: - name: dist @@ -1752,7 +1751,7 @@ buildvariants: ####################################### - name: rhel67-s390x-enterprise - display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.8) + display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.10) run_on: - rhel67-zseries-test stepback: false @@ -1777,7 +1776,7 @@ buildvariants: mongo_arch: "s390x" build_tags: "sasl ssl" resmoke_use_ssl: _ssl - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' excludes: requires_mmap_available,requires_mongo_24,requires_mongo_26,requires_mongo_30 resmoke_args: -j 2 multiversion_override: "skip" @@ -1800,13 +1799,13 @@ buildvariants: - name: dist - name: ubuntu1604-s390x-enterprise - display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.8) + display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.10) run_on: - ubuntu1604-zseries-small stepback: false batchtime: 10080 # weekly expansions: - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10 CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' build_tags: "sasl ssl" tasks: - name: dist @@ -1818,7 +1817,7 @@ buildvariants: - name: ubuntu-race stepback: false batchtime: 1440 # daily - display_name: z Race Detector Ubuntu 14.04 (Go 1.8) + display_name: z Race Detector Ubuntu 14.04 (Go 1.10) run_on: - ubuntu1404-test expansions: @@ -1826,8 +1825,8 @@ buildvariants: <<: *mongo_default_startup_args mongo_os: "ubuntu1404" mongo_edition: "enterprise" - gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' - build_tags: "ssl" + gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' + build_tags: "sasl ssl" arch: "linux/x86_64" args: "-race" excludes: requires_large_ram diff --git a/src/mongo/gotools/import.data b/src/mongo/gotools/import.data index d1d91264483..4b4c75c79f0 100644 --- a/src/mongo/gotools/import.data +++ b/src/mongo/gotools/import.data @@ -1,5 +1,5 @@ { - "commit": "4c5314b404c2d7aac7ceb50133faa3ac4fc3d2ea", + "commit": "38376e791d2c264b377ba3115344c860c146e0b2", "github": "mongodb/mongo-tools.git", "vendor": "tools", "branch": "v3.4" diff --git a/src/mongo/gotools/mongoreplay/stat_format.go b/src/mongo/gotools/mongoreplay/stat_format.go index ad64ef7bd95..613087fbc43 100644 --- a/src/mongo/gotools/mongoreplay/stat_format.go +++ b/src/mongo/gotools/mongoreplay/stat_format.go @@ -28,10 +28,10 @@ type OpStat struct { Ns string `json:"ns,omitempty"` // Data represents the payload of the request operation. - RequestData interface{} `json:"request_data, omitempty"` + RequestData interface{} `json:"request_data,omitempty"` // Data represents the payload of the reply operation. - ReplyData interface{} `json:"reply_data, omitempty"` + ReplyData interface{} `json:"reply_data,omitempty"` // NumReturned is the number of documents that were fetched as a result of this operation. NumReturned int `json:"nreturned,omitempty"` @@ -71,7 +71,7 @@ type OpStat struct { // RequestID is the ID of the mongodb operation as taken from the header. // The RequestID for a request operation is the same as the ResponseID for // the corresponding reply, so this field will be the same for request/reply pairs. - RequestID int32 `json:"request_id, omitempty"` + RequestID int32 `json:"request_id,omitempty"` } // jsonGet retrieves serialized json req/res via the channel-like arg; diff --git a/src/mongo/gotools/mongorestore/oplog.go b/src/mongo/gotools/mongorestore/oplog.go index 68e24d20b8f..010f16f410e 100644 --- a/src/mongo/gotools/mongorestore/oplog.go +++ b/src/mongo/gotools/mongorestore/oplog.go @@ -93,6 +93,9 @@ func (restore *MongoRestore) RestoreOplog() error { } log.Logvf(log.Info, "applied %v ops", totalOps) + if err := bsonSource.Err(); err != nil { + return fmt.Errorf("error reading oplog bson input: %v", err) + } return nil } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml new file mode 100644 index 00000000000..b8bbabba9a9 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.evergreen/config.yml @@ -0,0 +1,377 @@ +# default command type +command_type: system + +# run the same task in the previous revision if the current task fails +stepback: true + +functions: + + "set shell vars": + - command: shell.exec + params: + script: | + set -o errexit + set -o xtrace + export RAWGOPATH="$(pwd)/gopath" + export GOPATH="$RAWGOPATH" + if [ "Windows_NT" = "$OS" ]; then + set -o igncr + export GOPATH=$(echo $GOPATH | sed -e 's|/cygdrive/c|c:|') + fi + cat <<EOT > expansion.yml + rawgopath: $RAWGOPATH + repopath: $RAWGOPATH/src/github.com/10gen/openssl + prepare_shell: | + export GOPATH="$GOPATH" + set -o errexit + set -o xtrace + EOT + cat expansion.yml + exit 0 + - command: expansions.update + params: + file: expansion.yml + + "setup gopath" : + - command: shell.exec + params: + silent: false + script: | + ${prepare_shell} + ${gorootvars} go get github.com/spacemonkeygo/spacelog + exit 0 + + "fetch source" : + - command: git.get_project + params: + directory: src + - command: shell.exec + params: + script: | + ${prepare_shell} + mkdir -p $(dirname "${repopath}") + mv src "${repopath}" + exit 0 + + "go build" : + - command: shell.exec + type: test + params: + script: | + ${prepare_shell} + cd ${repopath} + ${gorootvars} go build ${args} -v -x -tags '${build_tags}' + exit 0 + + "go test" : + - command: shell.exec + type: test + params: + script: | + ${prepare_shell} + cd ${repopath} + ${gorootvars} go test ${args} -v -x -tags '${build_tags}' + exit 0 + +post: + - command: shell.exec + params: + silent: true + script: | + ${prepare_shell} + rm -rf "${rawgopath}" + exit 0 + +tasks: + +- name: "build" + commands: + - func: "set shell vars" + - func: "setup gopath" + - func: "fetch source" + - func: "go build" + +- name: "test" + depends_on: + - name: "build" + commands: + - func: "set shell vars" + - func: "setup gopath" + - func: "fetch source" + - func: "go test" + +buildvariants: + +####################################### +# Amazon Buildvariants # +####################################### + +- name: amazonlinux64 + display_name: Amazon Linux 64 (Go 1.8) + run_on: + - linux-64-amzn-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: amazon2 + display_name: Amazon Linux 64 v2 (Go 1.8) + run_on: + - amazon2-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Debian Buildvariants # +####################################### + +- name: debian71 + display_name: Debian 7.1 (Go 1.8) + run_on: + - debian71-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: debian81 + display_name: Debian 8.1 (Go 1.8) + run_on: + - debian81-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: debian92 + display_name: Debian 9.2 (Go 1.8) + run_on: + - debian92-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# macOS Buildvariant # +####################################### + +- name: macOS-1012 + display_name: MacOS 10.12 (Go 1.8) + run_on: + - macos-1012 + expansions: + gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.10 CGO_LDFLAGS=-mmacosx-version-min=10.10' + build_tags: "openssl_pre_1.0" + tasks: + - name: build + - name: test + +####################################### +# RHEL Buildvariants # +####################################### + +- name: rhel62 + display_name: RHEL 6.2 (Go 1.8) + run_on: + - rhel62-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: rhel70 + display_name: RHEL 7.0 (Go 1.8) + run_on: + - rhel70 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# SUSE Buildvariants # +####################################### + +- name: suse11 + display_name: SUSE 11 (Go 1.8) + run_on: + - suse11-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "openssl_pre_1.0" + tasks: + - name: build + - name: test + +- name: suse12 + display_name: SUSE 12 (Go 1.8) + run_on: + - suse12-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Ubuntu Buildvariants # +####################################### + +- name: ubuntu1404 + display_name: Ubuntu 14.04 (Go 1.8) + run_on: + - ubuntu1404-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604 + display_name: Ubuntu 16.04 (Go 1.8) + run_on: + - ubuntu1604-test + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Windows Buildvariants # +####################################### + +- name: windows-64 + display_name: Windows 64-bit (Go 1.8) + run_on: + - windows-64-vs2015-test + expansions: + gorootvars: 'PATH="/cygdrive/c/go1.8/go/bin:/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:$PATH" GOROOT="c:/go1.8/go"' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# ARM Buildvariants # +####################################### + +- name: ubuntu1604-arm64-go1.8 + display_name: ZAP ARM64 Ubuntu 16.04 SSL (Go 1.8) + run_on: + - ubuntu1604-arm64-small + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/aarch64-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Power Buildvariants # +####################################### + +- name: rhel71-ppc64le-enterprise-go1.8 + display_name: ZAP PPC64LE RHEL 7.1 Enterprise (Go 1.8) + run_on: + - rhel71-power8-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604-ppc64le-enterprise-go1.8 + display_name: ZAP PPC64LE Ubuntu 16.04 Enterprise (Go 1.8) + run_on: + - ubuntu1604-power8-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/ppc64le-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +####################################### +# Z (s390x) Buildvariants # +####################################### + +- name: rhel67-s390x-enterprise-go1.8 + display_name: ZAP s390x RHEL 6.7 Enterprise (Go 1.8) + run_on: + - rhel67-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: rhel72-s390x-enterprise-go1.8 + display_name: ZAP s390x RHEL 7.2 Enterprise (Go 1.8) + run_on: + - rhel72-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: suse12-s390x-enterprise-go1.8 + display_name: ZAP s390x SUSE 12 Enterprise (Go 1.8) + run_on: + - suse12-zseries-test + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test + +- name: ubuntu1604-s390x-enterprise-go1.8 + display_name: ZAP s390x Ubuntu 16.04 Enterprise (Go 1.8) + run_on: + - ubuntu1604-zseries-small + stepback: false + batchtime: 604800 + expansions: + gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go CC=/opt/mongodbtoolchain/v2/bin/s390x-mongodb-linux-gcc' + build_tags: "" + tasks: + - name: build + - name: test diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore new file mode 100644 index 00000000000..805d350b7e5 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/.gitignore @@ -0,0 +1 @@ +openssl.test diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS new file mode 100644 index 00000000000..bc88546999e --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/AUTHORS @@ -0,0 +1,23 @@ +Andrew Brampton <github@bramp.net> +Anton Baklanov <antonbaklanov@gmail.com> +Carlos MartÃn Nieto <cmn@dwim.me> +Charles Strahan <charles@cstrahan.com> +Christopher Dudley <chris@github.chrisdudley.xyz> +Christopher Fredericks <cfredmakecode@gmail.com> +Colin Misare +dequis <dx@dxzone.com.ar> +Gabriel Russell <gabriel.russell@mongodb.com> +Giulio <programmatore@ditieri.it> +Jakob Unterwurzacher <jakobunt@gmail.com> +Juuso Haavisto <juuso@mail.com> +kujenga <ataylor0123@gmail.com> +MongoDB, Inc. +Phus Lu <phuslu@hotmail.com> +Russ Egan <russ@safemonk.com> +Ryan Hileman <lunixbochs@gmail.com> +Scott J. Goldman <scottjg@github.com> +Scott Kidder <skidder@brightcove.com> +Space Monkey, Inc <hello@spacemonkey.com> +Stephen Gallagher <sgallagh@redhat.com> +Viacheslav Biriukov <v.v.biriukov@gmail.com> +Zack Owens <zowens2009@gmail.com> diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md index 6bd3383a0e8..2785366f5e1 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/README.md @@ -4,7 +4,7 @@ Please see http://godoc.org/github.com/spacemonkeygo/openssl for more info ### License -Copyright (C) 2014 Space Monkey, Inc. +Copyright (C) 2017. See AUTHORS. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,9 +18,33 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -### Using on Windows -1. Install [mingw-w64](http://mingw-w64.sourceforge.net/) -2. Install [pkg-config-lite](http://sourceforge.net/projects/pkgconfiglite) -3. Build (or install precompiled) openssl for mingw32-w64 -4. Set __PKG\_CONFIG\_PATH__ to the directory containing openssl.pc - (i.e. c:\mingw64\mingw64\lib\pkgconfig) +### Installing on a Unix-ish system with pkg-config + +1. (If necessary) install the openssl C library with a package manager + that provides an openssl.pc file OR install openssl manually and create + an openssl.pc file. + +2. Ensure that `pkg-config --cflags --libs openssl` finds your openssl + library. If it doesn't, try setting `PKG_CONFIG_PATH` to the directory + containing your openssl.pc file. E.g. for darwin: with MacPorts, + `PKG_CONFIG_PATH=/opt/local/lib/pkgconfig` or for Homebrew, + `PKG_CONFIG_PATH=/usr/local/Cellar/openssl/1.0.2l/lib/pkgconfig` + +### Installing on a Unix-ish system without pkg-config + +1. (If necessary) install the openssl C library in your customary way + +2. Set the `CGO_CPP_FLAGS`, `CGO_CFLAGS` and `CGO_LDFLAGS` as necessary to + provide `-I`, `-L` and other options to the compiler. E.g. on darwin, + MongoDB's darwin build servers use the native libssl, but provide the + missing headers in a custom directory, so it the build hosts set + `CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include` + +### Installing on Windows + +1. Install [mingw-w64](http://mingw-w64.sourceforge.net/) and add it to + your `PATH` + +2. Install the C openssl into `C:\openssl`. (Unfortunately, this is still + hard-coded.) You should have directories like `C:\openssl\include` and + `C:\openssl\bin`. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go index 8d0da8998eb..9fe32aa8032 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/bio.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,56 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <string.h> -#include <openssl/bio.h> - -extern int cbioNew(BIO *b); -static int cbioFree(BIO *b) { - return 1; -} - -extern int writeBioWrite(BIO *b, char *buf, int size); -extern long writeBioCtrl(BIO *b, int cmd, long arg1, void *arg2); -static int writeBioPuts(BIO *b, const char *str) { - return writeBioWrite(b, (char*)str, (int)strlen(str)); -} - -extern int readBioRead(BIO *b, char *buf, int size); -extern long readBioCtrl(BIO *b, int cmd, long arg1, void *arg2); - -static BIO_METHOD writeBioMethod = { - BIO_TYPE_SOURCE_SINK, - "Go Write BIO", - (int (*)(BIO *, const char *, int))writeBioWrite, - NULL, - writeBioPuts, - NULL, - writeBioCtrl, - cbioNew, - cbioFree, - NULL}; - -static BIO_METHOD* BIO_s_writeBio() { return &writeBioMethod; } - -static BIO_METHOD readBioMethod = { - BIO_TYPE_SOURCE_SINK, - "Go Read BIO", - NULL, - readBioRead, - NULL, - NULL, - readBioCtrl, - cbioNew, - cbioFree, - NULL}; - -static BIO_METHOD* BIO_s_readBio() { return &readBioMethod; } -*/ +// #include "shim.h" import "C" import ( @@ -89,16 +42,6 @@ func nonCopyCString(data *C.char, size C.int) []byte { return nonCopyGoBytes(uintptr(unsafe.Pointer(data)), int(size)) } -//export cbioNew -func cbioNew(b *C.BIO) C.int { - b.shutdown = 1 - b.init = 1 - b.num = -1 - b.ptr = nil - b.flags = 0 - return 1 -} - var writeBioMapping = newMapping() type writeBio struct { @@ -109,21 +52,20 @@ type writeBio struct { } func loadWritePtr(b *C.BIO) *writeBio { - return (*writeBio)(writeBioMapping.Get(token(b.ptr))) + t := token(C.X_BIO_get_data(b)) + return (*writeBio)(writeBioMapping.Get(t)) } func bioClearRetryFlags(b *C.BIO) { - // from BIO_clear_retry_flags and BIO_clear_flags - b.flags &= ^(C.BIO_FLAGS_RWS | C.BIO_FLAGS_SHOULD_RETRY) + C.X_BIO_clear_flags(b, C.BIO_FLAGS_RWS|C.BIO_FLAGS_SHOULD_RETRY) } func bioSetRetryRead(b *C.BIO) { - // from BIO_set_retry_read and BIO_set_flags - b.flags |= (C.BIO_FLAGS_READ | C.BIO_FLAGS_SHOULD_RETRY) + C.X_BIO_set_flags(b, C.BIO_FLAGS_READ|C.BIO_FLAGS_SHOULD_RETRY) } -//export writeBioWrite -func writeBioWrite(b *C.BIO, data *C.char, size C.int) (rc C.int) { +//export go_write_bio_write +func go_write_bio_write(b *C.BIO, data *C.char, size C.int) (rc C.int) { defer func() { if err := recover(); err != nil { logger.Critf("openssl: writeBioWrite panic'd: %v", err) @@ -141,8 +83,8 @@ func writeBioWrite(b *C.BIO, data *C.char, size C.int) (rc C.int) { return size } -//export writeBioCtrl -func writeBioCtrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( +//export go_write_bio_ctrl +func go_write_bio_ctrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( rc C.long) { defer func() { if err := recover(); err != nil { @@ -197,15 +139,15 @@ func (b *writeBio) WriteTo(w io.Writer) (rv int64, err error) { func (self *writeBio) Disconnect(b *C.BIO) { if loadWritePtr(b) == self { - writeBioMapping.Del(token(b.ptr)) - b.ptr = nil + writeBioMapping.Del(token(C.X_BIO_get_data(b))) + C.X_BIO_set_data(b, nil) } } func (b *writeBio) MakeCBIO() *C.BIO { - rv := C.BIO_new(C.BIO_s_writeBio()) + rv := C.X_BIO_new_write_bio() token := writeBioMapping.Add(unsafe.Pointer(b)) - rv.ptr = unsafe.Pointer(token) + C.X_BIO_set_data(rv, unsafe.Pointer(token)) return rv } @@ -220,14 +162,14 @@ type readBio struct { } func loadReadPtr(b *C.BIO) *readBio { - return (*readBio)(readBioMapping.Get(token(b.ptr))) + return (*readBio)(readBioMapping.Get(token(C.X_BIO_get_data(b)))) } -//export readBioRead -func readBioRead(b *C.BIO, data *C.char, size C.int) (rc C.int) { +//export go_read_bio_read +func go_read_bio_read(b *C.BIO, data *C.char, size C.int) (rc C.int) { defer func() { if err := recover(); err != nil { - logger.Critf("openssl: readBioRead panic'd: %v", err) + logger.Critf("openssl: go_read_bio_read panic'd: %v", err) rc = -1 } }() @@ -256,8 +198,8 @@ func readBioRead(b *C.BIO, data *C.char, size C.int) (rc C.int) { return C.int(n) } -//export readBioCtrl -func readBioCtrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( +//export go_read_bio_ctrl +func go_read_bio_ctrl(b *C.BIO, cmd C.int, arg1 C.long, arg2 unsafe.Pointer) ( rc C.long) { defer func() { @@ -316,16 +258,16 @@ func (b *readBio) ReadFromOnce(r io.Reader) (n int, err error) { } func (b *readBio) MakeCBIO() *C.BIO { - rv := C.BIO_new(C.BIO_s_readBio()) + rv := C.X_BIO_new_read_bio() token := readBioMapping.Add(unsafe.Pointer(b)) - rv.ptr = unsafe.Pointer(token) + C.X_BIO_set_data(rv, unsafe.Pointer(token)) return rv } func (self *readBio) Disconnect(b *C.BIO) { if loadReadPtr(b) == self { - readBioMapping.Del(token(b.ptr)) - b.ptr = nil + readBioMapping.Del(token(C.X_BIO_get_data(b))) + C.X_BIO_set_data(b, nil) } } @@ -343,7 +285,7 @@ func (b *anyBio) Read(buf []byte) (n int, err error) { if len(buf) == 0 { return 0, nil } - n = int(C.BIO_read((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) + n = int(C.X_BIO_read((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) if n <= 0 { return 0, io.EOF } @@ -354,7 +296,7 @@ func (b *anyBio) Write(buf []byte) (written int, err error) { if len(buf) == 0 { return 0, nil } - n := int(C.BIO_write((*C.BIO)(b), unsafe.Pointer(&buf[0]), + n := int(C.X_BIO_write((*C.BIO)(b), unsafe.Pointer(&buf[0]), C.int(len(buf)))) if n != len(buf) { return n, errors.New("BIO write failed") diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go index f71e285639a..d286163ffcb 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo +// +build !openssl_static package openssl -// #cgo linux pkg-config: openssl -// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -// #cgo windows LDFLAGS: -lcrypt32 -// #cgo darwin CFLAGS: -Wno-deprecated-declarations -I/usr/include -I/usr/local/opt/openssl/include -// #cgo darwin LDFLAGS: -L/usr/local/opt/openssl/lib -lssl -lcrypto -framework CoreFoundation -framework Foundation -framework Security +// #cgo linux darwin pkg-config: openssl +// #cgo CFLAGS: -Wno-deprecated-declarations +// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -I"c:/openssl/include" +// #cgo windows LDFLAGS: -lssleay32 -llibeay32 -lcrypt32 -L "c:/openssl/bin" +// #cgo darwin LDFLAGS: -framework CoreFoundation -framework Foundation -framework Security import "C" diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go new file mode 100644 index 00000000000..1450d52e1a9 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/build_static.go @@ -0,0 +1,24 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build openssl_static + +package openssl + +// #cgo linux windows darwin pkg-config: --static libssl libcrypto +// #cgo CFLAGS: -Wno-deprecated-declarations +// #cgo windows CFLAGS: -DWIN32_LEAN_AND_MEAN -I"c:/openssl/include" +// #cgo windows LDFLAGS: -lssleay32 -llibeay32 -lcrypt32 -L "c:/openssl/bin" +// #cgo darwin LDFLAGS: -framework CoreFoundation -framework Foundation -framework Security +import "C" diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go index 61637c649fa..d3df63507e3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/conf.h> -// #include <openssl/ssl.h> -// #include <openssl/x509v3.h> -// -// void OPENSSL_free_not_a_macro(void *ref) { OPENSSL_free(ref); } -// +// #include "shim.h" import "C" import ( @@ -229,7 +222,7 @@ func (c *Certificate) SetSerial(serial *big.Int) error { // SetIssueDate sets the certificate issue date relative to the current time. func (c *Certificate) SetIssueDate(when time.Duration) error { offset := C.long(when / time.Second) - result := C.X509_gmtime_adj(c.x.cert_info.validity.notBefore, offset) + result := C.X509_gmtime_adj(C.X_X509_get0_notBefore(c.x), offset) if result == nil { return errors.New("failed to set issue date") } @@ -239,7 +232,7 @@ func (c *Certificate) SetIssueDate(when time.Duration) error { // SetExpireDate sets the certificate issue date relative to the current time. func (c *Certificate) SetExpireDate(when time.Duration) error { offset := C.long(when / time.Second) - result := C.X509_gmtime_adj(c.x.cert_info.validity.notAfter, offset) + result := C.X509_gmtime_adj(C.X_X509_get0_notAfter(c.x), offset) if result == nil { return errors.New("failed to set expire date") } @@ -270,37 +263,41 @@ func (c *Certificate) Sign(privKey PrivateKey, digest EVP_MD) error { } func (c *Certificate) insecureSign(privKey PrivateKey, digest EVP_MD) error { - var md *C.EVP_MD + var md *C.EVP_MD = getDigestFunction(digest) + if C.X509_sign(c.x, privKey.evpPKey(), md) <= 0 { + return errors.New("failed to sign certificate") + } + return nil +} + +func getDigestFunction(digest EVP_MD) (md *C.EVP_MD) { switch digest { // please don't use these digest functions case EVP_NULL: - md = C.EVP_md_null() + md = C.X_EVP_md_null() case EVP_MD5: - md = C.EVP_md5() + md = C.X_EVP_md5() case EVP_SHA: - md = C.EVP_sha() + md = C.X_EVP_sha() case EVP_SHA1: - md = C.EVP_sha1() + md = C.X_EVP_sha1() case EVP_DSS: - md = C.EVP_dss() + md = C.X_EVP_dss() case EVP_DSS1: - md = C.EVP_dss1() + md = C.X_EVP_dss1() case EVP_RIPEMD160: - md = C.EVP_ripemd160() + md = C.X_EVP_ripemd160() case EVP_SHA224: - md = C.EVP_sha224() + md = C.X_EVP_sha224() // you actually want one of these case EVP_SHA256: - md = C.EVP_sha256() + md = C.X_EVP_sha256() case EVP_SHA384: - md = C.EVP_sha384() + md = C.X_EVP_sha384() case EVP_SHA512: - md = C.EVP_sha512() - } - if C.X509_sign(c.x, privKey.evpPKey(), md) <= 0 { - return errors.New("failed to sign certificate") + md = C.X_EVP_sha512() } - return nil + return md } // Add an extension to a certificate. @@ -388,7 +385,7 @@ func (c *Certificate) GetSerialNumberHex() (serial string) { hex := C.BN_bn2hex(bignum) serial = C.GoString(hex) C.BN_free(bignum) - C.OPENSSL_free_not_a_macro(unsafe.Pointer(hex)) + C.X_OPENSSL_free(unsafe.Pointer(hex)) return } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go index c32883ba4eb..96083260507 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/cert_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go index 12662707f54..e4f5771f8dc 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,43 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> -// -// int EVP_CIPHER_block_size_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_block_size(c); -// } -// -// int EVP_CIPHER_key_length_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_key_length(c); -// } -// -// int EVP_CIPHER_iv_length_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_iv_length(c); -// } -// -// int EVP_CIPHER_nid_not_a_macro(EVP_CIPHER *c) { -// return EVP_CIPHER_nid(c); -// } -// -// int EVP_CIPHER_CTX_block_size_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_block_size(ctx); -// } -// -// int EVP_CIPHER_CTX_key_length_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_key_length(ctx); -// } -// -// int EVP_CIPHER_CTX_iv_length_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_iv_length(ctx); -// } -// -// const EVP_CIPHER *EVP_CIPHER_CTX_cipher_not_a_macro(EVP_CIPHER_CTX *ctx) { -// return EVP_CIPHER_CTX_cipher(ctx); -// } +// #include "shim.h" import "C" import ( @@ -74,7 +40,7 @@ type Cipher struct { } func (c *Cipher) Nid() NID { - return NID(C.EVP_CIPHER_nid_not_a_macro(c.ptr)) + return NID(C.X_EVP_CIPHER_nid(c.ptr)) } func (c *Cipher) ShortName() (string, error) { @@ -82,15 +48,15 @@ func (c *Cipher) ShortName() (string, error) { } func (c *Cipher) BlockSize() int { - return int(C.EVP_CIPHER_block_size_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_block_size(c.ptr)) } func (c *Cipher) KeySize() int { - return int(C.EVP_CIPHER_key_length_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_key_length(c.ptr)) } func (c *Cipher) IVSize() int { - return int(C.EVP_CIPHER_iv_length_not_a_macro(c.ptr)) + return int(C.X_EVP_CIPHER_iv_length(c.ptr)) } func Nid2ShortName(nid NID) (string, error) { @@ -154,7 +120,7 @@ func (ctx *cipherCtx) applyKeyAndIV(key, iv []byte) error { } if kptr != nil || iptr != nil { var res C.int - if ctx.ctx.encrypt != 0 { + if C.X_EVP_CIPHER_CTX_encrypting(ctx.ctx) != 0 { res = C.EVP_EncryptInit_ex(ctx.ctx, nil, nil, kptr, iptr) } else { res = C.EVP_DecryptInit_ex(ctx.ctx, nil, nil, kptr, iptr) @@ -167,19 +133,19 @@ func (ctx *cipherCtx) applyKeyAndIV(key, iv []byte) error { } func (ctx *cipherCtx) Cipher() *Cipher { - return &Cipher{ptr: C.EVP_CIPHER_CTX_cipher_not_a_macro(ctx.ctx)} + return &Cipher{ptr: C.X_EVP_CIPHER_CTX_cipher(ctx.ctx)} } func (ctx *cipherCtx) BlockSize() int { - return int(C.EVP_CIPHER_CTX_block_size_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_block_size(ctx.ctx)) } func (ctx *cipherCtx) KeySize() int { - return int(C.EVP_CIPHER_CTX_key_length_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_key_length(ctx.ctx)) } func (ctx *cipherCtx) IVSize() int { - return int(C.EVP_CIPHER_CTX_iv_length_not_a_macro(ctx.ctx)) + return int(C.X_EVP_CIPHER_CTX_iv_length(ctx.ctx)) } func (ctx *cipherCtx) setCtrl(code, arg int) error { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go index 9f5d27ab1c3..463b30dfe55 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ciphers_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build !darwin // +build !openssl_pre_1.0 package openssl @@ -91,6 +90,10 @@ func doDecryption(key, iv, aad, ciphertext, tag []byte, blocksize, if err != nil { return nil, fmt.Errorf("Failed making GCM decryption ctx: %s", err) } + err = dctx.SetTag(tag) + if err != nil { + return nil, fmt.Errorf("Failed to set expected GCM tag: %s", err) + } aadbuf := bytes.NewBuffer(aad) for aadbuf.Len() > 0 { err = dctx.ExtraData(aadbuf.Next(bufsize)) @@ -107,10 +110,6 @@ func doDecryption(key, iv, aad, ciphertext, tag []byte, blocksize, } plainb.Write(moar) } - err = dctx.SetTag(tag) - if err != nil { - return nil, fmt.Errorf("Failed to set expected GCM tag: %s", err) - } moar, err := dctx.DecryptFinal() if err != nil { return nil, fmt.Errorf("Failed to finalize decryption: %s", err) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go index f77fb4d61b9..2d2f208489d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/conn.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,30 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <stdlib.h> -#include <openssl/ssl.h> -#include <openssl/conf.h> -#include <openssl/err.h> - -int sk_X509_num_not_a_macro(STACK_OF(X509) *sk) { return sk_X509_num(sk); } -X509 *sk_X509_value_not_a_macro(STACK_OF(X509)* sk, int i) { - return sk_X509_value(sk, i); -} -long SSL_set_tlsext_host_name_not_a_macro(SSL *ssl, const char *name) { - return SSL_set_tlsext_host_name(ssl, name); -} -const char * SSL_get_cipher_name_not_a_macro(const SSL *ssl) { - return SSL_get_cipher_name(ssl); -} -static int SSL_session_reused_not_a_macro(SSL *ssl) { - return SSL_session_reused(ssl); -} -*/ +// #include "shim.h" import "C" import ( @@ -59,8 +38,9 @@ var ( ) type Conn struct { + *SSL + conn net.Conn - ssl *C.SSL ctx *Ctx // for gc into_ssl *readBio from_ssl *writeBio @@ -156,9 +136,13 @@ func newConn(conn net.Conn, ctx *Ctx) (*Conn, error) { // the ssl object takes ownership of these objects now C.SSL_set_bio(ssl, into_ssl_cbio, from_ssl_cbio) + s := &SSL{ssl: ssl} + C.SSL_set_ex_data(s.ssl, get_ssl_idx(), unsafe.Pointer(s)) + c := &Conn{ + SSL: s, + conn: conn, - ssl: ssl, ctx: ctx, into_ssl: into_ssl, from_ssl: from_ssl} @@ -203,8 +187,10 @@ func Server(conn net.Conn, ctx *Ctx) (*Conn, error) { return c, nil } +func (c *Conn) GetCtx() *Ctx { return c.ctx } + func (c *Conn) CurrentCipher() (string, error) { - p := C.SSL_get_cipher_name_not_a_macro(c.ssl) + p := C.X_SSL_get_cipher_name(c.ssl) if p == nil { return "", errors.New("Session not established") } @@ -358,10 +344,10 @@ func (c *Conn) PeerCertificateChain() (rv []*Certificate, err error) { if sk == nil { return nil, errors.New("no peer certificates found") } - sk_num := int(C.sk_X509_num_not_a_macro(sk)) + sk_num := int(C.X_sk_X509_num(sk)) rv = make([]*Certificate, 0, sk_num) for i := 0; i < sk_num; i++ { - x := C.sk_X509_value_not_a_macro(sk, C.int(i)) + x := C.X_sk_X509_value(sk, C.int(i)) // ref holds on to the underlying connection memory so we don't need to // worry about incrementing refcounts manually or freeing the X509 rv = append(rv, &Certificate{x: x, ref: c}) @@ -578,7 +564,7 @@ func (c *Conn) SetTlsExtHostName(name string) error { defer C.free(unsafe.Pointer(cname)) runtime.LockOSThread() defer runtime.UnlockOSThread() - if C.SSL_set_tlsext_host_name_not_a_macro(c.ssl, cname) == 0 { + if C.X_SSL_set_tlsext_host_name(c.ssl, cname) == 0 { return errorFromErrorQueue() } return nil @@ -589,7 +575,7 @@ func (c *Conn) VerifyResult() VerifyResult { } func (c *Conn) SessionReused() bool { - return C.SSL_session_reused_not_a_macro(c.ssl) == 1 + return C.X_SSL_session_reused(c.ssl) == 1 } func (c *Conn) GetSession() ([]byte, error) { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go index 8daa1bbbb1f..f67a95d6ea3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,83 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* -#include <openssl/crypto.h> -#include <openssl/ssl.h> +#include "shim.h" #include <openssl/err.h> -#include <openssl/conf.h> -#include <openssl/x509.h> - -static long SSL_CTX_set_options_not_a_macro(SSL_CTX* ctx, long options) { - return SSL_CTX_set_options(ctx, options); -} - -static long SSL_CTX_clear_options_not_a_macro(SSL_CTX* ctx, long options) { - return SSL_CTX_clear_options(ctx, options); -} - -static long SSL_CTX_get_options_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_options(ctx); -} - -static long SSL_CTX_set_mode_not_a_macro(SSL_CTX* ctx, long modes) { - return SSL_CTX_set_mode(ctx, modes); -} - -static long SSL_CTX_get_mode_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_mode(ctx); -} - -static long SSL_CTX_set_session_cache_mode_not_a_macro(SSL_CTX* ctx, long modes) { - return SSL_CTX_set_session_cache_mode(ctx, modes); -} - -static long SSL_CTX_sess_set_cache_size_not_a_macro(SSL_CTX* ctx, long t) { - return SSL_CTX_sess_set_cache_size(ctx, t); -} - -static long SSL_CTX_sess_get_cache_size_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_sess_get_cache_size(ctx); -} - -static long SSL_CTX_set_timeout_not_a_macro(SSL_CTX* ctx, long t) { - return SSL_CTX_set_timeout(ctx, t); -} - -static long SSL_CTX_get_timeout_not_a_macro(SSL_CTX* ctx) { - return SSL_CTX_get_timeout(ctx); -} - -static int CRYPTO_add_not_a_macro(int *pointer,int amount,int type) { - return CRYPTO_add(pointer, amount, type); -} - -static long SSL_CTX_add_extra_chain_cert_not_a_macro(SSL_CTX* ctx, X509 *cert) { - return SSL_CTX_add_extra_chain_cert(ctx, cert); -} - -static long SSL_CTX_set_tlsext_servername_callback_not_a_macro( - SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)) { - return SSL_CTX_set_tlsext_servername_callback(ctx, cb); -} - -#ifndef SSL_MODE_RELEASE_BUFFERS -#define SSL_MODE_RELEASE_BUFFERS 0 -#endif - -#ifndef SSL_OP_NO_COMPRESSION -#define SSL_OP_NO_COMPRESSION 0 -#endif - -#if defined SSL_CTRL_SET_TLSEXT_HOSTNAME - extern int sni_cb(SSL *ssl_conn, int *ad, void *arg); -#endif - -extern int verify_cb(int ok, X509_STORE_CTX* store); typedef STACK_OF(X509_NAME) *STACK_OF_X509_NAME_not_a_macro; @@ -97,6 +25,7 @@ static void sk_X509_NAME_pop_free_not_a_macro(STACK_OF_X509_NAME_not_a_macro st) } extern int password_cb(char *buf, int size, int rwflag, void *password); + */ import "C" @@ -114,7 +43,7 @@ import ( ) var ( - ssl_ctx_idx = C.SSL_CTX_get_ex_new_index(0, nil, nil, nil, nil) + ssl_ctx_idx = C.X_SSL_CTX_new_index() logger = spacelog.GetLogger() ) @@ -169,10 +98,16 @@ const ( func NewCtxWithVersion(version SSLVersion) (*Ctx, error) { var method *C.SSL_METHOD switch version { + case SSLv3: + method = C.X_SSLv3_method() case TLSv1: - method = C.TLSv1_method() + method = C.X_TLSv1_method() + case TLSv1_1: + method = C.X_TLSv1_1_method() + case TLSv1_2: + method = C.X_TLSv1_2_method() case AnyVersion: - method = C.SSLv23_method() + method = C.X_SSLv23_method() } if method == nil { return nil, errors.New("unknown ssl/tls version") @@ -255,6 +190,8 @@ const ( Prime256v1 EllipticCurve = C.NID_X9_62_prime256v1 // P-384: NIST/SECG curve over a 384 bit prime field Secp384r1 EllipticCurve = C.NID_secp384r1 + // P-521: NIST/SECG curve over a 521 bit prime field + Secp521r1 EllipticCurve = C.NID_secp521r1 ) // UseCertificate configures the context to present the given certificate to @@ -386,7 +323,7 @@ func (c *Ctx) AddChainCertificate(cert *Certificate) error { runtime.LockOSThread() defer runtime.UnlockOSThread() c.chain = append(c.chain, cert) - if int(C.SSL_CTX_add_extra_chain_cert_not_a_macro(c.ctx, cert.x)) != 1 { + if int(C.X_SSL_CTX_add_extra_chain_cert(c.ctx, cert.x)) != 1 { return errorFromErrorQueue() } // OpenSSL takes ownership via SSL_CTX_add_extra_chain_cert @@ -581,7 +518,9 @@ func (self *CertificateStoreCtx) GetCurrentCert() *Certificate { return nil } // add a ref - C.CRYPTO_add_not_a_macro(&x509.references, 1, C.CRYPTO_LOCK_X509) + if 1 != C.X_X509_add_ref(x509) { + return nil + } cert := &Certificate{ x: x509, } @@ -617,10 +556,13 @@ type Options uint const ( // NoCompression is only valid if you are using OpenSSL 1.0.1 or newer - NoCompression Options = C.SSL_OP_NO_COMPRESSION - NoSSLv2 Options = C.SSL_OP_NO_SSLv2 - NoSSLv3 Options = C.SSL_OP_NO_SSLv3 - NoTLSv1 Options = C.SSL_OP_NO_TLSv1 + NoCompression Options = C.SSL_OP_NO_COMPRESSION + NoSSLv2 Options = C.SSL_OP_NO_SSLv2 + NoSSLv3 Options = C.SSL_OP_NO_SSLv3 + NoTLSv1 Options = C.SSL_OP_NO_TLSv1 + // NoTLSv1_1 and NoTLSv1_2 are only valid if you are using OpenSSL 1.0.1 or newer + NoTLSv1_1 Options = C.SSL_OP_NO_TLSv1_1 + NoTLSv1_2 Options = C.SSL_OP_NO_TLSv1_2 CipherServerPreference Options = C.SSL_OP_CIPHER_SERVER_PREFERENCE NoSessionResumptionOrRenegotiation Options = C.SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION NoTicket Options = C.SSL_OP_NO_TICKET @@ -630,19 +572,19 @@ const ( // SetOptions sets context options. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (c *Ctx) SetOptions(options Options) Options { - return Options(C.SSL_CTX_set_options_not_a_macro( + return Options(C.X_SSL_CTX_set_options( c.ctx, C.long(options))) } func (c *Ctx) ClearOptions(options Options) Options { - return Options(C.SSL_CTX_clear_options_not_a_macro( + return Options(C.X_SSL_CTX_clear_options( c.ctx, C.long(options))) } // GetOptions returns context options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (c *Ctx) GetOptions() Options { - return Options(C.SSL_CTX_get_options_not_a_macro(c.ctx)) + return Options(C.X_SSL_CTX_get_options(c.ctx)) } type Modes int @@ -656,13 +598,13 @@ const ( // SetMode sets context modes. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_mode.html func (c *Ctx) SetMode(modes Modes) Modes { - return Modes(C.SSL_CTX_set_mode_not_a_macro(c.ctx, C.long(modes))) + return Modes(C.X_SSL_CTX_set_mode(c.ctx, C.long(modes))) } // GetMode returns context modes. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_mode.html func (c *Ctx) GetMode() Modes { - return Modes(C.SSL_CTX_get_mode_not_a_macro(c.ctx)) + return Modes(C.X_SSL_CTX_get_mode(c.ctx)) } type VerifyOptions int @@ -683,8 +625,8 @@ const ( type VerifyCallback func(ok bool, store *CertificateStoreCtx) bool -//export verify_cb_thunk -func verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { +//export go_ssl_ctx_verify_cb_thunk +func go_ssl_ctx_verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { defer func() { if err := recover(); err != nil { logger.Critf("openssl: verify callback panic'd: %v", err) @@ -709,7 +651,7 @@ func verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { func (c *Ctx) SetVerify(options VerifyOptions, verify_cb VerifyCallback) { c.verify_cb = verify_cb if verify_cb != nil { - C.SSL_CTX_set_verify(c.ctx, C.int(options), (*[0]byte)(C.verify_cb)) + C.SSL_CTX_set_verify(c.ctx, C.int(options), (*[0]byte)(C.X_SSL_CTX_verify_cb)) } else { C.SSL_CTX_set_verify(c.ctx, C.int(options), nil) } @@ -752,7 +694,7 @@ type TLSExtServernameCallback func(ssl *SSL) SSLTLSExtErr // http://stackoverflow.com/questions/22373332/serving-multiple-domains-in-one-box-with-sni func (c *Ctx) SetTLSExtServernameCallback(sni_cb TLSExtServernameCallback) { c.sni_cb = sni_cb - C.SSL_CTX_set_tlsext_servername_callback_not_a_macro(c.ctx, (*[0]byte)(C.sni_cb)) + C.X_SSL_CTX_set_tlsext_servername_callback(c.ctx, (*[0]byte)(C.sni_cb)) } func (c *Ctx) SetSessionId(session_id []byte) error { @@ -800,30 +742,30 @@ const ( // http://www.openssl.org/docs/ssl/SSL_CTX_set_session_cache_mode.html func (c *Ctx) SetSessionCacheMode(modes SessionCacheModes) SessionCacheModes { return SessionCacheModes( - C.SSL_CTX_set_session_cache_mode_not_a_macro(c.ctx, C.long(modes))) + C.X_SSL_CTX_set_session_cache_mode(c.ctx, C.long(modes))) } // Set session cache timeout. Returns previously set value. // See https://www.openssl.org/docs/ssl/SSL_CTX_set_timeout.html func (c *Ctx) SetTimeout(t time.Duration) time.Duration { - prev := C.SSL_CTX_set_timeout_not_a_macro(c.ctx, C.long(t/time.Second)) + prev := C.X_SSL_CTX_set_timeout(c.ctx, C.long(t/time.Second)) return time.Duration(prev) * time.Second } // Get session cache timeout. // See https://www.openssl.org/docs/ssl/SSL_CTX_set_timeout.html func (c *Ctx) GetTimeout() time.Duration { - return time.Duration(C.SSL_CTX_get_timeout_not_a_macro(c.ctx)) * time.Second + return time.Duration(C.X_SSL_CTX_get_timeout(c.ctx)) * time.Second } // Set session cache size. Returns previously set value. // https://www.openssl.org/docs/ssl/SSL_CTX_sess_set_cache_size.html func (c *Ctx) SessSetCacheSize(t int) int { - return int(C.SSL_CTX_sess_set_cache_size_not_a_macro(c.ctx, C.long(t))) + return int(C.X_SSL_CTX_sess_set_cache_size(c.ctx, C.long(t))) } // Get session cache size. // https://www.openssl.org/docs/ssl/SSL_CTX_sess_set_cache_size.html func (c *Ctx) SessGetCacheSize() int { - return int(C.SSL_CTX_sess_get_cache_size_not_a_macro(c.ctx)) + return int(C.X_SSL_CTX_sess_get_cache_size(c.ctx)) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go index 9644e518bf3..cd2a82a5a66 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ctx_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go new file mode 100644 index 00000000000..7d0cc703985 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh.go @@ -0,0 +1,68 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" +import ( + "errors" + "unsafe" +) + +// DeriveSharedSecret derives a shared secret using a private key and a peer's +// public key. +// The specific algorithm that is used depends on the types of the +// keys, but it is most commonly a variant of Diffie-Hellman. +func DeriveSharedSecret(private PrivateKey, public PublicKey) ([]byte, error) { + // Create context for the shared secret derivation + dhCtx := C.EVP_PKEY_CTX_new(private.evpPKey(), nil) + if dhCtx == nil { + return nil, errors.New("failed creating shared secret derivation context") + } + defer C.EVP_PKEY_CTX_free(dhCtx) + + // Initialize the context + if int(C.EVP_PKEY_derive_init(dhCtx)) != 1 { + return nil, errors.New("failed initializing shared secret derivation context") + } + + // Provide the peer's public key + if int(C.EVP_PKEY_derive_set_peer(dhCtx, public.evpPKey())) != 1 { + return nil, errors.New("failed adding peer public key to context") + } + + // Determine how large of a buffer we need for the shared secret + var buffLen C.size_t + if int(C.EVP_PKEY_derive(dhCtx, nil, &buffLen)) != 1 { + return nil, errors.New("failed determining shared secret length") + } + + // Allocate a buffer + buffer := C.X_OPENSSL_malloc(buffLen) + if buffer == nil { + return nil, errors.New("failed allocating buffer for shared secret") + } + defer C.X_OPENSSL_free(buffer) + + // Derive the shared secret + if int(C.EVP_PKEY_derive(dhCtx, (*C.uchar)(buffer), &buffLen)) != 1 { + return nil, errors.New("failed deriving the shared secret") + } + + secret := C.GoBytes(unsafe.Pointer(buffer), C.int(buffLen)) + return secret, nil +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go new file mode 100644 index 00000000000..e6b5ae59905 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dh_test.go @@ -0,0 +1,51 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "bytes" + "testing" +) + +func TestECDH(t *testing.T) { + t.Parallel() + if !HasECDH() { + t.Skip("ECDH not available") + } + + myKey, err := GenerateECKey(Prime256v1) + if err != nil { + t.Fatal(err) + } + peerKey, err := GenerateECKey(Prime256v1) + if err != nil { + t.Fatal(err) + } + + mySecret, err := DeriveSharedSecret(myKey, peerKey) + if err != nil { + t.Fatal(err) + } + theirSecret, err := DeriveSharedSecret(peerKey, myKey) + if err != nil { + t.Fatal(err) + } + + if bytes.Compare(mySecret, theirSecret) != 0 { + t.Fatal("shared secrets are different") + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go index a698645c1ec..294d0645c03 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/dhparam.go @@ -1,21 +1,20 @@ -// +build cgo +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package openssl -/* -#include <openssl/crypto.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/conf.h> -#include <openssl/dh.h> - -static long SSL_CTX_set_tmp_dh_not_a_macro(SSL_CTX* ctx, DH *dh) { - return SSL_CTX_set_tmp_dh(ctx, dh); -} -static long PEM_read_DHparams_not_a_macro(SSL_CTX* ctx, DH *dh) { - return SSL_CTX_set_tmp_dh(ctx, dh); -} -*/ +// #include "shim.h" import "C" import ( @@ -58,7 +57,7 @@ func (c *Ctx) SetDHParameters(dh *DH) error { runtime.LockOSThread() defer runtime.UnlockOSThread() - if int(C.SSL_CTX_set_tmp_dh_not_a_macro(c.ctx, dh.dh)) != 1 { + if int(C.X_SSL_CTX_set_tmp_dh(c.ctx, dh.dh)) != 1 { return errorFromErrorQueue() } return nil diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go index 44d4d001b13..6d8d2635aee 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/digest.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> +// #include "shim.h" import "C" import ( @@ -34,7 +32,7 @@ type Digest struct { func GetDigestByName(name string) (*Digest, error) { cname := C.CString(name) defer C.free(unsafe.Pointer(cname)) - p := C.EVP_get_digestbyname(cname) + p := C.X_EVP_get_digestbyname(cname) if p == nil { return nil, fmt.Errorf("Digest %v not found", name) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go index 7a175b70f7c..78aef956fca 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/engine.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/features.go index 894c2676038..c091f0644e8 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/features.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <openssl/ssl.h> -#include <openssl/evp.h> -#include "_cgo_export.h" +package openssl -int ticket_key_cb(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc) { +// #include "shim.h" +import "C" - SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(s); - void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return ticket_key_cb_thunk(p, s, key_name, iv, cctx, hctx, enc); +func HasECDH() bool { + return C.X_OPENSSL_NO_ECDH() == 0 } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go index fcccb000a36..77e1dc3eddf 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips.go @@ -1,19 +1,56 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build cgo -// +build !darwin package openssl /* -#include <openssl/ssl.h> +#include "shim.h" + +static int X_FIPS_defined() { +#ifdef OPENSSL_FIPS + return 1; +#else + return 0; +#endif +} + */ import "C" +import "runtime" +// FIPSModeDefined indicates if the openssl library has the FIPS +// module complied in, specifically if the "OPENSSL_FIPS" macro is defined. +func FIPSModeDefined() bool { + if C.X_FIPS_defined() == 1 { + return true + } + return false +} + +// FIPSModeSet enables a FIPS 140-2 validated mode of operation. +// https://wiki.openssl.org/index.php/FIPS_mode_set() func FIPSModeSet(mode bool) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + var r C.int if mode { - r = C.FIPS_mode_set(1) + r = C.X_FIPS_mode_set(1) } else { - r = C.FIPS_mode_set(0) + r = C.X_FIPS_mode_set(0) } if r != 1 { return errorFromErrorQueue() @@ -22,8 +59,8 @@ func FIPSModeSet(mode bool) error { } func FIPSMode() bool { - if C.FIPS_mode() == 0 { - return false + if FIPSModeDefined() && C.X_FIPS_mode() != 0 { + return true } - return true + return false } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go index 63d353b4a41..31218edb33b 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/fips_test.go @@ -1,5 +1,3 @@ -// +build !darwin - package openssl_test import ( @@ -9,8 +7,12 @@ import ( ) func TestSetFIPSMode(t *testing.T) { + if !openssl.FIPSModeDefined() { + t.Skip("OPENSSL_FIPS not defined in headers") + } + if openssl.FIPSMode() { - t.Fatal("Expected FIPS mode to be disabled, but was enabled") + t.Skip("FIPS mode already enabled") } err := openssl.FIPSModeSet(true) @@ -22,12 +24,4 @@ func TestSetFIPSMode(t *testing.T) { t.Fatal("Expected FIPS mode to be enabled, but was disabled") } - err = openssl.FIPSModeSet(false) - if err != nil { - t.Fatal(err) - } - - if openssl.FIPSMode() { - t.Fatal("Expected FIPS mode to be disabled, but was enabled") - } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go new file mode 100644 index 00000000000..a8640cfac63 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac.go @@ -0,0 +1,91 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openssl + +// #include "shim.h" +import "C" + +import ( + "errors" + "runtime" + "unsafe" +) + +type HMAC struct { + ctx *C.HMAC_CTX + engine *Engine + md *C.EVP_MD +} + +func NewHMAC(key []byte, digestAlgorithm EVP_MD) (*HMAC, error) { + return NewHMACWithEngine(key, digestAlgorithm, nil) +} + +func NewHMACWithEngine(key []byte, digestAlgorithm EVP_MD, e *Engine) (*HMAC, error) { + var md *C.EVP_MD = getDigestFunction(digestAlgorithm) + h := &HMAC{engine: e, md: md} + h.ctx = C.X_HMAC_CTX_new() + if h.ctx == nil { + return nil, errors.New("unable to allocate HMAC_CTX") + } + + var c_e *C.ENGINE + if e != nil { + c_e = e.e + } + if rc := C.X_HMAC_Init_ex(h.ctx, + unsafe.Pointer(&key[0]), + C.int(len(key)), + md, + c_e); rc != 1 { + C.X_HMAC_CTX_free(h.ctx) + return nil, errors.New("failed to initialize HMAC_CTX") + } + + runtime.SetFinalizer(h, func(h *HMAC) { h.Close() }) + return h, nil +} + +func (h *HMAC) Close() { + C.X_HMAC_CTX_free(h.ctx) +} + +func (h *HMAC) Write(data []byte) (n int, err error) { + if len(data) == 0 { + return 0, nil + } + if rc := C.X_HMAC_Update(h.ctx, (*C.uchar)(unsafe.Pointer(&data[0])), + C.size_t(len(data))); rc != 1 { + return 0, errors.New("failed to update HMAC") + } + return len(data), nil +} + +func (h *HMAC) Reset() error { + if 1 != C.X_HMAC_Init_ex(h.ctx, nil, 0, nil, nil) { + return errors.New("failed to reset HMAC_CTX") + } + return nil +} + +func (h *HMAC) Final() (result []byte, err error) { + mdLength := C.X_EVP_MD_size(h.md) + result = make([]byte, mdLength) + if rc := C.X_HMAC_Final(h.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), + (*C.uint)(unsafe.Pointer(&mdLength))); rc != 1 { + return nil, errors.New("failed to finalized HMAC") + } + return result, h.Reset() +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go new file mode 100644 index 00000000000..424720e2171 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hmac_test.go @@ -0,0 +1,74 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "testing" +) + +func TestSHA256HMAC(t *testing.T) { + key := []byte("d741787cc61851af045ccd37") + data := []byte("5912EEFD-59EC-43E3-ADB8-D5325AEC3271") + + h, err := NewHMAC(key, EVP_SHA256) + if err != nil { + t.Fatalf("Unable to create new HMAC: %s", err) + } + if _, err := h.Write(data); err != nil { + t.Fatalf("Unable to write data into HMAC: %s", err) + } + + var actualHMACBytes []byte + if actualHMACBytes, err = h.Final(); err != nil { + t.Fatalf("Error while finalizing HMAC: %s", err) + } + actualString := hex.EncodeToString(actualHMACBytes) + + // generate HMAC with built-in crypto lib + mac := hmac.New(sha256.New, key) + mac.Write(data) + expectedString := hex.EncodeToString(mac.Sum(nil)) + + if expectedString != actualString { + t.Errorf("HMAC was incorrect: expected=%s, actual=%s", expectedString, actualString) + } +} + +func BenchmarkSHA256HMAC(b *testing.B) { + key := []byte("d741787cc61851af045ccd37") + data := []byte("5912EEFD-59EC-43E3-ADB8-D5325AEC3271") + + h, err := NewHMAC(key, EVP_SHA256) + if err != nil { + b.Fatalf("Unable to create new HMAC: %s", err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := h.Write(data); err != nil { + b.Fatalf("Unable to write data into HMAC: %s", err) + } + + var err error + if _, err = h.Final(); err != nil { + b.Fatalf("Error while finalizing HMAC: %s", err) + } + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c index 9a610292067..aef33355262 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.c @@ -1,7 +1,8 @@ -/* Go-OpenSSL notice: - This file is required for all OpenSSL versions prior to 1.1.0. This simply - provides the new 1.1.0 X509_check_* methods for hostname validation if they - don't already exist. +/* + * Go-OpenSSL notice: + * This file is required for all OpenSSL versions prior to 1.1.0. This simply + * provides the new 1.1.0 X509_check_* methods for hostname validation if they + * don't already exist. */ #include <openssl/x509.h> @@ -67,6 +68,7 @@ */ /* X509 v3 extension utilities */ +#include <string.h> #include <stdlib.h> #include <openssl/ssl.h> #include <openssl/conf.h> @@ -346,22 +348,26 @@ static int do_x509_check(X509 *x, const unsigned char *chk, size_t chklen, return 0; } -int _X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, - unsigned int flags) +#if OPENSSL_VERSION_NUMBER < 0x1000200fL + +int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags, char **peername) { return do_x509_check(x, chk, chklen, flags, GEN_DNS); } -int _X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, +int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags) { return do_x509_check(x, chk, chklen, flags, GEN_EMAIL); } -int _X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, +int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags) { return do_x509_check(x, chk, chklen, flags, GEN_IPADD); } +#endif /* OPENSSL_VERSION_NUMBER */ + #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go index c1d1202fb65..f0b36db678d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/hostname.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl /* @@ -25,11 +23,11 @@ package openssl #define X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT 0x1 #define X509_CHECK_FLAG_NO_WILDCARDS 0x2 -extern int _X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, - unsigned int flags); -extern int _X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, +extern int X509_check_host(X509 *x, const unsigned char *chk, size_t chklen, + unsigned int flags, char **peername); +extern int X509_check_email(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags); -extern int _X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, +extern int X509_check_ip(X509 *x, const unsigned char *chk, size_t chklen, unsigned int flags); #endif */ @@ -60,8 +58,9 @@ const ( func (c *Certificate) CheckHost(host string, flags CheckFlags) error { chost := unsafe.Pointer(C.CString(host)) defer C.free(chost) - rv := C._X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)), - C.uint(flags)) + + rv := C.X509_check_host(c.x, (*C.uchar)(chost), C.size_t(len(host)), + C.uint(flags), nil) if rv > 0 { return nil } @@ -79,7 +78,7 @@ func (c *Certificate) CheckHost(host string, flags CheckFlags) error { func (c *Certificate) CheckEmail(email string, flags CheckFlags) error { cemail := unsafe.Pointer(C.CString(email)) defer C.free(cemail) - rv := C._X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)), + rv := C.X509_check_email(c.x, (*C.uchar)(cemail), C.size_t(len(email)), C.uint(flags)) if rv > 0 { return nil @@ -97,7 +96,7 @@ func (c *Certificate) CheckEmail(email string, flags CheckFlags) error { // there was no internal error. func (c *Certificate) CheckIP(ip net.IP, flags CheckFlags) error { cip := unsafe.Pointer(&ip[0]) - rv := C._X509_check_ip(c.x, (*C.uchar)(cip), C.size_t(len(ip)), + rv := C.X509_check_ip(c.x, (*C.uchar)(cip), C.size_t(len(ip)), C.uint(flags)) if rv > 0 { return nil diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go index e3be32c264a..39bd5a28b5f 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/http.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go index 314e5415c18..ac2aa04327b 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,49 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - /* Package openssl is a light wrapper around OpenSSL for Go. -It strives to provide a near-drop-in replacement for the Go standard library -tls package, while allowing for: - -Performance - -OpenSSL is battle-tested and optimized C. While Go's built-in library shows -great promise, it is still young and in some places, inefficient. This simple -OpenSSL wrapper can often do at least 2x with the same cipher and protocol. - -On my lappytop, I get the following benchmarking speeds: - BenchmarkSHA1Large_openssl 1000 2611282 ns/op 401.56 MB/s - BenchmarkSHA1Large_stdlib 500 3963983 ns/op 264.53 MB/s - BenchmarkSHA1Small_openssl 1000000 3476 ns/op 0.29 MB/s - BenchmarkSHA1Small_stdlib 5000000 550 ns/op 1.82 MB/s - BenchmarkSHA256Large_openssl 200 8085314 ns/op 129.69 MB/s - BenchmarkSHA256Large_stdlib 100 18948189 ns/op 55.34 MB/s - BenchmarkSHA256Small_openssl 1000000 4262 ns/op 0.23 MB/s - BenchmarkSHA256Small_stdlib 1000000 1444 ns/op 0.69 MB/s - BenchmarkOpenSSLThroughput 100000 21634 ns/op 47.33 MB/s - BenchmarkStdlibThroughput 50000 58974 ns/op 17.36 MB/s - -Interoperability - -Many systems support OpenSSL with a variety of plugins and modules for things, -such as hardware acceleration in embedded devices. - -Greater flexibility and configuration - -OpenSSL allows for far greater configuration of corner cases and backwards -compatibility (such as support of SSLv2). You shouldn't be using SSLv2 if you -can help but, but sometimes you can't help it. - -Security - -Yeah yeah, Heartbleed. But according to the author of the standard library's -TLS implementation, Go's TLS library is vulnerable to timing attacks. And -whether or not OpenSSL received the appropriate amount of scrutiny -pre-Heartbleed, it sure is receiving it now. +This version has been forked from https://github.com/spacemonkeygo/openssl +for greater back-compatibility to older openssl libraries. Usage @@ -80,62 +42,26 @@ Making a client connection is straightforward too: } conn, err := openssl.Dial("tcp", "localhost:7777", ctx, 0) -Help wanted: To get this library to work with net/http's client, we -had to fork net/http. It would be nice if an alternate http client library -supported the generality needed to use OpenSSL instead of crypto/tls. */ package openssl -/* -#include <openssl/ssl.h> -#include <openssl/conf.h> -#include <openssl/err.h> -#include <openssl/evp.h> -#include <openssl/engine.h> - -extern int Goopenssl_init_locks(); -extern unsigned long Goopenssl_thread_id_callback(); -extern void Goopenssl_thread_locking_callback(int, int, const char*, int); - -static int Goopenssl_init_threadsafety() { - // Set up OPENSSL thread safety callbacks. - // TOOLS-1694 added setting of thread id callback for compatibility with openssl 0.9.8 - int rc = Goopenssl_init_locks(); - if (rc == 0) { - CRYPTO_set_locking_callback(Goopenssl_thread_locking_callback); - } - CRYPTO_set_id_callback(Goopenssl_thread_id_callback); - return rc; -} - -static void OpenSSL_add_all_algorithms_not_a_macro() { - OpenSSL_add_all_algorithms(); -} - -*/ +// #include "shim.h" import "C" import ( - "errors" "fmt" "strings" ) func init() { - C.ERR_load_crypto_strings() - C.OPENSSL_config(nil) - C.ENGINE_load_builtin_engines() - C.SSL_load_error_strings() - C.SSL_library_init() - C.OpenSSL_add_all_algorithms_not_a_macro() - rc := C.Goopenssl_init_threadsafety() - if rc != 0 { - panic(fmt.Errorf("Goopenssl_init_locks failed with %d", rc)) + if rc := C.X_shim_init(); rc != 0 { + panic(fmt.Errorf("X_shim_init failed with %d", rc)) } } // errorFromErrorQueue needs to run in the same OS thread as the operation -// that caused the possible error +// that caused the possible error. In some circumstances, ERR_get_error +// returns 0 when it shouldn't so we provide a message in that case. func errorFromErrorQueue() error { var errs []string for { @@ -143,10 +69,14 @@ func errorFromErrorQueue() error { if err == 0 { break } - errs = append(errs, fmt.Sprintf("%s:%s:%s", + errs = append(errs, fmt.Sprintf("%x:%s:%s:%s", + err, C.GoString(C.ERR_lib_error_string(err)), C.GoString(C.ERR_func_error_string(err)), C.GoString(C.ERR_reason_error_string(err)))) } - return errors.New(fmt.Sprintf("SSL errors: %s", strings.Join(errs, "\n"))) + if len(errs) == 0 { + errs = append(errs, "0:Error unavailable") + } + return fmt.Errorf("SSL errors: %s", strings.Join(errs, "\n")) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go index 99558298e3a..9e52b4e00be 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_posix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,13 +18,14 @@ package openssl /* +#if OPENSSL_VERSION_NUMBER < 0x10100000L #include <errno.h> #include <openssl/crypto.h> #include <pthread.h> pthread_mutex_t* goopenssl_locks; -int Goopenssl_init_locks() { +int go_init_locks() { int rc = 0; int nlock; int i; @@ -52,8 +53,7 @@ int Goopenssl_init_locks() { return rc; } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -void Goopenssl_thread_locking_callback(int mode, int n, const char *file, +void go_thread_locking_callback(int mode, int n, const char *file, int line) { if (mode & CRYPTO_LOCK) { pthread_mutex_lock(&goopenssl_locks[n]); @@ -61,7 +61,8 @@ void Goopenssl_thread_locking_callback(int mode, int n, const char *file, pthread_mutex_unlock(&goopenssl_locks[n]); } } -unsigned long Goopenssl_thread_id_callback() { + +unsigned long go_thread_id_callback() { return (unsigned long) pthread_self(); } #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go index ec817926b7a..4a096899074 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/init_windows.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,20 +17,14 @@ package openssl /* - -#cgo windows LDFLAGS: -lssleay32 -llibeay32 -L c:/openssl/bin -#cgo windows CFLAGS: -I"c:/openssl/include" - -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#endif +#if OPENSSL_VERSION_NUMBER < 0x10100000L #include <errno.h> #include <openssl/crypto.h> #include <windows.h> CRITICAL_SECTION* goopenssl_locks; -int Goopenssl_init_locks() { +int go_init_locks() { int rc = 0; int nlock; int i; @@ -48,7 +42,7 @@ int Goopenssl_init_locks() { return 0; } -void Goopenssl_thread_locking_callback(int mode, int n, const char *file, +void go_thread_locking_callback(int mode, int n, const char *file, int line) { if (mode & CRYPTO_LOCK) { EnterCriticalSection(&goopenssl_locks[n]); @@ -56,8 +50,8 @@ void Goopenssl_thread_locking_callback(int mode, int n, const char *file, LeaveCriticalSection(&goopenssl_locks[n]); } } -#if OPENSSL_VERSION_NUMBER < 0x10100000L -unsigned long Goopenssl_thread_id_callback() { + +unsigned long go_thread_id_callback() { return (unsigned long) GetCurrentThreadId(); } #endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go index cc17f5fcf7d..4e39a38a579 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,35 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -// #include <openssl/evp.h> -// #include <openssl/ssl.h> -// #include <openssl/conf.h> -// -// int EVP_SignInit_not_a_macro(EVP_MD_CTX *ctx, const EVP_MD *type) { -// return EVP_SignInit(ctx, type); -// } -// -// int EVP_SignUpdate_not_a_macro(EVP_MD_CTX *ctx, const void *d, -// unsigned int cnt) { -// return EVP_SignUpdate(ctx, d, cnt); -// } -// -// int EVP_VerifyInit_not_a_macro(EVP_MD_CTX *ctx, const EVP_MD *type) { -// return EVP_VerifyInit(ctx, type); -// } -// -// int EVP_VerifyUpdate_not_a_macro(EVP_MD_CTX *ctx, const void *d, -// unsigned int cnt) { -// return EVP_VerifyUpdate(ctx, d, cnt); -// } -// -// int EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key) { -// return EVP_PKEY_assign(pkey, type, key); -// } +// #include "shim.h" import "C" import ( @@ -53,25 +27,30 @@ import ( type Method *C.EVP_MD var ( - SHA1_Method Method = C.EVP_sha1() - SHA256_Method Method = C.EVP_sha256() - SHA512_Method Method = C.EVP_sha512() + SHA1_Method Method = C.X_EVP_sha1() + SHA256_Method Method = C.X_EVP_sha256() + SHA512_Method Method = C.X_EVP_sha512() ) -type PublicKey interface { - // Verifies the data signature using PKCS1.15 - VerifyPKCS1v15(method Method, data, sig []byte) error - - // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX - // format - MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) - - // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX - // format - MarshalPKIXPublicKeyDER() (der_block []byte, err error) - - evpPKey() *C.EVP_PKEY -} +// Constants for the various key types. +// Mapping of name -> NID taken from openssl/evp.h +const ( + KeyTypeNone = NID_undef + KeyTypeRSA = NID_rsaEncryption + KeyTypeRSA2 = NID_rsa + KeyTypeDSA = NID_dsa + KeyTypeDSA1 = NID_dsa_2 + KeyTypeDSA2 = NID_dsaWithSHA + KeyTypeDSA3 = NID_dsaWithSHA1 + KeyTypeDSA4 = NID_dsaWithSHA1_2 + KeyTypeDH = NID_dhKeyAgreement + KeyTypeDHX = NID_dhpublicnumber + KeyTypeEC = NID_X9_62_id_ecPublicKey + KeyTypeHMAC = NID_hmac + KeyTypeCMAC = NID_cmac + KeyTypeTLS1PRF = NID_tls1_prf + KeyTypeHKDF = NID_hkdf +) type PrivateKey interface { PublicKey @@ -95,22 +74,21 @@ type pKey struct { func (key *pKey) evpPKey() *C.EVP_PKEY { return key.key } func (key *pKey) SignPKCS1v15(method Method, data []byte) ([]byte, error) { - var ctx C.EVP_MD_CTX - C.EVP_MD_CTX_init(&ctx) - defer C.EVP_MD_CTX_cleanup(&ctx) + ctx := C.X_EVP_MD_CTX_new() + defer C.X_EVP_MD_CTX_free(ctx) - if 1 != C.EVP_SignInit_not_a_macro(&ctx, method) { + if 1 != C.X_EVP_SignInit(ctx, method) { return nil, errors.New("signpkcs1v15: failed to init signature") } if len(data) > 0 { - if 1 != C.EVP_SignUpdate_not_a_macro( - &ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { + if 1 != C.X_EVP_SignUpdate( + ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { return nil, errors.New("signpkcs1v15: failed to update signature") } } - sig := make([]byte, C.EVP_PKEY_size(key.key)) + sig := make([]byte, C.X_EVP_PKEY_size(key.key)) var sigblen C.uint - if 1 != C.EVP_SignFinal(&ctx, + if 1 != C.X_EVP_SignFinal(ctx, ((*C.uchar)(unsafe.Pointer(&sig[0]))), &sigblen, key.key) { return nil, errors.New("signpkcs1v15: failed to finalize signature") } @@ -118,45 +96,25 @@ func (key *pKey) SignPKCS1v15(method Method, data []byte) ([]byte, error) { } func (key *pKey) VerifyPKCS1v15(method Method, data, sig []byte) error { - var ctx C.EVP_MD_CTX - C.EVP_MD_CTX_init(&ctx) - defer C.EVP_MD_CTX_cleanup(&ctx) + ctx := C.X_EVP_MD_CTX_new() + defer C.X_EVP_MD_CTX_free(ctx) - if 1 != C.EVP_VerifyInit_not_a_macro(&ctx, method) { + if 1 != C.X_EVP_VerifyInit(ctx, method) { return errors.New("verifypkcs1v15: failed to init verify") } if len(data) > 0 { - if 1 != C.EVP_VerifyUpdate_not_a_macro( - &ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { + if 1 != C.X_EVP_VerifyUpdate( + ctx, unsafe.Pointer(&data[0]), C.uint(len(data))) { return errors.New("verifypkcs1v15: failed to update verify") } } - if 1 != C.EVP_VerifyFinal(&ctx, + if 1 != C.X_EVP_VerifyFinal(ctx, ((*C.uchar)(unsafe.Pointer(&sig[0]))), C.uint(len(sig)), key.key) { return errors.New("verifypkcs1v15: failed to finalize verify") } return nil } -func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, - err error) { - bio := C.BIO_new(C.BIO_s_mem()) - if bio == nil { - return nil, errors.New("failed to allocate memory BIO") - } - defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.PEM_write_bio_RSAPrivateKey(bio, rsa, nil, nil, C.int(0), nil, - nil)) != 1 { - return nil, errors.New("failed dumping private key") - } - return ioutil.ReadAll(asAnyBio(bio)) -} - func (key *pKey) MarshalPKCS1PrivateKeyDER() (der_block []byte, err error) { bio := C.BIO_new(C.BIO_s_mem()) @@ -164,14 +122,11 @@ func (key *pKey) MarshalPKCS1PrivateKeyDER() (der_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.i2d_RSAPrivateKey_bio(bio, rsa)) != 1 { + + if int(C.i2d_PrivateKey_bio(bio, key.key)) != 1 { return nil, errors.New("failed dumping private key der") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -182,14 +137,11 @@ func (key *pKey) MarshalPKIXPublicKeyPEM() (pem_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.PEM_write_bio_RSA_PUBKEY(bio, rsa)) != 1 { + + if int(C.PEM_write_bio_PUBKEY(bio, key.key)) != 1 { return nil, errors.New("failed dumping public key pem") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -200,14 +152,11 @@ func (key *pKey) MarshalPKIXPublicKeyDER() (der_block []byte, return nil, errors.New("failed to allocate memory BIO") } defer C.BIO_free(bio) - rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) - if rsa == nil { - return nil, errors.New("failed getting rsa key") - } - defer C.RSA_free(rsa) - if int(C.i2d_RSA_PUBKEY_bio(bio, rsa)) != 1 { + + if int(C.i2d_PUBKEY_bio(bio, key.key)) != 1 { return nil, errors.New("failed dumping public key der") } + return ioutil.ReadAll(asAnyBio(bio)) } @@ -223,31 +172,20 @@ func LoadPrivateKeyFromPEM(pem_block []byte) (PrivateKey, error) { } defer C.BIO_free(bio) - rsakey := C.PEM_read_bio_RSAPrivateKey(bio, nil, nil, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PrivateKey(bio, nil, nil, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } -// LoadPrivateKeyFromPEM loads a private key from a PEM-encoded block. -func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( +// LoadPrivateKeyFromPEMWithPassword loads a private key from a PEM-encoded block. +func LoadPrivateKeyFromPEMWithPassword(pem_block []byte, password string) ( PrivateKey, error) { if len(pem_block) == 0 { return nil, errors.New("empty pem block") @@ -260,25 +198,14 @@ func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( defer C.BIO_free(bio) cs := C.CString(password) defer C.free(unsafe.Pointer(cs)) - rsakey := C.PEM_read_bio_RSAPrivateKey(bio, nil, nil, unsafe.Pointer(cs)) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PrivateKey(bio, nil, nil, unsafe.Pointer(cs)) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -295,29 +222,25 @@ func LoadPrivateKeyFromDER(der_block []byte) (PrivateKey, error) { } defer C.BIO_free(bio) - rsakey := C.d2i_RSAPrivateKey_bio(bio, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.d2i_PrivateKey_bio(bio, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading private key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } +// LoadPrivateKeyFromPEMWidthPassword loads a private key from a PEM-encoded block. +// Backwards-compatible with typo +func LoadPrivateKeyFromPEMWidthPassword(pem_block []byte, password string) ( + PrivateKey, error) { + return LoadPrivateKeyFromPEMWithPassword(pem_block, password) +} + // LoadPublicKeyFromPEM loads a public key from a PEM-encoded block. func LoadPublicKeyFromPEM(pem_block []byte) (PublicKey, error) { if len(pem_block) == 0 { @@ -330,25 +253,14 @@ func LoadPublicKeyFromPEM(pem_block []byte) (PublicKey, error) { } defer C.BIO_free(bio) - rsakey := C.PEM_read_bio_RSA_PUBKEY(bio, nil, nil, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.PEM_read_bio_PUBKEY(bio, nil, nil, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading public key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -365,25 +277,14 @@ func LoadPublicKeyFromDER(der_block []byte) (PublicKey, error) { } defer C.BIO_free(bio) - rsakey := C.d2i_RSA_PUBKEY_bio(bio, nil) - if rsakey == nil { - return nil, errors.New("failed reading rsa key") - } - defer C.RSA_free(rsakey) - - // convert to PKEY - key := C.EVP_PKEY_new() + key := C.d2i_PUBKEY_bio(bio, nil) if key == nil { - return nil, errors.New("failed converting to evp_pkey") - } - if C.EVP_PKEY_set1_RSA(key, (*C.struct_rsa_st)(rsakey)) != 1 { - C.EVP_PKEY_free(key) - return nil, errors.New("failed converting to evp_pkey") + return nil, errors.New("failed reading public key der") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } @@ -399,17 +300,17 @@ func GenerateRSAKeyWithExponent(bits int, exponent int) (PrivateKey, error) { if rsa == nil { return nil, errors.New("failed to generate RSA key") } - key := C.EVP_PKEY_new() + key := C.X_EVP_PKEY_new() if key == nil { return nil, errors.New("failed to allocate EVP_PKEY") } - if C.EVP_PKEY_assign_charp(key, C.EVP_PKEY_RSA, (*C.char)(unsafe.Pointer(rsa))) != 1 { - C.EVP_PKEY_free(key) + if C.X_EVP_PKEY_assign_charp(key, C.EVP_PKEY_RSA, (*C.char)(unsafe.Pointer(rsa))) != 1 { + C.X_EVP_PKEY_free(key) return nil, errors.New("failed to assign RSA key") } p := &pKey{key: key} runtime.SetFinalizer(p, func(p *pKey) { - C.EVP_PKEY_free(p.key) + C.X_EVP_PKEY_free(p.key) }) return p, nil } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go new file mode 100644 index 00000000000..ed17ef08a40 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_0_9.go @@ -0,0 +1,58 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" +import ( + "errors" + "io/ioutil" +) + +type PublicKey interface { + // Verifies the data signature using PKCS1.15 + VerifyPKCS1v15(method Method, data, sig []byte) error + + // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX + // format + MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) + + // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX + // format + MarshalPKIXPublicKeyDER() (der_block []byte, err error) + + evpPKey() *C.EVP_PKEY +} + +func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, + err error) { + bio := C.BIO_new(C.BIO_s_mem()) + if bio == nil { + return nil, errors.New("failed to allocate memory BIO") + } + defer C.BIO_free(bio) + rsa := (*C.RSA)(C.EVP_PKEY_get1_RSA(key.key)) + if rsa == nil { + return nil, errors.New("failed getting rsa key") + } + defer C.RSA_free(rsa) + if int(C.PEM_write_bio_RSAPrivateKey(bio, rsa, nil, nil, C.int(0), nil, + nil)) != 1 { + return nil, errors.New("failed dumping private key") + } + return ioutil.ReadAll(asAnyBio(bio)) +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go new file mode 100644 index 00000000000..6ea2a46e073 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0.go @@ -0,0 +1,132 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +// #include "shim.h" +import "C" + +import ( + "errors" + "io/ioutil" + "runtime" +) + +type PublicKey interface { + // Verifies the data signature using PKCS1.15 + VerifyPKCS1v15(method Method, data, sig []byte) error + + // MarshalPKIXPublicKeyPEM converts the public key to PEM-encoded PKIX + // format + MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) + + // MarshalPKIXPublicKeyDER converts the public key to DER-encoded PKIX + // format + MarshalPKIXPublicKeyDER() (der_block []byte, err error) + + // KeyType returns an identifier for what kind of key is represented by this + // object. + KeyType() NID + + // BaseType returns an identifier for what kind of key is represented + // by this object. + // Keys that share same algorithm but use different legacy formats + // will have the same BaseType. + // + // For example, a key with a `KeyType() == KeyTypeRSA` and a key with a + // `KeyType() == KeyTypeRSA2` would both have `BaseType() == KeyTypeRSA`. + BaseType() NID + + evpPKey() *C.EVP_PKEY +} + +func (key *pKey) MarshalPKCS1PrivateKeyPEM() (pem_block []byte, + err error) { + bio := C.BIO_new(C.BIO_s_mem()) + if bio == nil { + return nil, errors.New("failed to allocate memory BIO") + } + defer C.BIO_free(bio) + + // PEM_write_bio_PrivateKey_traditional will use the key-specific PKCS1 + // format if one is available for that key type, otherwise it will encode + // to a PKCS8 key. + if int(C.X_PEM_write_bio_PrivateKey_traditional(bio, key.key, nil, nil, + C.int(0), nil, nil)) != 1 { + return nil, errors.New("failed dumping private key") + } + + return ioutil.ReadAll(asAnyBio(bio)) +} + +func (key *pKey) KeyType() NID { + return NID(C.EVP_PKEY_id(key.key)) +} + +func (key *pKey) BaseType() NID { + return NID(C.EVP_PKEY_base_id(key.key)) +} + +// GenerateECKey generates a new elliptic curve private key on the speicified +// curve. +func GenerateECKey(curve EllipticCurve) (PrivateKey, error) { + + // Create context for parameter generation + paramCtx := C.EVP_PKEY_CTX_new_id(C.EVP_PKEY_EC, nil) + if paramCtx == nil { + return nil, errors.New("failed creating EC parameter generation context") + } + defer C.EVP_PKEY_CTX_free(paramCtx) + + // Intialize the parameter generation + if int(C.EVP_PKEY_paramgen_init(paramCtx)) != 1 { + return nil, errors.New("failed initializing EC parameter generation context") + } + + // Set curve in EC parameter generation context + if int(C.X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(paramCtx, C.int(curve))) != 1 { + return nil, errors.New("failed setting curve in EC parameter generation context") + } + + // Create parameter object + var params *C.EVP_PKEY + if int(C.EVP_PKEY_paramgen(paramCtx, ¶ms)) != 1 { + return nil, errors.New("failed creating EC key generation parameters") + } + defer C.EVP_PKEY_free(params) + + // Create context for the key generation + keyCtx := C.EVP_PKEY_CTX_new(params, nil) + if keyCtx == nil { + return nil, errors.New("failed creating EC key generation context") + } + defer C.EVP_PKEY_CTX_free(keyCtx) + + // Generate the key + var privKey *C.EVP_PKEY + if int(C.EVP_PKEY_keygen_init(keyCtx)) != 1 { + return nil, errors.New("failed initializing EC key generation context") + } + if int(C.EVP_PKEY_keygen(keyCtx, &privKey)) != 1 { + return nil, errors.New("failed generating EC private key") + } + + p := &pKey{key: privKey} + runtime.SetFinalizer(p, func(p *pKey) { + C.X_EVP_PKEY_free(p.key) + }) + return p, nil +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go new file mode 100644 index 00000000000..2a2eda887b7 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_1_0_test.go @@ -0,0 +1,149 @@ +// Copyright (C) 2017. See AUTHORS. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !openssl_pre_1.0 + +package openssl + +import ( + "bytes" + "crypto/ecdsa" + "crypto/tls" + "crypto/x509" + "encoding/hex" + pem_pkg "encoding/pem" + "io/ioutil" + "testing" +) + +func TestMarshalEC(t *testing.T) { + if !HasECDH() { + t.Skip("ECDH not available") + } + + key, err := LoadPrivateKeyFromPEM(prime256v1KeyBytes) + if err != nil { + t.Fatal(err) + } + cert, err := LoadCertificateFromPEM(prime256v1CertBytes) + if err != nil { + t.Fatal(err) + } + + privateBlock, _ := pem_pkg.Decode(prime256v1KeyBytes) + key, err = LoadPrivateKeyFromDER(privateBlock.Bytes) + if err != nil { + t.Fatal(err) + } + + pem, err := cert.MarshalPEM() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(pem, prime256v1CertBytes) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", prime256v1CertBytes, 0644) + t.Fatal("invalid cert pem bytes") + } + + pem, err = key.MarshalPKCS1PrivateKeyPEM() + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(pem, prime256v1KeyBytes) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", prime256v1KeyBytes, 0644) + t.Fatal("invalid private key pem bytes") + } + tls_cert, err := tls.X509KeyPair(prime256v1CertBytes, prime256v1KeyBytes) + if err != nil { + t.Fatal(err) + } + tls_key, ok := tls_cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + t.Fatal("FASDFASDF") + } + _ = tls_key + + der, err := key.MarshalPKCS1PrivateKeyDER() + if err != nil { + t.Fatal(err) + } + tls_der, err := x509.MarshalECPrivateKey(tls_key) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(der, tls_der) { + t.Fatalf("invalid private key der bytes: %s\n v.s. %s\n", + hex.Dump(der), hex.Dump(tls_der)) + } + + der, err = key.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + tls_der, err = x509.MarshalPKIXPublicKey(&tls_key.PublicKey) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(der, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(der)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } + + pem, err = key.MarshalPKIXPublicKeyPEM() + if err != nil { + t.Fatal(err) + } + tls_pem := pem_pkg.EncodeToMemory(&pem_pkg.Block{ + Type: "PUBLIC KEY", Bytes: tls_der}) + if !bytes.Equal(pem, tls_pem) { + ioutil.WriteFile("generated", pem, 0644) + ioutil.WriteFile("hardcoded", tls_pem, 0644) + t.Fatal("invalid public key pem bytes") + } + + loaded_pubkey_from_pem, err := LoadPublicKeyFromPEM(pem) + if err != nil { + t.Fatal(err) + } + + loaded_pubkey_from_der, err := LoadPublicKeyFromDER(der) + if err != nil { + t.Fatal(err) + } + + new_der_from_pem, err := loaded_pubkey_from_pem.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + + new_der_from_der, err := loaded_pubkey_from_der.MarshalPKIXPublicKeyDER() + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(new_der_from_der, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(new_der_from_der)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } + + if !bytes.Equal(new_der_from_pem, tls_der) { + ioutil.WriteFile("generated", []byte(hex.Dump(new_der_from_pem)), 0644) + ioutil.WriteFile("hardcoded", []byte(hex.Dump(tls_der)), 0644) + t.Fatal("invalid public key der bytes") + } +} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go index 0af90128530..635ef638ec9 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/key_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ func TestMarshal(t *testing.T) { } tls_der := x509.MarshalPKCS1PrivateKey(tls_key) if !bytes.Equal(der, tls_der) { - t.Fatal("invalid private key der bytes: %s\n v.s. %s\n", + t.Fatalf("invalid private key der bytes: %s\n v.s. %s\n", hex.Dump(der), hex.Dump(tls_der)) } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go index 066aba6b5db..d78cc703472 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/mapping.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go index 7120d065d15..15c897addd1 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/net.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go index c80f237b605..6766b849e76 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/nid.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package openssl type NID int const ( + NID_undef NID = 0 NID_rsadsi NID = 1 NID_pkcs NID = 2 NID_md2 NID = 3 @@ -196,4 +197,10 @@ const ( NID_ad_OCSP NID = 178 NID_ad_ca_issuers NID = 179 NID_OCSP_sign NID = 180 + NID_X9_62_id_ecPublicKey NID = 408 + NID_hmac NID = 855 + NID_cmac NID = 894 + NID_dhpublicnumber NID = 920 + NID_tls1_prf NID = 1021 + NID_hkdf NID = 1036 ) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go deleted file mode 100644 index 30492f3b9d8..00000000000 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/oracle_stubs.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright (C) 2014 Space Monkey, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !cgo - -package openssl - -import ( - "errors" - "net" - "time" -) - -const ( - SSLRecordSize = 16 * 1024 -) - -type Conn struct{} - -func Client(conn net.Conn, ctx *Ctx) (*Conn, error) -func Server(conn net.Conn, ctx *Ctx) (*Conn, error) - -func (c *Conn) Handshake() error -func (c *Conn) PeerCertificate() (*Certificate, error) -func (c *Conn) Close() error -func (c *Conn) Read(b []byte) (n int, err error) -func (c *Conn) Write(b []byte) (written int, err error) - -func (c *Conn) VerifyHostname(host string) error - -func (c *Conn) LocalAddr() net.Addr -func (c *Conn) RemoteAddr() net.Addr -func (c *Conn) SetDeadline(t time.Time) error -func (c *Conn) SetReadDeadline(t time.Time) error -func (c *Conn) SetWriteDeadline(t time.Time) error - -type Ctx struct{} - -type SSLVersion int - -const ( - SSLv3 SSLVersion = 0x02 - TLSv1 SSLVersion = 0x03 - TLSv1_1 SSLVersion = 0x04 - TLSv1_2 SSLVersion = 0x05 - AnyVersion SSLVersion = 0x06 -) - -func NewCtxWithVersion(version SSLVersion) (*Ctx, error) -func NewCtx() (*Ctx, error) -func NewCtxFromFiles(cert_file string, key_file string) (*Ctx, error) -func (c *Ctx) UseCertificate(cert *Certificate) error -func (c *Ctx) UsePrivateKey(key PrivateKey) error - -type CertificateStore struct{} - -func (c *Ctx) GetCertificateStore() *CertificateStore - -func (s *CertificateStore) AddCertificate(cert *Certificate) error - -func (c *Ctx) LoadVerifyLocations(ca_file string, ca_path string) error - -type Options int - -const ( - NoCompression Options = 0 - NoSSLv2 Options = 0 - NoSSLv3 Options = 0 - NoTLSv1 Options = 0 - CipherServerPreference Options = 0 - NoSessionResumptionOrRenegotiation Options = 0 - NoTicket Options = 0 -) - -func (c *Ctx) SetOptions(options Options) Options - -type Modes int - -const ( - ReleaseBuffers Modes = 0 -) - -func (c *Ctx) SetMode(modes Modes) Modes - -type VerifyOptions int - -const ( - VerifyNone VerifyOptions = 0 - VerifyPeer VerifyOptions = 0 - VerifyFailIfNoPeerCert VerifyOptions = 0 - VerifyClientOnce VerifyOptions = 0 -) - -func (c *Ctx) SetVerify(options VerifyOptions) -func (c *Ctx) SetVerifyDepth(depth int) -func (c *Ctx) SetSessionId(session_id []byte) error - -func (c *Ctx) SetCipherList(list string) error - -type SessionCacheModes int - -const ( - SessionCacheOff SessionCacheModes = 0 - SessionCacheClient SessionCacheModes = 0 - SessionCacheServer SessionCacheModes = 0 - SessionCacheBoth SessionCacheModes = 0 - NoAutoClear SessionCacheModes = 0 - NoInternalLookup SessionCacheModes = 0 - NoInternalStore SessionCacheModes = 0 - NoInternal SessionCacheModes = 0 -) - -func (c *Ctx) SetSessionCacheMode(modes SessionCacheModes) SessionCacheModes - -var ( - ValidationError = errors.New("Host validation error") -) - -type CheckFlags int - -const ( - AlwaysCheckSubject CheckFlags = 0 - NoWildcards CheckFlags = 0 -) - -func (c *Certificate) CheckHost(host string, flags CheckFlags) error -func (c *Certificate) CheckEmail(email string, flags CheckFlags) error -func (c *Certificate) CheckIP(ip net.IP, flags CheckFlags) error -func (c *Certificate) VerifyHostname(host string) error - -type PublicKey interface { - MarshalPKIXPublicKeyPEM() (pem_block []byte, err error) - MarshalPKIXPublicKeyDER() (der_block []byte, err error) - evpPKey() struct{} -} - -type PrivateKey interface { - PublicKey - MarshalPKCS1PrivateKeyPEM() (pem_block []byte, err error) - MarshalPKCS1PrivateKeyDER() (der_block []byte, err error) -} - -func LoadPrivateKeyFromPEM(pem_block []byte) (PrivateKey, error) - -type Certificate struct{} - -func LoadCertificateFromPEM(pem_block []byte) (*Certificate, error) - -func (c *Certificate) MarshalPEM() (pem_block []byte, err error) - -func (c *Certificate) PublicKey() (PublicKey, error) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go index 6dad5972dbd..c8b0c1cf19d 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/pem.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Ryan Hileman +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go index 2592b6627d1..c227bee8461 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> - -#include "openssl/evp.h" -*/ +// #include "shim.h" import "C" import ( @@ -33,7 +24,7 @@ import ( ) type SHA1Hash struct { - ctx C.EVP_MD_CTX + ctx *C.EVP_MD_CTX engine *Engine } @@ -41,7 +32,10 @@ func NewSHA1Hash() (*SHA1Hash, error) { return NewSHA1HashWithEngine(nil) } func NewSHA1HashWithEngine(e *Engine) (*SHA1Hash, error) { hash := &SHA1Hash{engine: e} - C.EVP_MD_CTX_init(&hash.ctx) + hash.ctx = C.X_EVP_MD_CTX_new() + if hash.ctx == nil { + return nil, errors.New("openssl: sha1: unable to allocate ctx") + } runtime.SetFinalizer(hash, func(hash *SHA1Hash) { hash.Close() }) if err := hash.Reset(); err != nil { return nil, err @@ -50,7 +44,10 @@ func NewSHA1HashWithEngine(e *Engine) (*SHA1Hash, error) { } func (s *SHA1Hash) Close() { - C.EVP_MD_CTX_cleanup(&s.ctx) + if s.ctx != nil { + C.X_EVP_MD_CTX_free(s.ctx) + s.ctx = nil + } } func engineRef(e *Engine) *C.ENGINE { @@ -61,7 +58,7 @@ func engineRef(e *Engine) *C.ENGINE { } func (s *SHA1Hash) Reset() error { - if 1 != C.EVP_DigestInit_ex(&s.ctx, C.EVP_sha1(), engineRef(s.engine)) { + if 1 != C.X_EVP_DigestInit_ex(s.ctx, C.X_EVP_sha1(), engineRef(s.engine)) { return errors.New("openssl: sha1: cannot init digest ctx") } return nil @@ -71,7 +68,7 @@ func (s *SHA1Hash) Write(p []byte) (n int, err error) { if len(p) == 0 { return 0, nil } - if 1 != C.EVP_DigestUpdate(&s.ctx, unsafe.Pointer(&p[0]), + if 1 != C.X_EVP_DigestUpdate(s.ctx, unsafe.Pointer(&p[0]), C.size_t(len(p))) { return 0, errors.New("openssl: sha1: cannot update digest") } @@ -79,7 +76,7 @@ func (s *SHA1Hash) Write(p []byte) (n int, err error) { } func (s *SHA1Hash) Sum() (result [20]byte, err error) { - if 1 != C.EVP_DigestFinal_ex(&s.ctx, + if 1 != C.X_EVP_DigestFinal_ex(s.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), nil) { return result, errors.New("openssl: sha1: cannot finalize ctx") } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go index 37037e4468b..37808b5a53e 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha1_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( @@ -37,7 +35,7 @@ func TestSHA1(t *testing.T) { } if expected != got { - t.Fatal("exp:%x got:%x", expected, got) + t.Fatalf("exp:%x got:%x", expected, got) } } } @@ -75,7 +73,7 @@ func TestSHA1Writer(t *testing.T) { } if got != exp { - t.Fatal("exp:%x got:%x", exp, got) + t.Fatalf("exp:%x got:%x", exp, got) } } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go index 6785b32f881..d25c7a959d7 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <unistd.h> - -#include "openssl/evp.h" -*/ +// #include "shim.h" import "C" import ( @@ -33,7 +24,7 @@ import ( ) type SHA256Hash struct { - ctx C.EVP_MD_CTX + ctx *C.EVP_MD_CTX engine *Engine } @@ -41,7 +32,10 @@ func NewSHA256Hash() (*SHA256Hash, error) { return NewSHA256HashWithEngine(nil) func NewSHA256HashWithEngine(e *Engine) (*SHA256Hash, error) { hash := &SHA256Hash{engine: e} - C.EVP_MD_CTX_init(&hash.ctx) + hash.ctx = C.X_EVP_MD_CTX_new() + if hash.ctx == nil { + return nil, errors.New("openssl: sha256: unable to allocate ctx") + } runtime.SetFinalizer(hash, func(hash *SHA256Hash) { hash.Close() }) if err := hash.Reset(); err != nil { return nil, err @@ -50,11 +44,14 @@ func NewSHA256HashWithEngine(e *Engine) (*SHA256Hash, error) { } func (s *SHA256Hash) Close() { - C.EVP_MD_CTX_cleanup(&s.ctx) + if s.ctx != nil { + C.X_EVP_MD_CTX_free(s.ctx) + s.ctx = nil + } } func (s *SHA256Hash) Reset() error { - if 1 != C.EVP_DigestInit_ex(&s.ctx, C.EVP_sha256(), engineRef(s.engine)) { + if 1 != C.X_EVP_DigestInit_ex(s.ctx, C.X_EVP_sha256(), engineRef(s.engine)) { return errors.New("openssl: sha256: cannot init digest ctx") } return nil @@ -64,7 +61,7 @@ func (s *SHA256Hash) Write(p []byte) (n int, err error) { if len(p) == 0 { return 0, nil } - if 1 != C.EVP_DigestUpdate(&s.ctx, unsafe.Pointer(&p[0]), + if 1 != C.X_EVP_DigestUpdate(s.ctx, unsafe.Pointer(&p[0]), C.size_t(len(p))) { return 0, errors.New("openssl: sha256: cannot update digest") } @@ -72,7 +69,7 @@ func (s *SHA256Hash) Write(p []byte) (n int, err error) { } func (s *SHA256Hash) Sum() (result [32]byte, err error) { - if 1 != C.EVP_DigestFinal_ex(&s.ctx, + if 1 != C.X_EVP_DigestFinal_ex(s.ctx, (*C.uchar)(unsafe.Pointer(&result[0])), nil) { return result, errors.New("openssl: sha256: cannot finalize ctx") } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go index 89df88afd44..467e503ab42 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sha256_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl import ( @@ -37,7 +35,7 @@ func TestSHA256(t *testing.T) { } if expected != got { - t.Fatal("exp:%x got:%x", expected, got) + t.Fatalf("exp:%x got:%x", expected, got) } } } @@ -75,7 +73,7 @@ func TestSHA256Writer(t *testing.T) { } if got != exp { - t.Fatal("exp:%x got:%x", exp, got) + t.Fatalf("exp:%x got:%x", exp, got) } } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c new file mode 100644 index 00000000000..bb3239b0571 --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.c @@ -0,0 +1,746 @@ +/* + * Copyright (C) 2014 Space Monkey, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <string.h> + +#include "shim.h" + +#include "_cgo_export.h" + +/* + * Functions defined in other .c files + */ +extern int go_init_locks(); +extern unsigned long go_thread_id_callback(); +extern void go_thread_locking_callback(int, int, const char*, int); +static int go_write_bio_puts(BIO *b, const char *str) { + return go_write_bio_write(b, (char*)str, (int)strlen(str)); +} + +/* + * Functions to convey openssl feature defines at runtime + */ +int X_OPENSSL_NO_ECDH() { +#ifdef OPENSSL_NO_ECDH + return 1; +#else + return 0; +#endif +} + +/* + ************************************************ + * v1.1.X and later implementation + ************************************************ + */ +#if OPENSSL_VERSION_NUMBER >= 0x1010000fL + +void X_BIO_set_data(BIO* bio, void* data) { + BIO_set_data(bio, data); +} + +void* X_BIO_get_data(BIO* bio) { + return BIO_get_data(bio); +} + +EVP_MD_CTX* X_EVP_MD_CTX_new() { + return EVP_MD_CTX_new(); +} + +void X_EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_free(ctx); +} + +static int x_bio_create(BIO *b) { + BIO_set_shutdown(b, 1); + BIO_set_init(b, 1); + BIO_set_data(b, NULL); + BIO_clear_flags(b, ~0); + return 1; +} + +static int x_bio_free(BIO *b) { + return 1; +} + +static BIO_METHOD *writeBioMethod; +static BIO_METHOD *readBioMethod; + +BIO_METHOD* BIO_s_readBio() { return readBioMethod; } +BIO_METHOD* BIO_s_writeBio() { return writeBioMethod; } + +int x_bio_init_methods() { + writeBioMethod = BIO_meth_new(BIO_TYPE_SOURCE_SINK, "Go Write BIO"); + if (!writeBioMethod) { + return 1; + } + if (1 != BIO_meth_set_write(writeBioMethod, + (int (*)(BIO *, const char *, int))go_write_bio_write)) { + return 2; + } + if (1 != BIO_meth_set_puts(writeBioMethod, go_write_bio_puts)) { + return 3; + } + if (1 != BIO_meth_set_ctrl(writeBioMethod, go_write_bio_ctrl)) { + return 4; + } + if (1 != BIO_meth_set_create(writeBioMethod, x_bio_create)) { + return 5; + } + if (1 != BIO_meth_set_destroy(writeBioMethod, x_bio_free)) { + return 6; + } + + readBioMethod = BIO_meth_new(BIO_TYPE_SOURCE_SINK, "Go Read BIO"); + if (!readBioMethod) { + return 7; + } + if (1 != BIO_meth_set_read(readBioMethod, go_read_bio_read)) { + return 8; + } + if (1 != BIO_meth_set_ctrl(readBioMethod, go_read_bio_ctrl)) { + return 9; + } + if (1 != BIO_meth_set_create(readBioMethod, x_bio_create)) { + return 10; + } + if (1 != BIO_meth_set_destroy(readBioMethod, x_bio_free)) { + return 11; + } + + return 0; +} + +const EVP_MD *X_EVP_dss() { + return NULL; +} + +const EVP_MD *X_EVP_dss1() { + return NULL; +} + +const EVP_MD *X_EVP_sha() { + return NULL; +} + +int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_encrypting(ctx); +} + +int X_X509_add_ref(X509* x509) { + return X509_up_ref(x509); +} + +const ASN1_TIME *X_X509_get0_notBefore(const X509 *x) { + return X509_get0_notBefore(x); +} + +const ASN1_TIME *X_X509_get0_notAfter(const X509 *x) { + return X509_get0_notAfter(x); +} + +HMAC_CTX *X_HMAC_CTX_new(void) { + return HMAC_CTX_new(); +} + +void X_HMAC_CTX_free(HMAC_CTX *ctx) { + HMAC_CTX_free(ctx); +} + +int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u) { + return PEM_write_bio_PrivateKey_traditional(bio, key, enc, kstr, klen, cb, u); +} + +#endif + + + +/* + ************************************************ + * v1.0.X implementation + ************************************************ + */ +#if OPENSSL_VERSION_NUMBER < 0x1010000fL + +static int x_bio_create(BIO *b) { + b->shutdown = 1; + b->init = 1; + b->num = -1; + b->ptr = NULL; + b->flags = 0; + return 1; +} + +static int x_bio_free(BIO *b) { + return 1; +} + +static BIO_METHOD writeBioMethod = { + BIO_TYPE_SOURCE_SINK, + "Go Write BIO", + (int (*)(BIO *, const char *, int))go_write_bio_write, + NULL, + go_write_bio_puts, + NULL, + go_write_bio_ctrl, + x_bio_create, + x_bio_free, + NULL}; + +static BIO_METHOD* BIO_s_writeBio() { return &writeBioMethod; } + +static BIO_METHOD readBioMethod = { + BIO_TYPE_SOURCE_SINK, + "Go Read BIO", + NULL, + go_read_bio_read, + NULL, + NULL, + go_read_bio_ctrl, + x_bio_create, + x_bio_free, + NULL}; + +static BIO_METHOD* BIO_s_readBio() { return &readBioMethod; } + +int x_bio_init_methods() { + /* statically initialized above */ + return 0; +} + +void X_BIO_set_data(BIO* bio, void* data) { + bio->ptr = data; +} + +void* X_BIO_get_data(BIO* bio) { + return bio->ptr; +} + +EVP_MD_CTX* X_EVP_MD_CTX_new() { + return EVP_MD_CTX_create(); +} + +void X_EVP_MD_CTX_free(EVP_MD_CTX* ctx) { + EVP_MD_CTX_destroy(ctx); +} + +int X_X509_add_ref(X509* x509) { + CRYPTO_add(&x509->references, 1, CRYPTO_LOCK_X509); + return 1; +} + +const ASN1_TIME *X_X509_get0_notBefore(const X509 *x) { + return x->cert_info->validity->notBefore; +} + +const ASN1_TIME *X_X509_get0_notAfter(const X509 *x) { + return x->cert_info->validity->notAfter; +} + +const EVP_MD *X_EVP_dss() { + return EVP_dss(); +} + +const EVP_MD *X_EVP_dss1() { + return EVP_dss1(); +} + +const EVP_MD *X_EVP_sha() { + return EVP_sha(); +} + +int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx) { + return ctx->encrypt; +} + +HMAC_CTX *X_HMAC_CTX_new(void) { + /* v1.1.0 uses a OPENSSL_zalloc to allocate the memory which does not exist + * in previous versions. malloc+memset to get the same behavior */ + HMAC_CTX *ctx = (HMAC_CTX *)OPENSSL_malloc(sizeof(HMAC_CTX)); + if (ctx) { + memset(ctx, 0, sizeof(HMAC_CTX)); + HMAC_CTX_init(ctx); + } + return ctx; +} + +void X_HMAC_CTX_free(HMAC_CTX *ctx) { + if (ctx) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} + +int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + /* PEM_write_bio_PrivateKey always tries to use the PKCS8 format if it + * is available, instead of using the "traditional" format as stated in the + * OpenSSL man page. + * i2d_PrivateKey should give us the correct DER encoding, so we'll just + * use PEM_ASN1_write_bio directly to write the DER encoding with the correct + * type header. */ + + int ppkey_id, pkey_base_id, ppkey_flags; + const char *pinfo, *ppem_str; + char pem_type_str[80]; + + // Lookup the ASN1 method information to get the pem type + if (EVP_PKEY_asn1_get0_info(&ppkey_id, &pkey_base_id, &ppkey_flags, &pinfo, &ppem_str, key->ameth) != 1) { + return 0; + } + // Set up the PEM type string + if (BIO_snprintf(pem_type_str, 80, "%s PRIVATE KEY", ppem_str) <= 0) { + // Failed to write out the pem type string, something is really wrong. + return 0; + } + // Write out everything to the BIO + return PEM_ASN1_write_bio((i2d_of_void *)i2d_PrivateKey, + pem_type_str, bio, key, enc, kstr, klen, cb, u); +#else + return -1; +#endif +} + +#endif + + + +/* + ************************************************ + * common implementation + ************************************************ + */ + +int X_shim_init() { + int rc = 0; + + OPENSSL_config(NULL); + ENGINE_load_builtin_engines(); + SSL_load_error_strings(); + SSL_library_init(); + OpenSSL_add_all_algorithms(); + +#if OPENSSL_VERSION_NUMBER < 0x1010000fL + // Set up OPENSSL thread safety callbacks. + rc = go_init_locks(); + if (rc != 0) { + return rc; + } + CRYPTO_set_locking_callback(go_thread_locking_callback); + CRYPTO_set_id_callback(go_thread_id_callback); +#endif + rc = x_bio_init_methods(); + if (rc != 0) { + return rc; + } + + return 0; +} + +void * X_OPENSSL_malloc(size_t size) { + return OPENSSL_malloc(size); +} + +void X_OPENSSL_free(void *ref) { + OPENSSL_free(ref); +} + +long X_SSL_set_options(SSL* ssl, long options) { + return SSL_set_options(ssl, options); +} + +long X_SSL_get_options(SSL* ssl) { + return SSL_get_options(ssl); +} + +long X_SSL_clear_options(SSL* ssl, long options) { + return SSL_clear_options(ssl, options); +} + +long X_SSL_set_tlsext_host_name(SSL *ssl, const char *name) { + return SSL_set_tlsext_host_name(ssl, name); +} +const char * X_SSL_get_cipher_name(const SSL *ssl) { + return SSL_get_cipher_name(ssl); +} +int X_SSL_session_reused(SSL *ssl) { + return SSL_session_reused(ssl); +} + +int X_SSL_new_index() { + return SSL_get_ex_new_index(0, NULL, NULL, NULL, NULL); +} + +int X_SSL_verify_cb(int ok, X509_STORE_CTX* store) { + SSL* ssl = (SSL *)X509_STORE_CTX_get_ex_data(store, + SSL_get_ex_data_X509_STORE_CTX_idx()); + void* p = SSL_get_ex_data(ssl, get_ssl_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ssl_verify_cb_thunk(p, ok, store); +} + +const SSL_METHOD *X_SSLv23_method() { + return SSLv23_method(); +} + +const SSL_METHOD *X_SSLv3_method() { +#ifndef OPENSSL_NO_SSL3_METHOD + return SSLv3_method(); +#else + return NULL; +#endif +} + +const SSL_METHOD *X_TLSv1_method() { + return TLSv1_method(); +} + +const SSL_METHOD *X_TLSv1_1_method() { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return TLSv1_1_method(); +#else + return NULL; +#endif +} + +const SSL_METHOD *X_TLSv1_2_method() { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return TLSv1_2_method(); +#else + return NULL; +#endif +} + +int X_SSL_CTX_new_index() { + return SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL); +} + +long X_SSL_CTX_set_options(SSL_CTX* ctx, long options) { + return SSL_CTX_set_options(ctx, options); +} + +long X_SSL_CTX_clear_options(SSL_CTX* ctx, long options) { + return SSL_CTX_clear_options(ctx, options); +} + +long X_SSL_CTX_get_options(SSL_CTX* ctx) { + return SSL_CTX_get_options(ctx); +} + +long X_SSL_CTX_set_mode(SSL_CTX* ctx, long modes) { + return SSL_CTX_set_mode(ctx, modes); +} + +long X_SSL_CTX_get_mode(SSL_CTX* ctx) { + return SSL_CTX_get_mode(ctx); +} + +long X_SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long modes) { + return SSL_CTX_set_session_cache_mode(ctx, modes); +} + +long X_SSL_CTX_sess_set_cache_size(SSL_CTX* ctx, long t) { + return SSL_CTX_sess_set_cache_size(ctx, t); +} + +long X_SSL_CTX_sess_get_cache_size(SSL_CTX* ctx) { + return SSL_CTX_sess_get_cache_size(ctx); +} + +long X_SSL_CTX_set_timeout(SSL_CTX* ctx, long t) { + return SSL_CTX_set_timeout(ctx, t); +} + +long X_SSL_CTX_get_timeout(SSL_CTX* ctx) { + return SSL_CTX_get_timeout(ctx); +} + +long X_SSL_CTX_add_extra_chain_cert(SSL_CTX* ctx, X509 *cert) { + return SSL_CTX_add_extra_chain_cert(ctx, cert); +} + +long X_SSL_CTX_set_tlsext_servername_callback( + SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)) { + return SSL_CTX_set_tlsext_servername_callback(ctx, cb); +} + +int X_SSL_CTX_verify_cb(int ok, X509_STORE_CTX* store) { + SSL* ssl = (SSL *)X509_STORE_CTX_get_ex_data(store, + SSL_get_ex_data_X509_STORE_CTX_idx()); + SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(ssl); + void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ssl_ctx_verify_cb_thunk(p, ok, store); +} + +long X_SSL_CTX_set_tmp_dh(SSL_CTX* ctx, DH *dh) { + return SSL_CTX_set_tmp_dh(ctx, dh); +} + +long X_PEM_read_DHparams(SSL_CTX* ctx, DH *dh) { + return SSL_CTX_set_tmp_dh(ctx, dh); +} + +int X_SSL_CTX_set_tlsext_ticket_key_cb(SSL_CTX *sslctx, + int (*cb)(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)) { + return SSL_CTX_set_tlsext_ticket_key_cb(sslctx, cb); +} + +int X_SSL_CTX_ticket_key_cb(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc) { + + SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(s); + void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); + // get the pointer to the go Ctx object and pass it back into the thunk + return go_ticket_key_cb_thunk(p, s, key_name, iv, cctx, hctx, enc); +} + +int X_BIO_get_flags(BIO *b) { + return BIO_get_flags(b); +} + +void X_BIO_set_flags(BIO *b, int flags) { + return BIO_set_flags(b, flags); +} + +void X_BIO_clear_flags(BIO *b, int flags) { + BIO_clear_flags(b, flags); +} + +int X_BIO_read(BIO *b, void *buf, int len) { + return BIO_read(b, buf, len); +} + +int X_BIO_write(BIO *b, const void *buf, int len) { + return BIO_write(b, buf, len); +} + +BIO *X_BIO_new_write_bio() { + return BIO_new(BIO_s_writeBio()); +} + +BIO *X_BIO_new_read_bio() { + return BIO_new(BIO_s_readBio()); +} + +const EVP_MD *X_EVP_get_digestbyname(const char *name) { + return EVP_get_digestbyname(name); +} + +const EVP_MD *X_EVP_md_null() { + return EVP_md_null(); +} + +const EVP_MD *X_EVP_md5() { + return EVP_md5(); +} + +const EVP_MD *X_EVP_ripemd160() { + return EVP_ripemd160(); +} + +const EVP_MD *X_EVP_sha224() { + return EVP_sha224(); +} + +const EVP_MD *X_EVP_sha1() { + return EVP_sha1(); +} + +const EVP_MD *X_EVP_sha256() { + return EVP_sha256(); +} + +const EVP_MD *X_EVP_sha384() { + return EVP_sha384(); +} + +const EVP_MD *X_EVP_sha512() { + return EVP_sha512(); +} + +int X_EVP_MD_size(const EVP_MD *md) { + return EVP_MD_size(md); +} + +int X_EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl) { + return EVP_DigestInit_ex(ctx, type, impl); +} + +int X_EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, size_t cnt) { + return EVP_DigestUpdate(ctx, d, cnt); +} + +int X_EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s) { + return EVP_DigestFinal_ex(ctx, md, s); +} + +int X_EVP_SignInit(EVP_MD_CTX *ctx, const EVP_MD *type) { + return EVP_SignInit(ctx, type); +} + +int X_EVP_SignUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt) { + return EVP_SignUpdate(ctx, d, cnt); +} + +EVP_PKEY *X_EVP_PKEY_new(void) { + return EVP_PKEY_new(); +} + +void X_EVP_PKEY_free(EVP_PKEY *pkey) { + EVP_PKEY_free(pkey); +} + +int X_EVP_PKEY_size(EVP_PKEY *pkey) { + return EVP_PKEY_size(pkey); +} + +struct rsa_st *X_EVP_PKEY_get1_RSA(EVP_PKEY *pkey) { + return EVP_PKEY_get1_RSA(pkey); +} + +int X_EVP_PKEY_set1_RSA(EVP_PKEY *pkey, struct rsa_st *key) { + return EVP_PKEY_set1_RSA(pkey, key); +} + +int X_EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key) { + return EVP_PKEY_assign(pkey, type, key); +} + + + +int X_EVP_SignFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s, EVP_PKEY *pkey) { + return EVP_SignFinal(ctx, md, s, pkey); +} + +int X_EVP_VerifyInit(EVP_MD_CTX *ctx, const EVP_MD *type) { + return EVP_VerifyInit(ctx, type); +} + +int X_EVP_VerifyUpdate(EVP_MD_CTX *ctx, const void *d, + unsigned int cnt) { + return EVP_VerifyUpdate(ctx, d, cnt); +} + +int X_EVP_VerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sigbuf, unsigned int siglen, EVP_PKEY *pkey) { + return EVP_VerifyFinal(ctx, sigbuf, siglen, pkey); +} + +int X_EVP_CIPHER_block_size(EVP_CIPHER *c) { + return EVP_CIPHER_block_size(c); +} + +int X_EVP_CIPHER_key_length(EVP_CIPHER *c) { + return EVP_CIPHER_key_length(c); +} + +int X_EVP_CIPHER_iv_length(EVP_CIPHER *c) { + return EVP_CIPHER_iv_length(c); +} + +int X_EVP_CIPHER_nid(EVP_CIPHER *c) { + return EVP_CIPHER_nid(c); +} + +int X_EVP_CIPHER_CTX_block_size(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_block_size(ctx); +} + +int X_EVP_CIPHER_CTX_key_length(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_key_length(ctx); +} + +int X_EVP_CIPHER_CTX_iv_length(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_iv_length(ctx); +} + +const EVP_CIPHER *X_EVP_CIPHER_CTX_cipher(EVP_CIPHER_CTX *ctx) { + return EVP_CIPHER_CTX_cipher(ctx); +} + +#if OPENSSL_VERSION_NUMBER > 0x10000000L +#ifndef OPENSSL_NO_EC +int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid) { + return EVP_PKEY_CTX_set_ec_paramgen_curve_nid(ctx, nid); +} +#else +int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid) { + return -2; // not supported +} +#endif +#endif + +// END HERE + +size_t X_HMAC_size(const HMAC_CTX *e) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_size(e); +#else + return 0; +#endif +} + +int X_HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md, ENGINE *impl) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Init_ex(ctx, key, len, md, impl); +#else + return -1; +#endif +} + +int X_HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, size_t len) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Update(ctx, data, len); +#else + return -1; +#endif +} + +int X_HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len) { +#if OPENSSL_VERSION_NUMBER > 0x10000000L + return HMAC_Final(ctx, md, len); +#else + return -1; +#endif +} + +int X_sk_X509_num(STACK_OF(X509) *sk) { + return sk_X509_num(sk); +} + +X509 *X_sk_X509_value(STACK_OF(X509)* sk, int i) { + return sk_X509_value(sk, i); +} + +#ifdef OPENSSL_FIPS +int X_FIPS_mode(void) { + return FIPS_mode(); +} +int X_FIPS_mode_set(int r) { + return FIPS_mode_set(r); +} +#else +int X_FIPS_mode(void) { + return 0; +} +int X_FIPS_mode_set(int r) { + return 0; +} +#endif diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h new file mode 100644 index 00000000000..1e9ddebe8ab --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/shim.h @@ -0,0 +1,181 @@ +/* + * Copyright (C) 2014 Space Monkey, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include <stdlib.h> +#include <string.h> + +#include <openssl/opensslconf.h> + +#include <openssl/bio.h> +#include <openssl/conf.h> +#include <openssl/crypto.h> +#include <openssl/dh.h> +#include <openssl/engine.h> +#include <openssl/err.h> +#include <openssl/evp.h> +#include <openssl/hmac.h> +#include <openssl/pem.h> +#include <openssl/ssl.h> +#include <openssl/x509v3.h> + +#ifndef SSL_MODE_RELEASE_BUFFERS +#define SSL_MODE_RELEASE_BUFFERS 0 +#endif + +#ifndef SSL_OP_NO_COMPRESSION +#define SSL_OP_NO_COMPRESSION 0 +#endif + +#ifndef SSL_OP_NO_TLSv1_1 +#define SSL_OP_NO_TLSv1_1 0 +#endif + +#ifndef SSL_OP_NO_TLSv1_2 +#define SSL_OP_NO_TLSv1_2 0 +#endif + +/* shim methods */ +extern int X_shim_init(); + +/* Feature detection methods */ +extern int X_OPENSSL_NO_ECDH(); + +/* Library methods */ +extern void X_OPENSSL_free(void *ref); +extern void *X_OPENSSL_malloc(size_t size); + +/* SSL methods */ +extern long X_SSL_set_options(SSL* ssl, long options); +extern long X_SSL_get_options(SSL* ssl); +extern long X_SSL_clear_options(SSL* ssl, long options); +extern long X_SSL_set_tlsext_host_name(SSL *ssl, const char *name); +extern const char * X_SSL_get_cipher_name(const SSL *ssl); +extern int X_SSL_session_reused(SSL *ssl); +extern int X_SSL_new_index(); + +extern const SSL_METHOD *X_SSLv23_method(); +extern const SSL_METHOD *X_SSLv3_method(); +extern const SSL_METHOD *X_TLSv1_method(); +extern const SSL_METHOD *X_TLSv1_1_method(); +extern const SSL_METHOD *X_TLSv1_2_method(); + +#if defined SSL_CTRL_SET_TLSEXT_HOSTNAME +extern int sni_cb(SSL *ssl_conn, int *ad, void *arg); +#endif +extern int X_SSL_verify_cb(int ok, X509_STORE_CTX* store); + +/* SSL_CTX methods */ +extern int X_SSL_CTX_new_index(); +extern long X_SSL_CTX_set_options(SSL_CTX* ctx, long options); +extern long X_SSL_CTX_clear_options(SSL_CTX* ctx, long options); +extern long X_SSL_CTX_get_options(SSL_CTX* ctx); +extern long X_SSL_CTX_set_mode(SSL_CTX* ctx, long modes); +extern long X_SSL_CTX_get_mode(SSL_CTX* ctx); +extern long X_SSL_CTX_set_session_cache_mode(SSL_CTX* ctx, long modes); +extern long X_SSL_CTX_sess_set_cache_size(SSL_CTX* ctx, long t); +extern long X_SSL_CTX_sess_get_cache_size(SSL_CTX* ctx); +extern long X_SSL_CTX_set_timeout(SSL_CTX* ctx, long t); +extern long X_SSL_CTX_get_timeout(SSL_CTX* ctx); +extern long X_SSL_CTX_add_extra_chain_cert(SSL_CTX* ctx, X509 *cert); +extern long X_SSL_CTX_set_tlsext_servername_callback(SSL_CTX* ctx, int (*cb)(SSL *con, int *ad, void *args)); +extern int X_SSL_CTX_verify_cb(int ok, X509_STORE_CTX* store); +extern long X_SSL_CTX_set_tmp_dh(SSL_CTX* ctx, DH *dh); +extern long X_PEM_read_DHparams(SSL_CTX* ctx, DH *dh); +extern int X_SSL_CTX_set_tlsext_ticket_key_cb(SSL_CTX *sslctx, + int (*cb)(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)); +extern int X_SSL_CTX_ticket_key_cb(SSL *s, unsigned char key_name[16], + unsigned char iv[EVP_MAX_IV_LENGTH], + EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc); + +/* BIO methods */ +extern int X_BIO_get_flags(BIO *b); +extern void X_BIO_set_flags(BIO *bio, int flags); +extern void X_BIO_clear_flags(BIO *bio, int flags); +extern void X_BIO_set_data(BIO *bio, void* data); +extern void *X_BIO_get_data(BIO *bio); +extern int X_BIO_read(BIO *b, void *buf, int len); +extern int X_BIO_write(BIO *b, const void *buf, int len); +extern BIO *X_BIO_new_write_bio(); +extern BIO *X_BIO_new_read_bio(); + +/* EVP methods */ +extern const EVP_MD *X_EVP_get_digestbyname(const char *name); +extern EVP_MD_CTX *X_EVP_MD_CTX_new(); +extern void X_EVP_MD_CTX_free(EVP_MD_CTX *ctx); +extern const EVP_MD *X_EVP_md_null(); +extern const EVP_MD *X_EVP_md5(); +extern const EVP_MD *X_EVP_sha(); +extern const EVP_MD *X_EVP_sha1(); +extern const EVP_MD *X_EVP_dss(); +extern const EVP_MD *X_EVP_dss1(); +extern const EVP_MD *X_EVP_ripemd160(); +extern const EVP_MD *X_EVP_sha224(); +extern const EVP_MD *X_EVP_sha256(); +extern const EVP_MD *X_EVP_sha384(); +extern const EVP_MD *X_EVP_sha512(); +extern int X_EVP_MD_size(const EVP_MD *md); +extern int X_EVP_DigestInit_ex(EVP_MD_CTX *ctx, const EVP_MD *type, ENGINE *impl); +extern int X_EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *d, size_t cnt); +extern int X_EVP_DigestFinal_ex(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s); +extern int X_EVP_SignInit(EVP_MD_CTX *ctx, const EVP_MD *type); +extern int X_EVP_SignUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt); +extern EVP_PKEY *X_EVP_PKEY_new(void); +extern void X_EVP_PKEY_free(EVP_PKEY *pkey); +extern int X_EVP_PKEY_size(EVP_PKEY *pkey); +extern struct rsa_st *X_EVP_PKEY_get1_RSA(EVP_PKEY *pkey); +extern int X_EVP_PKEY_set1_RSA(EVP_PKEY *pkey, struct rsa_st *key); +extern int X_EVP_PKEY_assign_charp(EVP_PKEY *pkey, int type, char *key); +extern int X_EVP_SignFinal(EVP_MD_CTX *ctx, unsigned char *md, unsigned int *s, EVP_PKEY *pkey); +extern int X_EVP_VerifyInit(EVP_MD_CTX *ctx, const EVP_MD *type); +extern int X_EVP_VerifyUpdate(EVP_MD_CTX *ctx, const void *d, unsigned int cnt); +extern int X_EVP_VerifyFinal(EVP_MD_CTX *ctx, const unsigned char *sigbuf, unsigned int siglen, EVP_PKEY *pkey); +extern int X_EVP_CIPHER_block_size(EVP_CIPHER *c); +extern int X_EVP_CIPHER_key_length(EVP_CIPHER *c); +extern int X_EVP_CIPHER_iv_length(EVP_CIPHER *c); +extern int X_EVP_CIPHER_nid(EVP_CIPHER *c); +extern int X_EVP_CIPHER_CTX_block_size(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_key_length(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_iv_length(EVP_CIPHER_CTX *ctx); +extern const EVP_CIPHER *X_EVP_CIPHER_CTX_cipher(EVP_CIPHER_CTX *ctx); +extern int X_EVP_CIPHER_CTX_encrypting(const EVP_CIPHER_CTX *ctx); +#if OPENSSL_VERSION_NUMBER > 0x10000000L +extern int X_EVP_PKEY_CTX_set_ec_paramgen_curve_nid(EVP_PKEY_CTX *ctx, int nid); +#endif + +/* HMAC methods */ +extern size_t X_HMAC_size(const HMAC_CTX *e); +extern HMAC_CTX *X_HMAC_CTX_new(void); +extern void X_HMAC_CTX_free(HMAC_CTX *ctx); +extern int X_HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int len, const EVP_MD *md, ENGINE *impl); +extern int X_HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, size_t len); +extern int X_HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len); + +/* X509 methods */ +extern int X_X509_add_ref(X509* x509); +extern const ASN1_TIME *X_X509_get0_notBefore(const X509 *x); +extern const ASN1_TIME *X_X509_get0_notAfter(const X509 *x); +extern int X_sk_X509_num(STACK_OF(X509) *sk); +extern X509 *X_sk_X509_value(STACK_OF(X509)* sk, int i); + +/* PEM methods */ +extern int X_PEM_write_bio_PrivateKey_traditional(BIO *bio, EVP_PKEY *key, const EVP_CIPHER *enc, unsigned char *kstr, int klen, pem_password_cb *cb, void *u); + +/* FIPS methods */ +extern int X_FIPS_mode(void); +extern int X_FIPS_mode_set(int r); diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c index 5398da869b8..f9e8d16b0e3 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni.c @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go index ee3b1a8bbaf..09e831a45c9 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/sni_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go index 3cc630601d3..117c30c0f99 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,30 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <openssl/crypto.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/conf.h> - -static long SSL_set_options_not_a_macro(SSL* ssl, long options) { - return SSL_set_options(ssl, options); -} - -static long SSL_get_options_not_a_macro(SSL* ssl) { - return SSL_get_options(ssl); -} - -static long SSL_clear_options_not_a_macro(SSL* ssl, long options) { - return SSL_clear_options(ssl, options); -} - -extern int verify_ssl_cb(int ok, X509_STORE_CTX* store); -*/ +// #include "shim.h" import "C" import ( @@ -53,7 +32,7 @@ const ( ) var ( - ssl_idx = C.SSL_get_ex_new_index(0, nil, nil, nil, nil) + ssl_idx = C.X_SSL_new_index() ) //export get_ssl_idx @@ -66,8 +45,8 @@ type SSL struct { verify_cb VerifyCallback } -//export verify_ssl_cb_thunk -func verify_ssl_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { +//export go_ssl_verify_cb_thunk +func go_ssl_verify_cb_thunk(p unsafe.Pointer, ok C.int, ctx *C.X509_STORE_CTX) C.int { defer func() { if err := recover(); err != nil { logger.Critf("openssl: verify callback panic'd: %v", err) @@ -96,19 +75,19 @@ func (s *SSL) GetServername() string { // GetOptions returns SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) GetOptions() Options { - return Options(C.SSL_get_options_not_a_macro(s.ssl)) + return Options(C.X_SSL_get_options(s.ssl)) } // SetOptions sets SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) SetOptions(options Options) Options { - return Options(C.SSL_set_options_not_a_macro(s.ssl, C.long(options))) + return Options(C.X_SSL_set_options(s.ssl, C.long(options))) } // ClearOptions clear SSL options. See // https://www.openssl.org/docs/ssl/SSL_CTX_set_options.html func (s *SSL) ClearOptions(options Options) Options { - return Options(C.SSL_clear_options_not_a_macro(s.ssl, C.long(options))) + return Options(C.X_SSL_clear_options(s.ssl, C.long(options))) } // SetVerify controls peer verification settings. See @@ -116,7 +95,7 @@ func (s *SSL) ClearOptions(options Options) Options { func (s *SSL) SetVerify(options VerifyOptions, verify_cb VerifyCallback) { s.verify_cb = verify_cb if verify_cb != nil { - C.SSL_set_verify(s.ssl, C.int(options), (*[0]byte)(C.verify_ssl_cb)) + C.SSL_set_verify(s.ssl, C.int(options), (*[0]byte)(C.X_SSL_verify_cb)) } else { C.SSL_set_verify(s.ssl, C.int(options), nil) } @@ -131,7 +110,7 @@ func (s *SSL) SetVerifyMode(options VerifyOptions) { // SetVerifyCallback controls peer verification setting. See // http://www.openssl.org/docs/ssl/SSL_CTX_set_verify.html func (s *SSL) SetVerifyCallback(verify_cb VerifyCallback) { - s.SetVerify(s.VerifyMode(), s.verify_cb) + s.SetVerify(s.VerifyMode(), verify_cb) } // GetVerifyCallback returns callback function. See diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go index 0c088c2eed0..fe2e0de4592 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/ssl_test.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -81,6 +81,29 @@ ucCCa4lOGgPtXJ0Qf1c8yq5vh4yqkQjrgUTkr+CFDGR6y4CxmNDQxEMYIajaIiSY qmgvgyRayemfO2zR0CPgC6wSoGBth+xW6g+WA8y0z76ZSaWpFi8lVM4= -----END RSA PRIVATE KEY----- `) + prime256v1KeyBytes = []byte(`-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIB/XL0zZSsAu+IQF1AI/nRneabb2S126WFlvvhzmYr1KoAoGCCqGSM49 +AwEHoUQDQgAESSFGWwF6W1hoatKGPPorh4+ipyk0FqpiWdiH+4jIiU39qtOeZGSh +1QgSbzfdHxvoYI0FXM+mqE7wec0kIvrrHw== +-----END EC PRIVATE KEY----- +`) + prime256v1CertBytes = []byte(`-----BEGIN CERTIFICATE----- +MIIChTCCAiqgAwIBAgIJAOQII2LQl4uxMAoGCCqGSM49BAMCMIGcMQswCQYDVQQG +EwJVUzEPMA0GA1UECAwGS2Fuc2FzMRAwDgYDVQQHDAdOb3doZXJlMR8wHQYDVQQK +DBZGYWtlIENlcnRpZmljYXRlcywgSW5jMUkwRwYDVQQDDEBhMWJkZDVmZjg5ZjQy +N2IwZmNiOTdlNDMyZTY5Nzg2NjI2ODJhMWUyNzM4MDhkODE0ZWJiZjY4ODBlYzA3 +NDljMB4XDTE3MTIxNTIwNDU1MVoXDTI3MTIxMzIwNDU1MVowgZwxCzAJBgNVBAYT +AlVTMQ8wDQYDVQQIDAZLYW5zYXMxEDAOBgNVBAcMB05vd2hlcmUxHzAdBgNVBAoM +FkZha2UgQ2VydGlmaWNhdGVzLCBJbmMxSTBHBgNVBAMMQGExYmRkNWZmODlmNDI3 +YjBmY2I5N2U0MzJlNjk3ODY2MjY4MmExZTI3MzgwOGQ4MTRlYmJmNjg4MGVjMDc0 +OWMwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARJIUZbAXpbWGhq0oY8+iuHj6Kn +KTQWqmJZ2If7iMiJTf2q055kZKHVCBJvN90fG+hgjQVcz6aoTvB5zSQi+usfo1Mw +UTAdBgNVHQ4EFgQUfRYAFhlGM1wzvusyGrm26Vrbqm4wHwYDVR0jBBgwFoAUfRYA +FhlGM1wzvusyGrm26Vrbqm4wDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQDAgNJ +ADBGAiEA6PWNjm4B6zs3Wcha9qyDdfo1ILhHfk9rZEAGrnfyc2UCIQD1IDVJUkI4 +J/QVoOtP5DOdRPs/3XFy0Bk0qH+Uj5D7LQ== +-----END CERTIFICATE----- +`) ) func NetPipe(t testing.TB) (net.Conn, net.Conn) { diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go index 23dc3e08305..a064d38592f 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/tickets.go @@ -1,4 +1,4 @@ -// Copyright (C) 2015 Space Monkey, Inc. +// Copyright (C) 2017. See AUTHORS. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,26 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -// +build cgo - package openssl -/* -#include <openssl/ssl.h> -#include <openssl/evp.h> - -static int SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(SSL_CTX *sslctx, - int (*cb)(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *ctx, HMAC_CTX *hctx, int enc)) { - - return SSL_CTX_set_tlsext_ticket_key_cb(sslctx, cb); -} - -extern int ticket_key_cb(SSL *s, unsigned char key_name[16], - unsigned char iv[EVP_MAX_IV_LENGTH], - EVP_CIPHER_CTX *cctx, HMAC_CTX *hctx, int enc); -*/ +// #include "shim.h" import "C" import ( @@ -131,8 +114,8 @@ const ( ticket_req_lookupSession = 0 ) -//export ticket_key_cb_thunk -func ticket_key_cb_thunk(p unsafe.Pointer, s *C.SSL, key_name *C.uchar, +//export go_ticket_key_cb_thunk +func go_ticket_key_cb_thunk(p unsafe.Pointer, s *C.SSL, key_name *C.uchar, iv *C.uchar, cctx *C.EVP_CIPHER_CTX, hctx *C.HMAC_CTX, enc C.int) C.int { // no panic's allowed. it's super hard to guarantee any state at this point @@ -231,9 +214,9 @@ func (c *Ctx) SetTicketStore(store *TicketStore) { c.ticket_store = store if store == nil { - C.SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(c.ctx, nil) + C.X_SSL_CTX_set_tlsext_ticket_key_cb(c.ctx, nil) } else { - C.SSL_CTX_set_tlsext_ticket_key_cb_not_a_macro(c.ctx, - (*[0]byte)(C.ticket_key_cb)) + C.X_SSL_CTX_set_tlsext_ticket_key_cb(c.ctx, + (*[0]byte)(C.X_SSL_CTX_ticket_key_cb)) } } diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c deleted file mode 100644 index d55866c4cf0..00000000000 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/verify.c +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (C) 2014 Space Monkey, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <openssl/ssl.h> -#include "_cgo_export.h" - -int verify_cb(int ok, X509_STORE_CTX* store) { - SSL* ssl = (SSL *)X509_STORE_CTX_get_app_data(store); - SSL_CTX* ssl_ctx = SSL_get_SSL_CTX(ssl); - void* p = SSL_CTX_get_ex_data(ssl_ctx, get_ssl_ctx_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return verify_cb_thunk(p, ok, store); -} - -int verify_ssl_cb(int ok, X509_STORE_CTX* store) { - SSL* ssl = (SSL *)X509_STORE_CTX_get_app_data(store); - void* p = SSL_get_ex_data(ssl, get_ssl_idx()); - // get the pointer to the go Ctx object and pass it back into the thunk - return verify_ssl_cb_thunk(p, ok, store); -} diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go index 8f3d392cde8..86501c696d6 100644 --- a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version.go @@ -17,6 +17,11 @@ package openssl // #include <openssl/opensslv.h> +// #include <openssl/crypto.h> import "C" -const Version string = C.OPENSSL_VERSION_TEXT +const BuildVersion string = C.OPENSSL_VERSION_TEXT + +var Version string = C.GoString(C.SSLeay_version(C.SSLEAY_VERSION)) + +var VersionNumber uint32 = uint32(C.SSLeay()) diff --git a/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go new file mode 100644 index 00000000000..9877fb9c7dd --- /dev/null +++ b/src/mongo/gotools/vendor/src/github.com/10gen/openssl/version_test.go @@ -0,0 +1,29 @@ +// Copyright (C) MongoDB, Inc. 2018-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package openssl + +import ( + "testing" +) + +func TestVersion(t *testing.T) { + v := Version + b := BuildVersion + x := VersionNumber + if len(v) == 0 { + t.Fatal("Version string is empty") + } + if len(b) == 0 { + t.Fatal("BuildVersion string is empty") + } + if x == 0 { + t.Fatal("VersionNumber is zero") + } + t.Logf("Built with headers from: %s", BuildVersion) + t.Logf(" Tests linked against: %s", Version) + t.Logf(" Linked hex version is: %x", VersionNumber) +} diff --git a/src/mongo/rpc/metadata/client_metadata_ismaster.cpp b/src/mongo/rpc/metadata/client_metadata_ismaster.cpp index 234f61c78e8..8a6e84f2eef 100644 --- a/src/mongo/rpc/metadata/client_metadata_ismaster.cpp +++ b/src/mongo/rpc/metadata/client_metadata_ismaster.cpp @@ -65,17 +65,30 @@ const boost::optional<ClientMetadata>& ClientMetadataIsMasterState::getClientMet return _clientMetadata; } -void ClientMetadataIsMasterState::setClientMetadata( - Client* client, boost::optional<ClientMetadata> clientMetadata) { +void ClientMetadataIsMasterState::setClientMetadata(Client* client, + boost::optional<ClientMetadata> clientMetadata, + bool setViaMetadata) { auto& state = get(client); stdx::lock_guard<Client> lk(*client); state._clientMetadata = std::move(clientMetadata); + state._setViaMetadata = setViaMetadata; } Status ClientMetadataIsMasterState::readFromMetadata(OperationContext* txn, BSONElement& element) { + auto& clientMetadataIsMasterState = ClientMetadataIsMasterState::get(txn->getClient()); + + // If client metadata is not present in network requests, reset the in-memory metadata to be + // blank so that the wrong + // app name is not propagated. if (element.eoo()) { + auto client = txn->getClient(); + + if (clientMetadataIsMasterState._setViaMetadata && !client->isInDirectClient()) { + clientMetadataIsMasterState.setClientMetadata(client, boost::none, true); + } + return Status::OK(); } @@ -85,10 +98,8 @@ Status ClientMetadataIsMasterState::readFromMetadata(OperationContext* txn, BSON return swParseClientMetadata.getStatus(); } - auto& clientMetadataIsMasterState = ClientMetadataIsMasterState::get(txn->getClient()); - - clientMetadataIsMasterState.setClientMetadata(txn->getClient(), - std::move(swParseClientMetadata.getValue())); + clientMetadataIsMasterState.setClientMetadata( + txn->getClient(), std::move(swParseClientMetadata.getValue()), true); return Status::OK(); } diff --git a/src/mongo/rpc/metadata/client_metadata_ismaster.h b/src/mongo/rpc/metadata/client_metadata_ismaster.h index 305018343d0..73760abfead 100644 --- a/src/mongo/rpc/metadata/client_metadata_ismaster.h +++ b/src/mongo/rpc/metadata/client_metadata_ismaster.h @@ -59,7 +59,9 @@ public: /** * Set the optional client metadata object. */ - static void setClientMetadata(Client* client, boost::optional<ClientMetadata> clientMetadata); + static void setClientMetadata(Client* client, + boost::optional<ClientMetadata> clientMetadata, + bool setViaMetadata = false); /** * Check a flag to indicate that isMaster has been seen for this Client. @@ -102,6 +104,11 @@ private: // Thread-Safety: // None - must be only be read and written from the thread owning "Client". bool _hasSeenIsMaster{false}; + + // Indicates whether we have set isMaster based on metadata or via isMaster + // Thread-Safety: + // None - must be only be read and written from the thread owning "Client". + bool _setViaMetadata{false}; }; } // namespace mongo diff --git a/src/mongo/s/chunk_version.h b/src/mongo/s/chunk_version.h index 05517ffb609..54c1ca4f576 100644 --- a/src/mongo/s/chunk_version.h +++ b/src/mongo/s/chunk_version.h @@ -59,7 +59,7 @@ public: ChunkVersion() : _combined(0), _epoch(OID()) {} - ChunkVersion(int major, int minor, const OID& epoch) + ChunkVersion(uint32_t major, uint32_t minor, const OID& epoch) : _combined(static_cast<uint64_t>(minor) | (static_cast<uint64_t>(major) << 32)), _epoch(epoch) {} @@ -148,12 +148,12 @@ public: return _combined > 0; } - int majorVersion() const { + uint32_t majorVersion() const { return _combined >> 32; } - int minorVersion() const { - return _combined & 0xFFFF; + uint32_t minorVersion() const { + return _combined & 0xFFFFFFFF; } OID epoch() const { diff --git a/src/mongo/s/chunk_version_test.cpp b/src/mongo/s/chunk_version_test.cpp index 4bea7f466bd..51c7f9d1cf5 100644 --- a/src/mongo/s/chunk_version_test.cpp +++ b/src/mongo/s/chunk_version_test.cpp @@ -28,6 +28,8 @@ #include "mongo/platform/basic.h" +#include <limits> + #include "mongo/db/jsobj.h" #include "mongo/s/chunk_version.h" #include "mongo/unittest/unittest.h" @@ -44,16 +46,16 @@ TEST(Parsing, EpochIsOptional) { ASSERT(canParse); ASSERT(chunkVersionComplete.epoch().isSet()); ASSERT(chunkVersionComplete.epoch() == oid); - ASSERT_EQ(2, chunkVersionComplete.majorVersion()); - ASSERT_EQ(3, chunkVersionComplete.minorVersion()); + ASSERT_EQ(2u, chunkVersionComplete.majorVersion()); + ASSERT_EQ(3u, chunkVersionComplete.minorVersion()); canParse = false; ChunkVersion chunkVersionNoEpoch = ChunkVersion::fromBSON(BSON("lastmod" << Timestamp(Seconds(3), 4)), "lastmod", &canParse); ASSERT(canParse); ASSERT(!chunkVersionNoEpoch.epoch().isSet()); - ASSERT_EQ(3, chunkVersionNoEpoch.majorVersion()); - ASSERT_EQ(4, chunkVersionNoEpoch.minorVersion()); + ASSERT_EQ(3u, chunkVersionNoEpoch.majorVersion()); + ASSERT_EQ(4u, chunkVersionNoEpoch.minorVersion()); } TEST(Comparison, StrictEqual) { @@ -83,5 +85,16 @@ TEST(Comparison, OlderThan) { ASSERT(!ChunkVersion(3, 1, epoch).isOlderThan(ChunkVersion(3, 1, epoch))); } +TEST(ChunkVersionConstruction, CreateWithLargeValues) { + const auto minorVersion = std::numeric_limits<uint32_t>::max(); + const uint32_t majorVersion = 1 << 24; + const auto epoch = OID::gen(); + + ChunkVersion version(majorVersion, minorVersion, epoch); + ASSERT_EQ(majorVersion, version.majorVersion()); + ASSERT_EQ(minorVersion, version.minorVersion()); + ASSERT_EQ(epoch, version.epoch()); +} + } // unnamed namespace } // namespace mongo diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 1e0ea51ed4b..00c9141f2d6 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -44,6 +44,7 @@ #include "mongo/db/operation_context.h" #include "mongo/db/query/query_request.h" #include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/server_parameters.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -72,6 +73,11 @@ const BSONObj kReplMetadata(BSON(rpc::kReplSetMetadataFieldName << 1)); // Allow the command to be executed on a secondary (see ServerSelectionMetadata). const BSONObj kSecondaryOkMetadata{rpc::ServerSelectionMetadata(true, boost::none).toBSON()}; +constexpr bool internalProhibitShardOperationRetryByDefault = false; +MONGO_EXPORT_SERVER_PARAMETER(internalProhibitShardOperationRetry, + bool, + internalProhibitShardOperationRetryByDefault); + /** * Returns a new BSONObj describing the same command and arguments as 'cmdObj', but with maxTimeMS * replaced by maxTimeMSOverride (or removed if maxTimeMSOverride is Milliseconds::max()). @@ -104,6 +110,10 @@ ShardRemote::ShardRemote(const ShardId& id, ShardRemote::~ShardRemote() = default; bool ShardRemote::isRetriableError(ErrorCodes::Error code, RetryPolicy options) { + if (internalProhibitShardOperationRetry.load()) { + return false; + } + if (options == RetryPolicy::kNoRetry) { return false; } diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 90c59f0ee55..30b1ff718f2 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -213,6 +213,9 @@ char* shellReadline(const char* prompt, int handlesigint = 0) { } void setupSignals() { +#ifndef _WIN32 + signal(SIGHUP, quitNicely); +#endif signal(SIGINT, quitNicely); } diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js index 59eb411cc3b..0e677ee79b1 100644 --- a/src/mongo/shell/replsettest.js +++ b/src/mongo/shell/replsettest.js @@ -64,9 +64,6 @@ * nodes {Array.<Mongo>} - connection to replica set members */ -/* Global default timeout variable */ -const kReplDefaultTimeoutMS = 10 * 60 * 1000; - var ReplSetTest = function(opts) { 'use strict'; @@ -91,7 +88,9 @@ var ReplSetTest = function(opts) { var _causalConsistency; - this.kDefaultTimeoutMS = kReplDefaultTimeoutMS; + // Some code still references kDefaultTimeoutMS as a (non-static) member variable, so make sure + // it's still accessible that way. + this.kDefaultTimeoutMS = ReplSetTest.kDefaultTimeoutMS; var oplogName = 'oplog.rs'; // Publicly exposed variables @@ -1290,6 +1289,9 @@ var ReplSetTest = function(opts) { // liveNodes must have been populated. var primary = rst.liveNodes.master; var combinedDBs = new Set(primary.getDBNames()); + // replSetConfig will be undefined for master/slave passthrough. + const replSetConfig = + rst.getReplSetConfigFromNode ? rst.getReplSetConfigFromNode() : undefined; rst.getSecondaries().forEach(secondary => { secondary.getDBNames().forEach(dbName => combinedDBs.add(dbName)); @@ -1378,8 +1380,10 @@ var ReplSetTest = function(opts) { // Check that the following collection stats are the same across replica set // members: // capped - // nindexes + // nindexes, except on nodes with buildIndexes: false // ns + const hasSecondaryIndexes = !replSetConfig || + replSetConfig.members[rst.getNodeId(secondary)].buildIndexes !== false; primaryCollections.forEach(collName => { var primaryCollStats = primary.getDB(dbName).runCommand({collStats: collName}); @@ -1389,7 +1393,8 @@ var ReplSetTest = function(opts) { assert.commandWorked(secondaryCollStats); if (primaryCollStats.capped !== secondaryCollStats.capped || - primaryCollStats.nindexes !== secondaryCollStats.nindexes || + (hasSecondaryIndexes && + primaryCollStats.nindexes !== secondaryCollStats.nindexes) || primaryCollStats.ns !== secondaryCollStats.ns) { print(msgPrefix + ', the primary and secondary have different stats for the ' + @@ -1887,17 +1892,21 @@ var ReplSetTest = function(opts) { } if (typeof opts === 'string' || opts instanceof String) { - _constructFromExistingSeedNode(opts); + retryOnNetworkError(function() { + // The primary may unexpectedly step down during startup if under heavy load + // and too slowly processing heartbeats. When it steps down, it closes all of + // its connections. + _constructFromExistingSeedNode(opts); + }, 10); } else { _constructStartNewInstances(opts); } }; /** - * Declare kDefaultTimeoutMS as a static property so we don't have to initialize - * a ReplSetTest object to use it. + * Global default timeout (10 minutes). */ -ReplSetTest.kDefaultTimeoutMS = kReplDefaultTimeoutMS; +ReplSetTest.kDefaultTimeoutMS = 10 * 60 * 1000; /** * Set of states that the replica set can be in. Used for the wait functions. diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js index a1bb04c6bfb..d45beb0d675 100644 --- a/src/mongo/shell/utils.js +++ b/src/mongo/shell/utils.js @@ -37,6 +37,32 @@ function _getErrorWithCode(codeOrObj, message) { return e; } +/** + * Executes the specified function and retries it if it fails due to exception related to network + * error. If it exhausts the number of allowed retries, it simply throws the last exception. + * + * Returns the return value of the input call. + */ +function retryOnNetworkError(func, numRetries, sleepMs) { + numRetries = numRetries || 1; + sleepMs = sleepMs || 1000; + + while (true) { + try { + return func(); + } catch (e) { + if (isNetworkError(e) && numRetries > 0) { + print("Network error occurred and the call will be retried: " + + tojson({error: e.toString(), stack: e.stack})); + numRetries--; + sleep(sleepMs); + } else { + throw e; + } + } + } +} + // Checks if a javascript exception is a network error. function isNetworkError(error) { return error.message.indexOf("error doing query") >= 0 || diff --git a/src/mongo/transport/service_entry_point_test_suite.cpp b/src/mongo/transport/service_entry_point_test_suite.cpp index 5d2945919e3..8cbd86bae06 100644 --- a/src/mongo/transport/service_entry_point_test_suite.cpp +++ b/src/mongo/transport/service_entry_point_test_suite.cpp @@ -133,7 +133,9 @@ void ServiceEntryPointTestSuite::MockTLHarness::asyncWait(Ticket&& ticket, SSLPeerInfo ServiceEntryPointTestSuite::MockTLHarness::getX509PeerInfo( const ConstSessionHandle& session) const { - return SSLPeerInfo("mock", stdx::unordered_set<RoleName>{}); + auto name = SSLX509Name(std::vector<std::vector<SSLX509Name::Entry>>( + {{{kOID_CommonName.toString(), 19 /* Printable String */, "mock"}}})); + return SSLPeerInfo(name, stdx::unordered_set<RoleName>{}); } TransportLayer::Stats ServiceEntryPointTestSuite::MockTLHarness::sessionStats() { diff --git a/src/mongo/transport/transport_layer_legacy.cpp b/src/mongo/transport/transport_layer_legacy.cpp index 680853ffe04..cc998286aae 100644 --- a/src/mongo/transport/transport_layer_legacy.cpp +++ b/src/mongo/transport/transport_layer_legacy.cpp @@ -324,7 +324,7 @@ Status TransportLayerLegacy::_runTicket(Ticket ticket) { // If we didn't have an X509 subject name, see if we have one now if (!conn->sslPeerInfo) { auto info = conn->amp->getX509PeerInfo(); - if (info.subjectName != "") { + if (!info.subjectName.empty()) { conn->sslPeerInfo = info; } } diff --git a/src/mongo/util/concurrency/notification.h b/src/mongo/util/concurrency/notification.h index d24fc84e5f9..25f0c65f187 100644 --- a/src/mongo/util/concurrency/notification.h +++ b/src/mongo/util/concurrency/notification.h @@ -102,12 +102,10 @@ public: * set (in which case a subsequent call to get is guaranteed to not block) or false otherwise. * If the wait is interrupted, throws an exception. */ - bool waitFor(OperationContext* txn, Microseconds waitTimeout) { - const auto waitDeadline = Date_t::now() + waitTimeout; - + bool waitFor(OperationContext* txn, Milliseconds waitTimeout) { stdx::unique_lock<stdx::mutex> lock(_mutex); - return _condVar.wait_until( - lock, waitDeadline.toSystemTimePoint(), [&]() { return !!_value; }); + return txn->waitForConditionOrInterruptFor( + _condVar, lock, waitTimeout, [&]() { return !!_value; }); } private: @@ -137,7 +135,7 @@ public: _notification.set(true); } - bool waitFor(OperationContext* txn, Microseconds waitTimeout) { + bool waitFor(OperationContext* txn, Milliseconds waitTimeout) { return _notification.waitFor(txn, waitTimeout); } diff --git a/src/mongo/util/exception_filter_win32.cpp b/src/mongo/util/exception_filter_win32.cpp index db0e3e9bb56..30d904a88cf 100644 --- a/src/mongo/util/exception_filter_win32.cpp +++ b/src/mongo/util/exception_filter_win32.cpp @@ -129,8 +129,8 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { sizeof(addressString), "0x%p", excPointers->ExceptionRecord->ExceptionAddress); - log() << "*** unhandled exception " << exceptionString << " at " << addressString - << ", terminating"; + severe() << "*** unhandled exception " << exceptionString << " at " << addressString + << ", terminating"; if (excPointers->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { ULONG acType = excPointers->ExceptionRecord->ExceptionInformation[0]; const char* acTypeString; @@ -152,10 +152,10 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { sizeof(addressString), " 0x%p", excPointers->ExceptionRecord->ExceptionInformation[1]); - log() << "*** access violation was a " << acTypeString << addressString; + severe() << "*** access violation was a " << acTypeString << addressString; } - log() << "*** stack trace for unhandled exception:"; + severe() << "*** stack trace for unhandled exception:"; // Create a copy of context record because printWindowsStackTrace will mutate it. CONTEXT contextCopy(*(excPointers->ContextRecord)); @@ -166,7 +166,7 @@ LONG WINAPI exceptionFilter(struct _EXCEPTION_POINTERS* excPointers) { // Don't go through normal shutdown procedure. It may make things worse. // Do not go through _exit or ExitProcess(), terminate immediately - log() << "*** immediate exit due to unhandled exception"; + severe() << "*** immediate exit due to unhandled exception"; TerminateProcess(GetCurrentProcess(), EXIT_ABRUPT); // We won't reach here diff --git a/src/mongo/util/hex.cpp b/src/mongo/util/hex.cpp index d589d751b45..4ee2967ddff 100644 --- a/src/mongo/util/hex.cpp +++ b/src/mongo/util/hex.cpp @@ -62,6 +62,10 @@ std::string integerToHexDef(T inInt) { } template <> +std::string integerToHex<char>(char val) { + return integerToHexDef(val); +} +template <> std::string integerToHex<int>(int val) { return integerToHexDef(val); } diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript index 59b546fad65..d2647622471 100644 --- a/src/mongo/util/net/SConscript +++ b/src/mongo/util/net/SConscript @@ -64,6 +64,17 @@ networkEnv.Library( ) env.Library( + target='ssl_manager_status', + source=[ + "ssl_manager_status.cpp", + ], + LIBDEPS=[ + 'network', + '$BUILD_DIR/mongo/db/commands/core', + ], +) + +env.Library( target='message_port_mock', source=[ "message_port_mock.cpp", diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 1b120a3be83..dd6e73c3e08 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -51,6 +51,7 @@ #include "mongo/util/concurrency/threadlocal.h" #include "mongo/util/debug_util.h" #include "mongo/util/exit.h" +#include "mongo/util/hex.h" #include "mongo/util/log.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/net/message.h" @@ -82,6 +83,8 @@ const SSLParams& getSSLGlobalParams() { return sslGlobalParams; } +namespace { + /** * Configurable via --setParameter disableNonSSLConnectionLogging=true. If false (default) * if the sslMode is set to preferSSL, we will log connections that are not using SSL. @@ -92,6 +95,18 @@ ExportedServerParameter<bool, ServerParameterType::kStartupOnly> "disableNonSSLConnectionLogging", &sslGlobalParams.disableNonSSLConnectionLogging); +ExportedServerParameter<bool, ServerParameterType::kStartupOnly> + suppressNoTLSPeerCertificateWarning(ServerParameterSet::getGlobal(), + "suppressNoTLSPeerCertificateWarning", + &sslGlobalParams.suppressNoTLSPeerCertificateWarning); + +ExportedServerParameter<bool, ServerParameterType::kStartupOnly> sslWithholdClientCertificate( + ServerParameterSet::getGlobal(), + "sslWithholdClientCertificate", + &sslGlobalParams.tlsWithholdClientCertificate); + +} // namespace + class OpenSSLCipherConfigParameter : public ExportedServerParameter<std::string, ServerParameterType::kStartupOnly> { public: @@ -159,6 +174,9 @@ IMPLEMENT_ASN1_ENCODE_FUNCTIONS_const_fname(ASN1_SEQUENCE_ANY, ASN1_SET_ANY, ASN const STACK_OF(X509_EXTENSION) * X509_get0_extensions(const X509* peerCert) { return peerCert->cert_info->extensions; } +inline int X509_NAME_ENTRY_set(const X509_NAME_ENTRY* ne) { + return ne->set; +} #endif /** @@ -316,6 +334,7 @@ private: bool _weakValidation; bool _allowInvalidCertificates; bool _allowInvalidHostnames; + bool _suppressNoCertificateWarning; SSLConfiguration _sslConfiguration; /** @@ -356,7 +375,7 @@ private: */ bool _parseAndValidateCertificate(const std::string& keyFile, const std::string& keyPassword, - std::string* subjectName, + SSLX509Name* subjectName, Date_t* serverNotAfter); @@ -418,12 +437,18 @@ void setupFIPS() { fassertFailedNoTrace(17089); #endif } + +TLSVersionCounts tlsVersionCounts; + } // namespace +TLSVersionCounts& TLSVersionCounts::get() { + return tlsVersionCounts; +} + // Global variable indicating if this is a server or a client instance bool isSSLServer = false; - MONGO_INITIALIZER(SetupOpenSSL)(InitializerContext*) { SSL_library_init(); SSL_load_error_strings(); @@ -463,23 +488,42 @@ SSLManagerInterface* getSSLManager() { return NULL; } -std::string getCertificateSubjectName(X509* cert) { - std::string result; +SSLX509Name getCertificateSubjectX509Name(X509* cert) { + std::vector<std::vector<SSLX509Name::Entry>> entries; + + auto name = X509_get_subject_name(cert); + int count = X509_NAME_entry_count(name); + int prevSet = -1; + std::vector<SSLX509Name::Entry> rdn; + for (int i = count - 1; i >= 0; --i) { + auto* entry = X509_NAME_get_entry(name, i); + + const auto currentSet = X509_NAME_ENTRY_set(entry); + if (currentSet != prevSet) { + if (!rdn.empty()) { + entries.push_back(std::move(rdn)); + rdn = std::vector<SSLX509Name::Entry>(); + } + prevSet = currentSet; + } - BIO* out = BIO_new(BIO_s_mem()); - uassert(16884, "unable to allocate BIO memory", NULL != out); - ON_BLOCK_EXIT(BIO_free, out); + char buffer[128]; + // OBJ_obj2txt can only fail if we pass a nullptr from get_object, + // or if OpenSSL's BN library falls over. + // In either case, just panic. + uassert(ErrorCodes::InvalidSSLConfiguration, + "Unable to parse certiciate subject name", + OBJ_obj2txt(buffer, sizeof(buffer), X509_NAME_ENTRY_get_object(entry), 1) > 0); - if (X509_NAME_print_ex(out, X509_get_subject_name(cert), 0, XN_FLAG_RFC2253) >= 0) { - if (BIO_number_written(out) > 0) { - result.resize(BIO_number_written(out)); - BIO_read(out, &result[0], result.size()); - } - } else { - log() << "failed to convert subject name to RFC2253 format"; + const auto* str = X509_NAME_ENTRY_get_data(entry); + rdn.emplace_back( + buffer, str->type, std::string(reinterpret_cast<const char*>(str->data), str->length)); + } + if (!rdn.empty()) { + entries.push_back(std::move(rdn)); } - return result; + return SSLX509Name(std::move(entries)); } SSLConnection::SSLConnection(SSL_CTX* context, Socket* sock, const char* initialBytes, int len) @@ -512,6 +556,96 @@ SSLConnection::~SSLConnection() { } namespace { +std::string x509OidToShortName(const std::string& name) { + const auto nid = OBJ_txt2nid(name.c_str()); + if (nid == 0) { + return name; + } + const auto* sn = OBJ_nid2sn(nid); + if (!sn) { + return name; + } + return sn; +} + +// Characters that need to be escaped in RFC 2253 +const std::array<char, 7> rfc2253EscapeChars = {',', '+', '"', '\\', '<', '>', ';'}; + +// See section "2.4 Converting an AttributeValue from ASN.1 to a String" in RFC 2243 +std::string escapeRfc2253(StringData str) { + std::string ret; + + if (str.size() > 0) { + size_t pos = 0; + + // a space or "#" character occurring at the beginning of the string + if (str[0] == ' ') { + ret = "\\ "; + pos = 1; + } else if (str[0] == '#') { + ret = "\\#"; + pos = 1; + } + + while (pos < str.size()) { + if (static_cast<signed char>(str[pos]) < 0) { + ret += '\\'; + ret += integerToHex(str[pos]); + } else { + if (std::find(rfc2253EscapeChars.cbegin(), rfc2253EscapeChars.cend(), str[pos]) != + rfc2253EscapeChars.cend()) { + ret += '\\'; + } + + ret += str[pos]; + } + ++pos; + } + + // a space character occurring at the end of the string + if (ret.size() > 2 && ret[ret.size() - 1] == ' ') { + ret[ret.size() - 1] = '\\'; + ret += ' '; + } + } + + return ret; +} + +} // namespace + +StatusWith<std::string> SSLX509Name::getOID(StringData oid) const { + for (const auto& rdn : _entries) { + for (const auto& entry : rdn) { + if (entry.oid == oid) { + return entry.value; + } + } + } + return {ErrorCodes::KeyNotFound, "OID does not exist"}; +} + +StringBuilder& operator<<(StringBuilder& os, const SSLX509Name& name) { + std::string comma; + for (const auto& rdn : name._entries) { + std::string plus; + os << comma; + for (const auto& entry : rdn) { + os << plus << x509OidToShortName(entry.oid) << "=" << escapeRfc2253(entry.value); + plus = "+"; + } + comma = ","; + } + return os; +} + +std::string SSLX509Name::toString() const { + StringBuilder os; + os << *this; + return os.str(); +} + +namespace { void canonicalizeClusterDN(std::vector<std::string>* dn) { // remove all RDNs we don't care about for (size_t i = 0; i < dn->size(); i++) { @@ -526,30 +660,62 @@ void canonicalizeClusterDN(std::vector<std::string>* dn) { } std::stable_sort(dn->begin(), dn->end()); } + +constexpr StringData kOID_DC = "0.9.2342.19200300.100.1.25"_sd; +constexpr StringData kOID_O = "2.5.4.10"_sd; +constexpr StringData kOID_OU = "2.5.4.11"_sd; + +std::vector<SSLX509Name::Entry> canonicalizeClusterDN( + const std::vector<std::vector<SSLX509Name::Entry>>& entries) { + std::vector<SSLX509Name::Entry> ret; + + for (const auto& rdn : entries) { + for (const auto& entry : rdn) { + if ((entry.oid != kOID_DC) && (entry.oid != kOID_O) && (entry.oid != kOID_OU)) { + continue; + } + ret.push_back(entry); + } + } + std::stable_sort(ret.begin(), ret.end()); + return ret; +} +} // namespace + +/** + * The behavior of isClusterMember() is subtly different when passed + * an SSLX509Name versus a StringData. + * + * The SSLX509Name version (immediately below) compares distinguished + * names in their raw, unescaped forms and provides a more reliable match. + * + * The StringData version attempts to do a simplified string compare + * with the serialized version of the server subject name. + * + * Because escaping is not checked in the StringData version, + * some not-strictly matching RDNs will appear to share O/OU/DC with the + * server subject name. Therefore, that variant should be called with care. + */ +bool SSLConfiguration::isClusterMember(const SSLX509Name& subject) const { + auto client = canonicalizeClusterDN(subject._entries); + auto server = canonicalizeClusterDN(serverSubjectName._entries); + + return !client.empty() && (client == server); } bool SSLConfiguration::isClusterMember(StringData subjectName) const { std::vector<std::string> clientRDN = StringSplitter::split(subjectName.toString(), ","); - std::vector<std::string> serverRDN = StringSplitter::split(serverSubjectName, ","); + std::vector<std::string> serverRDN = StringSplitter::split(serverSubjectName.toString(), ","); canonicalizeClusterDN(&clientRDN); canonicalizeClusterDN(&serverRDN); - if (clientRDN.size() == 0 || clientRDN.size() != serverRDN.size()) { - return false; - } - - for (size_t i = 0; i < serverRDN.size(); i++) { - if (clientRDN[i] != serverRDN[i]) { - return false; - } - } - return true; + return !clientRDN.empty() && (clientRDN == serverRDN); } BSONObj SSLConfiguration::getServerStatusBSON() const { BSONObjBuilder security; - security.append("SSLServerSubjectName", serverSubjectName); + security.append("SSLServerSubjectName", serverSubjectName.toString()); security.appendBool("SSLServerHasCertificateAuthority", hasCA); security.appendDate("SSLServerCertificateExpirationDate", serverCertificateExpirationDate); return security.obj(); @@ -562,7 +728,8 @@ SSLManager::SSLManager(const SSLParams& params, bool isServer) _clientContext(nullptr, _free_ssl_context), _weakValidation(params.sslWeakCertificateValidation), _allowInvalidCertificates(params.sslAllowInvalidCertificates), - _allowInvalidHostnames(params.sslAllowInvalidHostnames) { + _allowInvalidHostnames(params.sslAllowInvalidHostnames), + _suppressNoCertificateWarning(params.suppressNoTLSPeerCertificateWarning) { if (!_initSynchronousSSLContext(&_clientContext, params, ConnectionDirection::kOutgoing)) { uasserted(16768, "ssl initialization problem"); } @@ -716,23 +883,33 @@ Status SSLManager::initSSLContext(SSL_CTX* context, << getSSLErrorMessage(ERR_get_error())); } - if (direction == ConnectionDirection::kOutgoing && !params.sslClusterFile.empty()) { + if (direction == ConnectionDirection::kOutgoing && params.tlsWithholdClientCertificate) { + // Do not send a client certificate if they have been suppressed. + + } else if (direction == ConnectionDirection::kOutgoing && !params.sslClusterFile.empty()) { + // Use the configured clusterFile as our client certificate. ::EVP_set_pw_prompt("Enter cluster certificate passphrase"); if (!_setupPEM(context, params.sslClusterFile, params.sslClusterPassword)) { return Status(ErrorCodes::InvalidSSLConfiguration, "Can not set up ssl clusterFile."); } + } else if (!params.sslPEMKeyFile.empty()) { - // Use the pemfile for everything else + // Use the base pemKeyFile for any other outgoing connections, + // as well as all incoming connections. ::EVP_set_pw_prompt("Enter PEM passphrase"); if (!_setupPEM(context, params.sslPEMKeyFile, params.sslPEMKeyPassword)) { return Status(ErrorCodes::InvalidSSLConfiguration, "Can not set up PEM key file."); } } - const auto status = - params.sslCAFile.empty() ? _setupSystemCA(context) : _setupCA(context, params.sslCAFile); - if (!status.isOK()) + std::string cafile = params.sslCAFile; + if (direction == ConnectionDirection::kIncoming && !params.sslClusterCAFile.empty()) { + cafile = params.sslClusterCAFile; + } + const auto status = cafile.empty() ? _setupSystemCA(context) : _setupCA(context, cafile); + if (!status.isOK()) { return status; + } if (!params.sslCRLFile.empty()) { if (!_setupCRL(context, params.sslCRLFile)) { @@ -795,7 +972,7 @@ unsigned long long SSLManager::_convertASN1ToMillis(ASN1_TIME* asn1time) { bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, const std::string& keyPassword, - std::string* subjectName, + SSLX509Name* subjectName, Date_t* serverCertificateExpirationDate) { BIO* inBIO = BIO_new(BIO_s_file()); if (inBIO == NULL) { @@ -822,7 +999,7 @@ bool SSLManager::_parseAndValidateCertificate(const std::string& keyFile, } ON_BLOCK_EXIT(X509_free, x509); - *subjectName = getCertificateSubjectName(x509); + *subjectName = getCertificateSubjectX509Name(x509); if (serverCertificateExpirationDate != NULL) { unsigned long long notBeforeMillis = _convertASN1ToMillis(X509_get_notBefore(x509)); if (notBeforeMillis == 0) { @@ -1210,8 +1387,36 @@ bool SSLManager::_hostNameMatch(const char* nameToMatch, const char* certHostNam } } +void recordTLSVersion(const SSL* conn) { + int protocol = SSL_version(conn); + + auto& counts = mongo::TLSVersionCounts::get(); + switch (protocol) { + case TLS1_VERSION: + counts.tls10.addAndFetch(1); + break; + case TLS1_1_VERSION: + counts.tls11.addAndFetch(1); + break; + case TLS1_2_VERSION: + counts.tls12.addAndFetch(1); + break; +#ifdef TLS1_3_VERSION + case TLS1_3_VERSION: + counts.tls13.addAndFetch(1); + break; +#endif + default: + // Do nothing + break; + } +} + StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertificate( SSL* conn, const std::string& remoteHost) { + + recordTLSVersion(conn); + if (!_sslConfiguration.hasCA && isSSLServer) return {boost::none}; @@ -1219,7 +1424,11 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi if (NULL == peerCert) { // no certificate presented by peer if (_weakValidation) { - warning() << "no SSL certificate provided by peer"; + // do not give warning if certificate warnings are suppressed + if (!_suppressNoCertificateWarning) { + warning() << "no SSL certificate provided by peer"; + } + return {boost::none}; } else { auto msg = "no SSL certificate provided by peer; connection rejected"; error() << msg; @@ -1246,8 +1455,8 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } // TODO: check optional cipher restriction, using cert. - std::string peerSubjectName = getCertificateSubjectName(peerCert); - LOG(2) << "Accepted TLS connection from peer: " << peerSubjectName; + auto peerSubject = getCertificateSubjectX509Name(peerCert); + LOG(2) << "Accepted TLS connection from peer: " << peerSubject; StatusWith<stdx::unordered_set<RoleName>> swPeerCertificateRoles = _parsePeerRoles(peerCert); if (!swPeerCertificateRoles.isOK()) { @@ -1258,7 +1467,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi // perform hostname validation of the remote server if (remoteHost.empty()) { return boost::make_optional( - SSLPeerInfo(peerSubjectName, std::move(swPeerCertificateRoles.getValue()))); + SSLPeerInfo(peerSubject, std::move(swPeerCertificateRoles.getValue()))); } // Try to match using the Subject Alternate Name, if it exists. @@ -1288,19 +1497,19 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } } sk_GENERAL_NAME_pop_free(sanNames, GENERAL_NAME_free); - } else if (peerSubjectName.find("CN=") != std::string::npos) { + } else { // If Subject Alternate Name (SAN) doesn't exist and Common Name (CN) does, // check Common Name. - int cnBegin = peerSubjectName.find("CN=") + 3; - int cnEnd = peerSubjectName.find(",", cnBegin); - std::string commonName = peerSubjectName.substr(cnBegin, cnEnd - cnBegin); - - if (_hostNameMatch(remoteHost.c_str(), commonName.c_str())) { - cnMatch = true; + auto swCN = peerSubject.getOID(kOID_CommonName); + if (swCN.isOK()) { + auto commonName = std::move(swCN.getValue()); + if (_hostNameMatch(remoteHost.c_str(), commonName.c_str())) { + cnMatch = true; + } + certificateNames << "CN: " << commonName; + } else { + certificateNames << "No Common Name (CN) or Subject Alternate Names (SAN) found"; } - certificateNames << "CN: " << commonName; - } else { - certificateNames << "No Common Name (CN) or Subject Alternate Names (SAN) found"; } if (!sanMatch && !cnMatch) { @@ -1316,7 +1525,7 @@ StatusWith<boost::optional<SSLPeerInfo>> SSLManager::parseAndValidatePeerCertifi } } - return boost::make_optional(SSLPeerInfo(peerSubjectName, stdx::unordered_set<RoleName>())); + return boost::make_optional(SSLPeerInfo(peerSubject, stdx::unordered_set<RoleName>())); } diff --git a/src/mongo/util/net/ssl_manager.h b/src/mongo/util/net/ssl_manager.h index bbdafdabff1..0058f7223b2 100644 --- a/src/mongo/util/net/ssl_manager.h +++ b/src/mongo/util/net/ssl_manager.h @@ -38,6 +38,8 @@ #include "mongo/base/disallow_copying.h" #include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" +#include "mongo/db/service_context.h" +#include "mongo/platform/atomic_word.h" #include "mongo/util/decorable.h" #include "mongo/util/net/sock.h" #include "mongo/util/net/ssl_types.h" @@ -72,18 +74,11 @@ public: }; struct SSLConfiguration { - SSLConfiguration() : serverSubjectName(""), clientSubjectName("") {} - SSLConfiguration(const std::string& serverSubjectName, - const std::string& clientSubjectName, - const Date_t& serverCertificateExpirationDate) - : serverSubjectName(serverSubjectName), - clientSubjectName(clientSubjectName), - serverCertificateExpirationDate(serverCertificateExpirationDate) {} - bool isClusterMember(StringData subjectName) const; + bool isClusterMember(const SSLX509Name& subjectName) const; BSONObj getServerStatusBSON() const; - std::string serverSubjectName; - std::string clientSubjectName; + SSLX509Name serverSubjectName; + SSLX509Name clientSubjectName; Date_t serverCertificateExpirationDate; bool hasCA = false; }; @@ -106,6 +101,17 @@ const ASN1OID mongodbRolesOID("1.3.6.1.4.1.34601.2.1.1", "MongoRoles", "Sequence of MongoDB Database Roles"); +/** + * Counts of negogtiated version used by TLS connections. + */ +struct TLSVersionCounts { + AtomicInt64 tls10; + AtomicInt64 tls11; + AtomicInt64 tls12; + + static TLSVersionCounts& get(); +}; + class SSLManagerInterface : public Decorable<SSLManagerInterface> { public: static std::unique_ptr<SSLManagerInterface> create(const SSLParams& params, bool isServer); diff --git a/src/mongo/util/net/ssl_manager_status.cpp b/src/mongo/util/net/ssl_manager_status.cpp new file mode 100644 index 00000000000..559d06d4dd2 --- /dev/null +++ b/src/mongo/util/net/ssl_manager_status.cpp @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2018 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects + * for all of the code used other than as permitted herein. If you modify + * file(s) with this exception, you may extend this exception to your + * version of the file(s), but you are not obligated to do so. If you do not + * wish to do so, delete this exception statement from your version. If you + * delete this exception statement from all source files in the program, + * then also delete it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/net/ssl_manager.h" + +#include "mongo/config.h" +#include "mongo/db/commands/server_status.h" + + +#ifdef MONGO_CONFIG_SSL + +namespace mongo { +namespace { + +/** + * Status section of which tls versions connected to MongoDB and completed an SSL handshake. + * Note: Clients are only not counted if they try to connect to the server with a unsupported TLS + * version. They are still counted if the server rejects them for certificate issues in + * parseAndValidatePeerCertificate. + */ +class TLSVersionSatus : public ServerStatusSection { +public: + TLSVersionSatus() : ServerStatusSection("transportSecurity") {} + + bool includeByDefault() const final { + return true; + } + + BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const final { + auto& counts = TLSVersionCounts::get(); + + BSONObjBuilder builder; + builder.append("1.0", counts.tls10.load()); + builder.append("1.1", counts.tls11.load()); + builder.append("1.2", counts.tls12.load()); + return builder.obj(); + } +} tlsVersionStatus; + +} // namespace +} // namespace mongo + +#endif diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp index b9785d1b83f..0d1d3504ae7 100644 --- a/src/mongo/util/net/ssl_options.cpp +++ b/src/mongo/util/net/ssl_options.cpp @@ -81,6 +81,11 @@ Status addSSLServerOptions(moe::OptionSection* options) { options->addOptionChaining( "net.ssl.CAFile", "sslCAFile", moe::String, "Certificate Authority file for SSL"); + options->addOptionChaining("net.ssl.clusterCAFile", + "sslClusterCAFile", + moe::String, + "CA used for verifying remotes during outbound connections"); + options->addOptionChaining( "net.ssl.CRLFile", "sslCRLFile", moe::String, "Certificate Revocation List file for SSL"); @@ -327,6 +332,12 @@ Status storeSSLServerOptions(const moe::Environment& params) { .generic_string(); } + if (params.count("net.ssl.clusterCAFile")) { + sslGlobalParams.sslClusterCAFile = + boost::filesystem::absolute(params["net.ssl.clusterCAFile"].as<std::string>()) + .generic_string(); + } + if (params.count("net.ssl.CRLFile")) { sslGlobalParams.sslCRLFile = boost::filesystem::absolute(params["net.ssl.CRLFile"].as<std::string>()) diff --git a/src/mongo/util/net/ssl_options.h b/src/mongo/util/net/ssl_options.h index aef2860093b..02a82108529 100644 --- a/src/mongo/util/net/ssl_options.h +++ b/src/mongo/util/net/ssl_options.h @@ -50,6 +50,7 @@ struct SSLParams { std::string sslClusterFile; // --sslInternalKeyFile std::string sslClusterPassword; // --sslInternalKeyPassword std::string sslCAFile; // --sslCAFile + std::string sslClusterCAFile; // --sslClusterCAFile std::string sslCRLFile; // --sslCRLFile std::string sslCipherConfig; // --sslCipherConfig std::vector<Protocols> sslDisabledProtocols; // --sslDisabledProtocols @@ -59,6 +60,9 @@ struct SSLParams { bool sslAllowInvalidHostnames = false; // --sslAllowInvalidHostnames bool disableNonSSLConnectionLogging = false; // --setParameter disableNonSSLConnectionLogging=true + bool suppressNoTLSPeerCertificateWarning = + false; // --setParameter suppressNoTLSPeerCertificateWarning + bool tlsWithholdClientCertificate = false; // --setParameter tlsWithholdClientCertificate SSLParams() { sslMode.store(SSLMode_disabled); diff --git a/src/mongo/util/net/ssl_types.h b/src/mongo/util/net/ssl_types.h index fc8f600625c..f7c2fa33050 100644 --- a/src/mongo/util/net/ssl_types.h +++ b/src/mongo/util/net/ssl_types.h @@ -29,21 +29,84 @@ #include <string> +#include "mongo/bson/util/builder.h" #include "mongo/db/auth/role_name.h" #include "mongo/stdx/unordered_set.h" namespace mongo { +constexpr StringData kOID_CommonName = "2.5.4.3"_sd; + +/** + * Represents a structed X509 certificate subject name. + * For example: C=US,O=MongoDB,OU=KernelTeam,CN=server + * would be held as a four element vector of Entries. + * The first entry of which yould be broken down something like: + * {{"2.5.4.6", 19, "US"}}. + * Note that _entries is a vector of vectors to accomodate + * multi-value RDNs. + */ +class SSLX509Name { +public: + struct Entry { + Entry(std::string oid, int type, std::string value) + : oid(std::move(oid)), type(type), value(std::move(value)) {} + std::string oid; // e.g. "2.5.4.8" (ST) + int type; // e.g. 19 (PRINTABLESTRING) + std::string value; + std::tuple<const std::string&, const int&, const std::string&> equalityLens() const { + return std::tie(oid, type, value); + } + }; + + SSLX509Name() = default; + explicit SSLX509Name(std::vector<std::vector<Entry>> entries) : _entries(std::move(entries)) {} + + /** + * Retreive the first instance of the value for a given OID in this name. + * Returns ErrorCodes::KeyNotFound if the OID does not exist. + */ + StatusWith<std::string> getOID(StringData oid) const; + + bool empty() const { + return std::all_of(_entries.cbegin(), _entries.cend(), [](const std::vector<Entry>& e) { + return e.empty(); + }); + } + + friend StringBuilder& operator<<(StringBuilder&, const SSLX509Name&); + std::string toString() const; + + friend bool operator==(const SSLX509Name& lhs, const SSLX509Name& rhs) { + return lhs._entries == rhs._entries; + } + friend bool operator!=(const SSLX509Name& lhs, const SSLX509Name& rhs) { + return !(lhs._entries == rhs._entries); + } + +private: + friend struct SSLConfiguration; + std::vector<std::vector<Entry>> _entries; +}; + +std::ostream& operator<<(std::ostream&, const SSLX509Name&); +inline bool operator==(const SSLX509Name::Entry& lhs, const SSLX509Name::Entry& rhs) { + return lhs.equalityLens() == rhs.equalityLens(); +} +inline bool operator<(const SSLX509Name::Entry& lhs, const SSLX509Name::Entry& rhs) { + return lhs.equalityLens() < rhs.equalityLens(); +} + /** * Contains information extracted from the peer certificate which is consumed by subsystems * outside of the networking stack. */ struct SSLPeerInfo { - SSLPeerInfo(std::string subjectName, stdx::unordered_set<RoleName> roles) + SSLPeerInfo(SSLX509Name subjectName, stdx::unordered_set<RoleName> roles) : subjectName(std::move(subjectName)), roles(std::move(roles)) {} SSLPeerInfo() = default; - std::string subjectName; + SSLX509Name subjectName; stdx::unordered_set<RoleName> roles; }; diff --git a/src/third_party/wiredtiger/import.data b/src/third_party/wiredtiger/import.data index d7df48cee9a..9db71335d7b 100644 --- a/src/third_party/wiredtiger/import.data +++ b/src/third_party/wiredtiger/import.data @@ -1,5 +1,5 @@ { - "commit": "65d96ccb972b239c8af5aa24a03d215eb143b0e4", + "commit": "7a6598ca9b54c358803aa6290dce618f0abed63f", "github": "wiredtiger/wiredtiger.git", "vendor": "wiredtiger", "branch": "mongodb-3.4" diff --git a/src/third_party/wiredtiger/src/reconcile/rec_write.c b/src/third_party/wiredtiger/src/reconcile/rec_write.c index 688efa10398..b76192c0cf9 100644 --- a/src/third_party/wiredtiger/src/reconcile/rec_write.c +++ b/src/third_party/wiredtiger/src/reconcile/rec_write.c @@ -391,6 +391,18 @@ __wt_reconcile(WT_SESSION_IMPL *session, WT_REF *ref, */ WT_PAGE_LOCK(session, page); + /* + * Now that the page is locked, if attempting to evict it, check again + * whether eviction is permitted. The page's state could have changed + * while we were waiting to acquire the lock (e.g., the page could have + * split). + */ + if (LF_ISSET(WT_EVICTING) && + !__wt_page_can_evict(session, ref, NULL)) { + WT_PAGE_UNLOCK(session, page); + return (EBUSY); + } + oldest_id = __wt_txn_oldest_id(session); if (LF_ISSET(WT_EVICTING)) mod->last_eviction_id = oldest_id; |