diff options
Diffstat (limited to 'src/mongo/db/index_builds_coordinator.cpp')
| -rw-r--r-- | src/mongo/db/index_builds_coordinator.cpp | 402 |
1 files changed, 151 insertions, 251 deletions
diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index e6f6d2110ab..45c1afefdae 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -39,9 +39,9 @@ #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/index_build_entry_gen.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/concurrency/replication_state_transition_lock_guard.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbhelpers.h" @@ -63,12 +63,6 @@ #include "mongo/db/storage/storage_util.h" #include "mongo/db/storage/two_phase_index_build_knobs_gen.h" #include "mongo/logv2/log.h" -#include "mongo/logv2/log_attr.h" -#include "mongo/logv2/log_component.h" -#include "mongo/logv2/log_severity_suppressor.h" -#include "mongo/logv2/redaction.h" -#include "mongo/platform/compiler.h" -#include "mongo/rpc/message.h" #include "mongo/s/shard_key_pattern.h" #include "mongo/util/assert_util.h" #include "mongo/util/scoped_counter.h" @@ -93,7 +87,6 @@ MONGO_FAIL_POINT_DEFINE(hangBeforeBuildingIndex); MONGO_FAIL_POINT_DEFINE(hangBeforeBuildingIndexSecond); MONGO_FAIL_POINT_DEFINE(hangIndexBuildBeforeWaitingUntilMajorityOpTime); MONGO_FAIL_POINT_DEFINE(failSetUpResumeIndexBuild); -MONGO_FAIL_POINT_DEFINE(hangAbortIndexBuildByBuildUUIDAfterLocks); IndexBuildsCoordinator::ActiveIndexBuildsSSS::ActiveIndexBuildsSSS() : ServerStatusSection("activeIndexBuilds"), @@ -113,7 +106,6 @@ constexpr StringData kAbortIndexBuildFieldName = "abortIndexBuild"_sd; constexpr StringData kIndexesFieldName = "indexes"_sd; constexpr StringData kKeyFieldName = "key"_sd; constexpr StringData kUniqueFieldName = "unique"_sd; -constexpr StringData kPrepareUniqueFieldName = "prepareUnique"_sd; /** * Checks if unique index specification is compatible with sharding configuration. @@ -129,9 +121,9 @@ void checkShardKeyRestrictions(OperationContext* opCtx, const ShardKeyPattern shardKeyPattern(collDesc.getKeyPattern()); uassert(ErrorCodes::CannotCreateIndex, - str::stream() << "cannot create index with 'unique' or 'prepareUnique' option over " - << newIdxKey << " with shard key pattern " << shardKeyPattern.toBSON(), - shardKeyPattern.isIndexUniquenessCompatible(newIdxKey)); + str::stream() << "cannot create unique index over " << newIdxKey + << " with shard key pattern " << shardKeyPattern.toBSON(), + shardKeyPattern.isUniqueIndexCompatible(newIdxKey)); } /** @@ -196,12 +188,6 @@ void removeIndexBuildEntryAfterCommitOrAbort(OperationContext* opCtx, return; } - if (replCoord->getSettings().shouldRecoverFromOplogAsStandalone()) { - // Writes to the 'config.system.indexBuilds' collection are replicated and the index entry - // will be removed when the delete oplog entry is replayed at a later time. - return; - } - auto status = indexbuildentryhelpers::removeIndexBuildEntry( opCtx, indexBuildEntryCollection, replState.buildUUID); if (!status.isOK()) { @@ -557,52 +543,79 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC CollectionWriter collection(opCtx, nss); { - // These steps are combined into a single WUOW to ensure there are no commits without the - // indexes for repair. + // These steps are combined into a single WUOW to ensure there are no commits without + // the indexes. + // 1) Drop all unfinished indexes. + // 2) Start, but do not complete the index build process. WriteUnitOfWork wuow(opCtx); + auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); - // We need to initialize the collection to rebuild the indexes. The collection may already - // be initialized when rebuilding multiple unfinished indexes on the same collection. - if (!collection->isInitialized()) { - collection.getWritableCollection()->init(opCtx); - } - if (storageGlobalParams.repair) { - Status status = _dropIndexesForRepair(opCtx, collection, indexNames); - if (!status.isOK()) { - return status; - } - } else { - // Unfinished index builds that are not resumable will drop and recreate the index table - // using the same ident to avoid doing untimestamped writes to the catalog. - for (const auto& indexName : indexNames) { - auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); - auto desc = - indexCatalog->findIndexByName(opCtx, - indexName, - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); - Status status = indexCatalog->resetUnfinishedIndexForRecovery( - opCtx, collection.getWritableCollection(), desc); - if (!status.isOK()) { - return status; + for (size_t i = 0; i < indexNames.size(); i++) { + bool includeUnfinished = false; + auto descriptor = + indexCatalog->findIndexByName(opCtx, indexNames[i], includeUnfinished); + if (descriptor) { + Status s = + indexCatalog->dropIndex(opCtx, collection.getWritableCollection(), descriptor); + if (!s.isOK()) { + return s; } + continue; + } - const auto durableBuildUUID = collection->getIndexBuildUUID(indexName); + // If the index is not present in the catalog, then we are trying to drop an already + // aborted index. This may happen when rollback-via-refetch restarts an index build + // after an abort has been rolled back. + if (!collection->isIndexPresent(indexNames[i])) { + LOGV2(20652, + "An index was not found in the catalog while trying to drop the index during " + "recovery", + "buildUUID"_attr = buildUUID, + "index"_attr = indexNames[i]); + continue; + } - // A build UUID is present if and only if we are rebuilding a two-phase build. - invariant((protocol == IndexBuildProtocol::kTwoPhase) == - durableBuildUUID.has_value()); - // When a buildUUID is present, it must match the build UUID parameter to this - // function. - invariant(!durableBuildUUID || *durableBuildUUID == buildUUID, - str::stream() << "durable build UUID: " << durableBuildUUID - << "buildUUID: " << buildUUID); + const auto durableBuildUUID = collection->getIndexBuildUUID(indexNames[i]); + + // A build UUID is present if and only if we are rebuilding a two-phase build. + invariant((protocol == IndexBuildProtocol::kTwoPhase) == + durableBuildUUID.is_initialized()); + // When a buildUUID is present, it must match the build UUID parameter to this + // function. + invariant(!durableBuildUUID || *durableBuildUUID == buildUUID, + str::stream() << "durable build UUID: " << durableBuildUUID + << "buildUUID: " << buildUUID); + + // If the unfinished index is in the IndexCatalog, drop it through there, otherwise drop + // it from the DurableCatalog. Rollback-via-refetch does not clear any in-memory state, + // so we should do it manually here. + includeUnfinished = true; + descriptor = indexCatalog->findIndexByName(opCtx, indexNames[i], includeUnfinished); + if (descriptor) { + Status s = indexCatalog->dropUnfinishedIndex( + opCtx, collection.getWritableCollection(), descriptor); + if (!s.isOK()) { + return s; + } + } else { + // There are no concurrent users of the index during startup recovery, so it is OK + // to pass in a nullptr for the index 'ident', promising that the index is not in + // use. + catalog::removeIndex( + opCtx, indexNames[i], collection.getWritableCollection(), nullptr /* ident */); } } + // We need to initialize the collection to rebuild the indexes. The collection may already + // be initialized when rebuilding indexes with rollback-via-refetch. + if (!collection->isInitialized()) { + collection.getWritableCollection()->init(opCtx); + } + + auto dbName = nss.db().toString(); auto replIndexBuildState = std::make_shared<ReplIndexBuildState>( - buildUUID, collection->uuid(), nss.db().toString(), specs, protocol); + buildUUID, collection->uuid(), dbName, specs, protocol); Status status = activeIndexBuilds.registerIndexBuild(replIndexBuildState); if (!status.isOK()) { @@ -611,8 +624,6 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC IndexBuildsManager::SetupOptions options; options.protocol = protocol; - // All indexes are dropped during repair and should be rebuilt normally. - options.forRecovery = !storageGlobalParams.repair; status = _indexBuildsManager.setUpIndexBuild( opCtx, collection, specs, buildUUID, MultiIndexBlock::kNoopOnInitFn, options); if (!status.isOK()) { @@ -627,39 +638,6 @@ Status IndexBuildsCoordinator::_startIndexBuildForRecovery(OperationContext* opC return Status::OK(); } -Status IndexBuildsCoordinator::_dropIndexesForRepair(OperationContext* opCtx, - CollectionWriter& collection, - const std::vector<std::string>& indexNames) { - invariant(collection->isInitialized()); - for (const auto& indexName : indexNames) { - auto indexCatalog = collection.getWritableCollection()->getIndexCatalog(); - auto descriptor = - indexCatalog->findIndexByName(opCtx, indexName, IndexCatalog::InclusionPolicy::kReady); - if (descriptor) { - Status s = - indexCatalog->dropIndex(opCtx, collection.getWritableCollection(), descriptor); - if (!s.isOK()) { - return s; - } - continue; - } - - // The index must be unfinished or frozen if it isn't ready. - descriptor = indexCatalog->findIndexByName(opCtx, - indexName, - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); - invariant(descriptor); - Status s = indexCatalog->dropUnfinishedIndex( - opCtx, collection.getWritableCollection(), descriptor); - if (!s.isOK()) { - return s; - } - } - - return Status::OK(); -} - Status IndexBuildsCoordinator::_setUpResumeIndexBuild(OperationContext* opCtx, std::string dbName, const UUID& collectionUUID, @@ -944,6 +922,7 @@ void IndexBuildsCoordinator::applyStartIndexBuild(OperationContext* opCtx, IndexCatalog* indexCatalog = coll.getWritableCollection(opCtx)->getIndexCatalog(); + const bool includeUnfinished = false; for (const auto& spec : oplogEntry.indexSpecs) { std::string name = spec.getStringField(IndexDescriptor::kIndexNameFieldName).toString(); @@ -951,18 +930,7 @@ void IndexBuildsCoordinator::applyStartIndexBuild(OperationContext* opCtx, str::stream() << "Index spec is missing the 'name' field " << spec, !name.empty()); - if (auto desc = indexCatalog->findIndexByName( - opCtx, name, IndexCatalog::InclusionPolicy::kReady)) { - uassertStatusOK( - indexCatalog->dropIndex(opCtx, coll.getWritableCollection(opCtx), desc)); - } - - const IndexDescriptor* desc = indexCatalog->findIndexByKeyPatternAndOptions( - opCtx, - spec.getObjectField(IndexDescriptor::kKeyPatternFieldName), - spec, - IndexCatalog::InclusionPolicy::kReady); - if (desc) { + if (auto desc = indexCatalog->findIndexByName(opCtx, name, includeUnfinished)) { uassertStatusOK( indexCatalog->dropIndex(opCtx, coll.getWritableCollection(opCtx), desc)); } @@ -1149,8 +1117,7 @@ void IndexBuildsCoordinator::applyAbortIndexBuild(OperationContext* opCtx, const IndexDescriptor* desc = indexCatalog->findIndexByName( opCtx, indexSpec.getStringField(IndexDescriptor::kIndexNameFieldName), - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + /*includeUnfinishedIndexes=*/true); LOGV2(6455400, "Dropping unfinished index during oplog recovery as standalone", @@ -1268,8 +1235,6 @@ bool IndexBuildsCoordinator::abortIndexBuildByBuildUUID(OperationContext* opCtx, AutoGetCollection indexBuildEntryColl( opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); - hangAbortIndexBuildByBuildUUIDAfterLocks.pauseWhileSet(); - // If we are using two-phase index builds and are no longer primary after receiving an // abort, we cannot replicate an abortIndexBuild oplog entry. Continue holding the RSTL to // check the replication state and to prevent any state transitions from happening while @@ -1901,35 +1866,6 @@ Status IndexBuildsCoordinator::_setUpIndexBuildForTwoPhaseRecovery( return _startIndexBuildForRecovery(opCtx, nss, specs, buildUUID, protocol); } -StatusWith<AutoGetCollection> IndexBuildsCoordinator::_autoGetCollectionExclusiveWithTimeout( - OperationContext* opCtx, ReplIndexBuildState* replState, bool retry) { - const Milliseconds kStateTransitionBlockedMaxMs{10}; - boost::optional<logv2::SeveritySuppressor> logSeveritySuppressor; - int retryCount = 0; - while (true) { - try { - return AutoGetCollection(opCtx, - {replState->dbName, replState->collectionUUID}, - MODE_X, - AutoGetCollectionViewMode::kViewsForbidden, - Date_t::now() + kStateTransitionBlockedMaxMs); - } catch (const ExceptionFor<ErrorCodes::LockTimeout>& ex) { - if (!retry) { - return ex.toStatus(); - } - if (!logSeveritySuppressor) { - logSeveritySuppressor.emplace( - Seconds{1}, logv2::LogSeverity::Info(), logv2::LogSeverity::Debug(2)); - } - ++retryCount; - LOGV2_DEBUG(7866200, - (*logSeveritySuppressor)().toInt(), - "Index build: collection lock acquisition timeout, retrying", - "retries"_attr = retryCount); - } - } -} - StatusWith<boost::optional<SharedSemiFuture<ReplIndexBuildState::IndexCatalogStats>>> IndexBuildsCoordinator::_filterSpecsAndRegisterBuild(OperationContext* opCtx, StringData dbName, @@ -2023,10 +1959,10 @@ IndexBuildsCoordinator::PostSetupAction IndexBuildsCoordinator::_setUpIndexBuild std::shared_ptr<ReplIndexBuildState> replState, Timestamp startTimestamp, const IndexBuildOptions& indexBuildOptions) { - auto autoGetColl = - std::move(_autoGetCollectionExclusiveWithTimeout(opCtx, replState.get()).getValue()); + const NamespaceStringOrUUID nssOrUuid{replState->dbName, replState->collectionUUID}; - CollectionWriter collection(opCtx, replState->collectionUUID); + AutoGetCollection coll(opCtx, nssOrUuid, MODE_X); + CollectionWriter collection(opCtx, coll); CollectionShardingState::get(opCtx, collection->ns())->checkShardVersionOrThrow(opCtx); auto replCoord = repl::ReplicationCoordinator::get(opCtx); @@ -2267,26 +2203,17 @@ void IndexBuildsCoordinator::_cleanUpSinglePhaseAfterFailure( runOnAlternateContext( opCtx, "self-abort", [this, replState, status](OperationContext* abortCtx) { ShouldNotConflictWithSecondaryBatchApplicationBlock noConflict(abortCtx->lockState()); - // To avoid potential deadlocks with concurrent external aborts, which hold the - // collection MODE_X lock while waiting for this thread to signal its exit, the - // collection lock is acquired with a timeout, and retried only if the build is not - // already aborted (externally). - while (!replState->isAborted()) { - auto swLocks = - _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get(), false); - if (!swLocks.isOK()) { - LOGV2(7677700, - "Unable to acquire collection lock within the timeout, a concurrent " - "abort might be waiting for the builder thread to exit. Rechecking if " - "self abort is still required.", - "buildUUID"_attr = replState->buildUUID); - continue; - } + Lock::DBLock dbLock(abortCtx, replState->dbName, MODE_IX); - AutoGetCollection indexBuildEntryColl( - abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); - _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); - } + // Unlock RSTL to avoid deadlocks with prepare conflicts and state transitions caused by + // taking a strong collection lock. See SERVER-42621. + unlockRSTL(abortCtx); + + const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); + Lock::CollectionLock collLock(abortCtx, dbAndUUID, MODE_X); + AutoGetCollection indexBuildEntryColl( + abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); }); } @@ -2308,40 +2235,25 @@ void IndexBuildsCoordinator::_cleanUpTwoPhaseAfterFailure( opCtx, "self-abort", [this, replState, status](OperationContext* abortCtx) { ShouldNotConflictWithSecondaryBatchApplicationBlock noConflict(abortCtx->lockState()); - // To avoid potential deadlocks with concurrent external aborts, which hold the - // collection MODE_X lock while waiting for this thread to signal its exit, the - // collection lock is acquired with a timeout, and retried only if the build is not - // already aborted (externally). - while (!replState->isAborted()) { - // Take RSTL to observe and prevent replication state from changing. This is done - // with the release/reacquire strategy to avoid deadlock with prepared txns. - auto swLocks = - _autoGetCollectionExclusiveWithTimeout(abortCtx, replState.get(), false); - if (!swLocks.isOK()) { - LOGV2_DEBUG(7677701, - 1, - "Index build: lock acquisition for self-abort failed, will retry.", - "buildUUD"_attr = replState->buildUUID, - "error"_attr = swLocks.getStatus()); - continue; - } - - const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); - auto replCoord = repl::ReplicationCoordinator::get(abortCtx); - if (!replCoord->canAcceptWritesFor(abortCtx, dbAndUUID)) { - // Index builds may not fail on secondaries. If a primary replicated an - // abortIndexBuild oplog entry, then this index build would have received an - // IndexBuildAborted error code. - fassert(51101, - status.withContext(str::stream() - << "Index build: " << replState->buildUUID - << "; Database: " << replState->dbName)); - } - - AutoGetCollection indexBuildEntryColl( - abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); - _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); + // Take RSTL (implicitly by DBLock) to observe and prevent replication state from + // changing. + Lock::DBLock dbLock(abortCtx, replState->dbName, MODE_IX); + + // Index builds may not fail on secondaries. If a primary replicated an abortIndexBuild + // oplog entry, then this index build would have received an IndexBuildAborted error + // code. + const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); + auto replCoord = repl::ReplicationCoordinator::get(abortCtx); + if (!replCoord->canAcceptWritesFor(abortCtx, dbAndUUID)) { + fassert(51101, + status.withContext(str::stream() << "Index build: " << replState->buildUUID + << "; Database: " << replState->dbName)); } + + Lock::CollectionLock collLock(abortCtx, dbAndUUID, MODE_X); + AutoGetCollection indexBuildEntryColl( + abortCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + _completeSelfAbort(abortCtx, replState, *indexBuildEntryColl, status); }); } @@ -2399,13 +2311,6 @@ void IndexBuildsCoordinator::_runIndexBuildInner( uassertStatusOK(status); } - // It is also possible for the concurrent abort to happen after the check. This is an issue as - // external aborters hold the collection MODE_X lock while waiting for this thread to signal the - // promise, but if this thread proceeds beyond this check first it will try to acquire the - // collection lock before signaling the promise, potentially creating a deadlock. This is worked - // around by adding a timeout to the collection lock in the self-abort path, and rechecking if - // the build was aborted externally on timeout. - // We do not hold a collection lock here, but we are protected against the collection being // dropped while the index build is still registered for the collection -- until abortIndexBuild // is called. The collection can be renamed, but it is OK for the name to be stale just for @@ -2729,22 +2634,42 @@ IndexBuildsCoordinator::CommitResult IndexBuildsCoordinator::_insertKeysFromSide hangIndexBuildBeforeCommit.pauseWhileSet(); } - // Need to return the collection lock back to exclusive mode to complete the index build. - auto locksOrStatus = - _autoGetCollectionExclusiveWithTimeout(opCtx, replState.get(), /*retry=*/false); - if (!locksOrStatus.isOK()) { - return CommitResult::kLockTimeout; - } + Lock::DBLock autoDb(opCtx, replState->dbName, MODE_IX); + + // Unlock RSTL to avoid deadlocks with prepare conflicts and state transitions caused by waiting + // for a a strong collection lock. See SERVER-42621. + unlockRSTL(opCtx); + // Need to return the collection lock back to exclusive mode to complete the index build. + const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); + Lock::CollectionLock collLock(opCtx, dbAndUUID, MODE_X); AutoGetCollection indexBuildEntryColl( opCtx, NamespaceString::kIndexBuildEntryNamespace, MODE_IX); + // If we can't acquire the RSTL within a given time period, there is an active state transition + // and we should release our locks and try again. We would otherwise introduce a deadlock with + // step-up by holding the Collection lock in exclusive mode. After it has enqueued its RSTL X + // lock, step-up tries to reacquire the Collection locks for prepared transactions, which will + // conflict with the X lock we currently hold. + repl::ReplicationStateTransitionLockGuard rstl( + opCtx, MODE_IX, repl::ReplicationStateTransitionLockGuard::EnqueueOnly()); + auto replCoord = repl::ReplicationCoordinator::get(opCtx); + try { + // Since this thread is not killable by state transitions, this deadline is effectively the + // longest period of time we can block a step-up. State transitions are infrequent, but + // need to happen quickly. It should be okay to set this to a low value because the RSTL is + // rarely contended, and if this times out, we will retry and reacquire the RSTL again + // without a deadline at the beginning of this function. + auto deadline = Date_t::now() + Milliseconds(10); + rstl.waitForLockUntil(deadline); + } catch (const ExceptionFor<ErrorCodes::LockTimeout>&) { + return CommitResult::kLockTimeout; + } + // If we are no longer primary after receiving a commit quorum, we must restart and wait for a // new signal from a new primary because we cannot commit. Note that two-phase index builds can // retry because a new signal should be received. Single-phase builds will be unable to commit // and will self-abort. - auto replCoord = repl::ReplicationCoordinator::get(opCtx); - const NamespaceStringOrUUID dbAndUUID(replState->dbName, replState->collectionUUID); bool isPrimary = replCoord->canAcceptWritesFor(opCtx, dbAndUUID) && !replCoord->getSettings().shouldRecoverFromOplogAsStandalone(); if (!isPrimary && IndexBuildAction::kCommitQuorumSatisfied == action) { @@ -3018,46 +2943,8 @@ std::vector<BSONObj> IndexBuildsCoordinator::prepareSpecListForCreate( // During secondary oplog application, the index specs have already been normalized in the // oplog entries read from the primary. We should not be modifying the specs any further. - auto indexCatalog = collection->getIndexCatalog(); auto replCoord = repl::ReplicationCoordinator::get(opCtx); if (!replCoord->canAcceptWritesFor(opCtx, nss)) { - // A secondary node with a subset of the indexes already built will not vote for the commit - // quorum, which can stall the index build indefinitely on a replica set. - auto specsToBuild = indexCatalog->removeExistingIndexes( - opCtx, collection, indexSpecs, /*removeIndexBuildsToo=*/true); - if (indexSpecs.size() != specsToBuild.size()) { - if (specsToBuild.size() == 0) { - LOGV2_WARNING( - 7731100, - "Secondary node already has all indexes built, which can happen as a result of " - "a previous, incomplete rolling index build. The node will not proceed with " - "the index build, and consequently will not participate in voting towards the " - "commit quorum. Use the 'setIndexCommitQuorum' command to adjust the commit " - "quorum accordingly. Caveat: to ensure the index build completes, this node " - "should not become primary for the duration of the build; step it down if it " - "happens", - logAttrs(nss), - logAttrs(collection->uuid()), - "requestedSpecs"_attr = indexSpecs, - "specsToBuild"_attr = specsToBuild); - } else { - LOGV2_WARNING( - 7731101, - "Secondary node already has a subset of indexes built, which can happen as a " - "result of a previous, incomplete rolling index build. The node will not " - "proceed with the index build, and consequently will not participate in voting " - "towards the commit quorum. Use the 'setIndexCommitQuorum' command to adjust " - "the commit quorum accordingly. Caveat: to ensure the index build completes, " - "this node should not become primary for the duration of the build; step it " - "down if it happens. Additionally, this node will be missing a subset of the " - "indices present in the rest of the replica set. To remediate this, manually " - "build the missing indexes on this node as a standalone.", - logAttrs(nss), - logAttrs(collection->uuid()), - "requestedSpecs"_attr = indexSpecs, - "specsToBuild"_attr = specsToBuild); - } - } return indexSpecs; } @@ -3065,12 +2952,13 @@ std::vector<BSONObj> IndexBuildsCoordinator::prepareSpecListForCreate( auto normalSpecs = normalizeIndexSpecs(opCtx, collection, indexSpecs); // Remove any index specifications which already exist in the catalog. + auto indexCatalog = collection->getIndexCatalog(); auto resultSpecs = indexCatalog->removeExistingIndexes( opCtx, collection, normalSpecs, true /*removeIndexBuildsToo*/); // Verify that each spec is compatible with the collection's sharding state. for (const BSONObj& spec : resultSpecs) { - if (spec[kUniqueFieldName].trueValue() || spec[kPrepareUniqueFieldName].trueValue()) { + if (spec[kUniqueFieldName].trueValue()) { checkShardKeyRestrictions(opCtx, nss, spec[kKeyFieldName].Obj()); } } @@ -3078,7 +2966,6 @@ std::vector<BSONObj> IndexBuildsCoordinator::prepareSpecListForCreate( return resultSpecs; } -// Returns normalized versions of 'indexSpecs' for the catalog. std::vector<BSONObj> IndexBuildsCoordinator::normalizeIndexSpecs( OperationContext* opCtx, const CollectionPtr& collection, @@ -3100,13 +2987,26 @@ std::vector<BSONObj> IndexBuildsCoordinator::normalizeIndexSpecs( // for clients to validate (via the listIndexes output) whether a given partialFilterExpression // is equivalent to the filter that they originally submitted. Omitting this normalization does // not impact our internal index comparison semantics, since we compare based on the parsed - // MatchExpression trees rather than the serialized BSON specs. - // - // For similar reasons we do not normalize index projection objects here, if any, so their - // original forms get persisted in the catalog. Projection normalization to detect whether a - // candidate new index would duplicate an existing index is done only in the memory-only - // 'IndexDescriptor._normalizedProjection' field. - + // MatchExpression trees rather than the serialized BSON specs. See SERVER-54357. + + // If any of the specs describe wildcard indexes, normalize the wildcard projections if present. + // This will change all specs of the form {"a.b.c": 1} to normalized form {a: {b: {c : 1}}}. + std::transform(normalSpecs.begin(), normalSpecs.end(), normalSpecs.begin(), [](auto& spec) { + const auto kProjectionName = IndexDescriptor::kPathProjectionFieldName; + const auto pathProjectionSpec = spec.getObjectField(kProjectionName); + static const auto kWildcardKeyPattern = BSON("$**" << 1); + // It's illegal for the user to explicitly specify an empty wildcardProjection for creating + // a {"$**":1} index, and specify any wildcardProjection for a {"field.$**": 1} index. If + // the projection is empty, then it means that there is no projection to normalize. + if (pathProjectionSpec.isEmpty()) { + return spec; + } + auto wildcardProjection = + WildcardKeyGenerator::createProjectionExecutor(kWildcardKeyPattern, pathProjectionSpec); + auto normalizedProjection = + wildcardProjection.exec()->serializeTransformation(boost::none).toBson(); + return spec.addField(BSON(kProjectionName << normalizedProjection).firstElement()); + }); return normalSpecs; } |