diff options
Diffstat (limited to 'src/mongo/db/storage/storage_engine_impl.cpp')
| -rw-r--r-- | src/mongo/db/storage/storage_engine_impl.cpp | 165 |
1 files changed, 67 insertions, 98 deletions
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 21f04d4445a..5f5e0de5a9d 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -35,13 +35,11 @@ #include "mongo/db/audit.h" #include "mongo/db/catalog/catalog_control.h" -#include "mongo/db/catalog/clustered_collection_util.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_catalog_helper.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/multitenancy.h" #include "mongo/db/operation_context.h" @@ -86,13 +84,10 @@ StorageEngineImpl::StorageEngineImpl(OperationContext* opCtx, _dropPendingIdentReaper(_engine.get()), _minOfCheckpointAndOldestTimestampListener( TimestampMonitor::TimestampType::kMinOfCheckpointAndOldest, - [this](OperationContext* opCtx, Timestamp timestamp) { - _onMinOfCheckpointAndOldestTimestampChanged(opCtx, timestamp); - }), + [this](Timestamp timestamp) { _onMinOfCheckpointAndOldestTimestampChanged(timestamp); }), _historicalIdentTimestampListener( TimestampMonitor::TimestampType::kCheckpoint, - [serviceContext = opCtx->getServiceContext()](OperationContext* opCtx, - Timestamp timestamp) { + [serviceContext = opCtx->getServiceContext()](Timestamp timestamp) { HistoricalIdentTracker::get(serviceContext).removeEntriesOlderThan(timestamp); }), _supportsCappedCollections(_engine->supportsCappedCollections()) { @@ -197,7 +192,7 @@ void StorageEngineImpl::loadCatalog(OperationContext* opCtx, LastShutdownState l // 'local.orphan.xxxxx' for it. However, in a nonrepair context, the orphaned idents // will be dropped in reconcileCatalogAndIdents(). for (const auto& ident : identsKnownToStorageEngine) { - if (DurableCatalog::isCollectionIdent(ident)) { + if (_catalog->isCollectionIdent(ident)) { bool isOrphan = !std::any_of( catalogEntries.begin(), catalogEntries.end(), @@ -206,36 +201,19 @@ void StorageEngineImpl::loadCatalog(OperationContext* opCtx, LastShutdownState l // If the catalog does not have information about this // collection, we create an new entry for it. WriteUnitOfWork wuow(opCtx); - - auto keyFormat = _engine->getKeyFormat(opCtx, ident); - bool isClustered = keyFormat == KeyFormat::String; - CollectionOptions optionsWithUUID; - optionsWithUUID.uuid.emplace(UUID::gen()); - if (isClustered) { - optionsWithUUID.clusteredIndex = - clustered_util::makeDefaultClusteredIdIndex(); - } - - StatusWith<std::string> statusWithNs = - _catalog->newOrphanedIdent(opCtx, ident, optionsWithUUID); - + StatusWith<std::string> statusWithNs = _catalog->newOrphanedIdent(opCtx, ident); if (statusWithNs.isOK()) { wuow.commit(); auto orphanCollNs = statusWithNs.getValue(); LOGV2(22247, "Successfully created an entry in the catalog for orphaned " "collection", - "namespace"_attr = orphanCollNs, - "options"_attr = optionsWithUUID); - - if (!isClustered) { - // The _id index is already implicitly created on collections clustered - // by _id. - LOGV2_WARNING(22265, - "Collection does not have an _id index. Please manually " - "build the index", - "namespace"_attr = orphanCollNs); - } + "namespace"_attr = orphanCollNs); + LOGV2_WARNING(22265, + "Collection does not have an _id index. Please manually " + "build the index", + "namespace"_attr = orphanCollNs); + StorageRepairObserver::get(getGlobalServiceContext()) ->benignModification(str::stream() << "Orphan collection created: " << statusWithNs.getValue()); @@ -407,7 +385,7 @@ void StorageEngineImpl::_initCollection(OperationContext* opCtx, collection->setMinimumVisibleSnapshot(minVisibleTs); CollectionCatalog::write(opCtx, [&](CollectionCatalog& catalog) { - catalog.registerCollection(opCtx, std::move(collection)); + catalog.registerCollection(opCtx, md->options.uuid.get(), std::move(collection)); }); } @@ -614,7 +592,7 @@ StatusWith<StorageEngine::ReconcileResult> StorageEngineImpl::reconcileCatalogAn // In repair context, any orphaned collection idents from the engine should already be // recovered in the catalog in loadCatalog(). - invariant(!(DurableCatalog::isCollectionIdent(it) && _options.forRepair)); + invariant(!(_catalog->isCollectionIdent(it) && _options.forRepair)); // Leave drop-pending idents alone. // These idents have to be retained as long as the corresponding drops are not part of a @@ -687,19 +665,21 @@ StatusWith<StorageEngine::ReconcileResult> StorageEngineImpl::reconcileCatalogAn logAttrs(nss)); } - if (!engineIdents.count(indexIdent)) { - // There are cetain cases where the catalog entry may reference an index ident which - // is no longer present. One example of this is when an unclean shutdown occurs - // before a checkpoint is taken during startup recovery. Since we drop the index - // ident without a timestamp when restarting the index build for startup recovery, - // the subsequent startup recovery can see the now-dropped ident referenced by the - // old index catalog entry. - LOGV2(6386500, - "Index catalog entry ident not found", - "ident"_attr = indexIdent, - "entry"_attr = indexMetaData.spec, - logAttrs(nss)); - } + // Two-phase index drop ensures that the underlying data table for an index in the + // catalog is not dropped until the index removal from the catalog has been majority + // committed and become part of the latest checkpoint. Therefore, there should almost + // never be a case where the index catalog entry remains but the index table (identified + // by ident) has been removed. + // + // There is an exception to this due to the fact that we drop the index ident without a + // timestamp when restarting an index build for startup recovery. Then, if we experience + // an unclean shutdown before a checkpoint is taken, the subsequent startup recovery can + // see the now-dropped ident referenced by the old index catalog entry. + invariant(engineIdents.find(indexIdent) != engineIdents.end() || + lastShutdownState == LastShutdownState::kUnclean, + str::stream() << "Failed to find an index data table matching " << indexIdent + << " for durable index catalog entry " << indexMetaData.spec + << " in collection " << nss.ns()); // Any index build with a UUID is an unfinished two-phase build and must be restarted. // There are no special cases to handle on primaries or secondaries. An index build may @@ -797,16 +777,19 @@ std::string StorageEngineImpl::getFilesystemPathForDb( } void StorageEngineImpl::cleanShutdown() { - _timestampMonitor.reset(); + if (_timestampMonitor) { + _timestampMonitor->clearListeners(); + } CollectionCatalog::write(getGlobalServiceContext(), [](CollectionCatalog& catalog) { - catalog.onCloseCatalog(); catalog.deregisterAllCollectionsAndViews(); }); _catalog.reset(); _catalogRecordStore.reset(); + _timestampMonitor.reset(); + _engine->cleanShutdown(); // intentionally not deleting _engine } @@ -901,10 +884,8 @@ Status StorageEngineImpl::_dropCollectionsNoTimestamp(OperationContext* opCtx, // No need to remove the indexes from the IndexCatalog because eliminating the Collection // will have the same effect. - auto ii = coll->getIndexCatalog()->getIndexIterator( - opCtx, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + auto ii = + coll->getIndexCatalog()->getIndexIterator(opCtx, true /* includeUnfinishedIndexes */); while (ii->more()) { const IndexCatalogEntry* ice = ii->next(); @@ -1172,37 +1153,28 @@ void StorageEngineImpl::_dumpCatalog(OperationContext* opCtx) { opCtx->recoveryUnit()->abandonSnapshot(); } -void StorageEngineImpl::addDropPendingIdent( - const stdx::variant<Timestamp, StorageEngine::CheckpointIteration>& dropTime, - std::shared_ptr<Ident> ident, - DropIdentCallback&& onDrop) { - _dropPendingIdentReaper.addDropPendingIdent(dropTime, ident, std::move(onDrop)); +void StorageEngineImpl::addDropPendingIdent(const Timestamp& dropTimestamp, + std::shared_ptr<Ident> ident, + DropIdentCallback&& onDrop) { + _dropPendingIdentReaper.addDropPendingIdent(dropTimestamp, ident, std::move(onDrop)); } void StorageEngineImpl::checkpoint() { _engine->checkpoint(); } -StorageEngine::CheckpointIteration StorageEngineImpl::getCheckpointIteration() const { - return _engine->getCheckpointIteration(); -} - -bool StorageEngineImpl::hasDataBeenCheckpointed( - StorageEngine::CheckpointIteration checkpointIteration) const { - return _engine->hasDataBeenCheckpointed(checkpointIteration); -} +void StorageEngineImpl::_onMinOfCheckpointAndOldestTimestampChanged(const Timestamp& timestamp) { + // No drop-pending idents present if getEarliestDropTimestamp() returns boost::none. + if (auto earliestDropTimestamp = _dropPendingIdentReaper.getEarliestDropTimestamp()) { + if (timestamp >= *earliestDropTimestamp) { + LOGV2(22260, + "Removing drop-pending idents with drop timestamps before timestamp", + "timestamp"_attr = timestamp); + auto opCtx = cc().getOperationContext(); + invariant(opCtx); -void StorageEngineImpl::_onMinOfCheckpointAndOldestTimestampChanged(OperationContext* opCtx, - const Timestamp& timestamp) { - if (_dropPendingIdentReaper.hasExpiredIdents(timestamp)) { - LOGV2(22260, - "Removing drop-pending idents with drop timestamps before timestamp", - "timestamp"_attr = timestamp); - - _dropPendingIdentReaper.dropIdentsOlderThan(opCtx, timestamp); - } else { - LOGV2_DEBUG( - 8097401, 1, "No drop-pending idents have expired", "timestamp"_attr = timestamp); + _dropPendingIdentReaper.dropIdentsOlderThan(opCtx, timestamp); + } } } @@ -1213,6 +1185,8 @@ StorageEngineImpl::TimestampMonitor::TimestampMonitor(KVEngine* engine, Periodic StorageEngineImpl::TimestampMonitor::~TimestampMonitor() { LOGV2(22261, "Timestamp monitor shutting down"); + stdx::lock_guard<Latch> lock(_monitorMutex); + invariant(_listeners.empty()); } void StorageEngineImpl::TimestampMonitor::_startup() { @@ -1236,12 +1210,12 @@ void StorageEngineImpl::TimestampMonitor::_startup() { } try { - auto uniqueOpCtx = client->makeOperationContext(); - auto opCtx = uniqueOpCtx.get(); - - // The TimestampMonitor is an important background cleanup task for the storage - // engine and needs to be able to make progress to free up resources. - SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx); + auto opCtx = client->getOperationContext(); + mongo::ServiceContext::UniqueOperationContext uOpCtx; + if (!opCtx) { + uOpCtx = client->makeOperationContext(); + opCtx = uOpCtx.get(); + } Timestamp checkpoint; Timestamp oldest; @@ -1268,19 +1242,19 @@ void StorageEngineImpl::TimestampMonitor::_startup() { stdx::lock_guard<Latch> lock(_monitorMutex); for (const auto& listener : _listeners) { if (listener->getType() == TimestampType::kCheckpoint) { - listener->notify(opCtx, checkpoint); + listener->notify(checkpoint); } else if (listener->getType() == TimestampType::kOldest) { - listener->notify(opCtx, oldest); + listener->notify(oldest); } else if (listener->getType() == TimestampType::kStable) { - listener->notify(opCtx, stable); + listener->notify(stable); } else if (listener->getType() == TimestampType::kMinOfCheckpointAndOldest) { - listener->notify(opCtx, minOfCheckpointAndOldest); + listener->notify(minOfCheckpointAndOldest); } else if (stable == Timestamp::min()) { // Special case notification of all listeners when writes do not have // timestamps. This handles standalone mode and storage engines that // don't support timestamps. - listener->notify(opCtx, Timestamp::min()); + listener->notify(Timestamp::min()); } } } @@ -1338,9 +1312,7 @@ int64_t StorageEngineImpl::sizeOnDiskForDb(OperationContext* opCtx, auto perCollectionWork = [&](const CollectionPtr& collection) { size += collection->getRecordStore()->storageSize(opCtx); - auto it = collection->getIndexCatalog()->getIndexIterator( - opCtx, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + auto it = collection->getIndexCatalog()->getIndexIterator(opCtx, true); while (it->more()) { size += _engine->getIdentSize(opCtx, it->next()->getIdent()); } @@ -1350,8 +1322,10 @@ int64_t StorageEngineImpl::sizeOnDiskForDb(OperationContext* opCtx, if (opCtx->isLockFreeReadsOp()) { auto collectionCatalog = CollectionCatalog::get(opCtx); - for (auto&& coll : collectionCatalog->range(tenantDbName)) { - perCollectionWork(coll); + for (auto it = collectionCatalog->begin(opCtx, tenantDbName); + it != collectionCatalog->end(opCtx); + ++it) { + perCollectionWork(*it); } } else { catalog::forEachCollectionFromDb(opCtx, tenantDbName, MODE_IS, perCollectionWork); @@ -1385,11 +1359,6 @@ const DurableCatalog* StorageEngineImpl::getCatalog() const { return _catalog.get(); } -BSONObj StorageEngineImpl::getSanitizedStorageOptionsForSecondaryReplication( - const BSONObj& options) const { - return _engine->getSanitizedStorageOptionsForSecondaryReplication(options); -} - void StorageEngineImpl::dump() const { _engine->dump(); } |