Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 67 insertions, 98 deletions

diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 21f04d4445a..5f5e0de5a9d 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -35,13 +35,11 @@
 
 #include "mongo/db/audit.h"
 #include "mongo/db/catalog/catalog_control.h"
-#include "mongo/db/catalog/clustered_collection_util.h"
 #include "mongo/db/catalog/collection_catalog.h"
 #include "mongo/db/catalog/collection_catalog_helper.h"
 #include "mongo/db/catalog_raii.h"
 #include "mongo/db/client.h"
 #include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/concurrency/lock_state.h"
 #include "mongo/db/index_builds_coordinator.h"
 #include "mongo/db/multitenancy.h"
 #include "mongo/db/operation_context.h"
@@ -86,13 +84,10 @@ StorageEngineImpl::StorageEngineImpl(OperationContext* opCtx,
       _dropPendingIdentReaper(_engine.get()),
       _minOfCheckpointAndOldestTimestampListener(
           TimestampMonitor::TimestampType::kMinOfCheckpointAndOldest,
-          [this](OperationContext* opCtx, Timestamp timestamp) {
-              _onMinOfCheckpointAndOldestTimestampChanged(opCtx, timestamp);
-          }),
+          [this](Timestamp timestamp) { _onMinOfCheckpointAndOldestTimestampChanged(timestamp); }),
       _historicalIdentTimestampListener(
           TimestampMonitor::TimestampType::kCheckpoint,
-          [serviceContext = opCtx->getServiceContext()](OperationContext* opCtx,
-                                                        Timestamp timestamp) {
+          [serviceContext = opCtx->getServiceContext()](Timestamp timestamp) {
               HistoricalIdentTracker::get(serviceContext).removeEntriesOlderThan(timestamp);
           }),
       _supportsCappedCollections(_engine->supportsCappedCollections()) {
@@ -197,7 +192,7 @@ void StorageEngineImpl::loadCatalog(OperationContext* opCtx, LastShutdownState l
         // 'local.orphan.xxxxx' for it. However, in a nonrepair context, the orphaned idents
         // will be dropped in reconcileCatalogAndIdents().
         for (const auto& ident : identsKnownToStorageEngine) {
-            if (DurableCatalog::isCollectionIdent(ident)) {
+            if (_catalog->isCollectionIdent(ident)) {
                 bool isOrphan = !std::any_of(
                     catalogEntries.begin(),
                     catalogEntries.end(),
@@ -206,36 +201,19 @@ void StorageEngineImpl::loadCatalog(OperationContext* opCtx, LastShutdownState l
                     // If the catalog does not have information about this
                     // collection, we create an new entry for it.
                     WriteUnitOfWork wuow(opCtx);
-
-                    auto keyFormat = _engine->getKeyFormat(opCtx, ident);
-                    bool isClustered = keyFormat == KeyFormat::String;
-                    CollectionOptions optionsWithUUID;
-                    optionsWithUUID.uuid.emplace(UUID::gen());
-                    if (isClustered) {
-                        optionsWithUUID.clusteredIndex =
-                            clustered_util::makeDefaultClusteredIdIndex();
-                    }
-
-                    StatusWith<std::string> statusWithNs =
-                        _catalog->newOrphanedIdent(opCtx, ident, optionsWithUUID);
-
+                    StatusWith<std::string> statusWithNs = _catalog->newOrphanedIdent(opCtx, ident);
                     if (statusWithNs.isOK()) {
                         wuow.commit();
                         auto orphanCollNs = statusWithNs.getValue();
                         LOGV2(22247,
                               "Successfully created an entry in the catalog for orphaned "
                               "collection",
-                              "namespace"_attr = orphanCollNs,
-                              "options"_attr = optionsWithUUID);
-
-                        if (!isClustered) {
-                            // The _id index is already implicitly created on collections clustered
-                            // by _id.
-                            LOGV2_WARNING(22265,
-                                          "Collection does not have an _id index. Please manually "
-                                          "build the index",
-                                          "namespace"_attr = orphanCollNs);
-                        }
+                              "namespace"_attr = orphanCollNs);
+                        LOGV2_WARNING(22265,
+                                      "Collection does not have an _id index. Please manually "
+                                      "build the index",
+                                      "namespace"_attr = orphanCollNs);
+
                         StorageRepairObserver::get(getGlobalServiceContext())
                             ->benignModification(str::stream() << "Orphan collection created: "
                                                                << statusWithNs.getValue());
@@ -407,7 +385,7 @@ void StorageEngineImpl::_initCollection(OperationContext* opCtx,
     collection->setMinimumVisibleSnapshot(minVisibleTs);
 
     CollectionCatalog::write(opCtx, [&](CollectionCatalog& catalog) {
-        catalog.registerCollection(opCtx, std::move(collection));
+        catalog.registerCollection(opCtx, md->options.uuid.get(), std::move(collection));
     });
 }
 
@@ -614,7 +592,7 @@ StatusWith<StorageEngine::ReconcileResult> StorageEngineImpl::reconcileCatalogAn
 
         // In repair context, any orphaned collection idents from the engine should already be
         // recovered in the catalog in loadCatalog().
-        invariant(!(DurableCatalog::isCollectionIdent(it) && _options.forRepair));
+        invariant(!(_catalog->isCollectionIdent(it) && _options.forRepair));
 
         // Leave drop-pending idents alone.
         // These idents have to be retained as long as the corresponding drops are not part of a
@@ -687,19 +665,21 @@ StatusWith<StorageEngine::ReconcileResult> StorageEngineImpl::reconcileCatalogAn
                     logAttrs(nss));
             }
 
-            if (!engineIdents.count(indexIdent)) {
-                // There are cetain cases where the catalog entry may reference an index ident which
-                // is no longer present. One example of this is when an unclean shutdown occurs
-                // before a checkpoint is taken during startup recovery. Since we drop the index
-                // ident without a timestamp when restarting the index build for startup recovery,
-                // the subsequent startup recovery can see the now-dropped ident referenced by the
-                // old index catalog entry.
-                LOGV2(6386500,
-                      "Index catalog entry ident not found",
-                      "ident"_attr = indexIdent,
-                      "entry"_attr = indexMetaData.spec,
-                      logAttrs(nss));
-            }
+            // Two-phase index drop ensures that the underlying data table for an index in the
+            // catalog is not dropped until the index removal from the catalog has been majority
+            // committed and become part of the latest checkpoint. Therefore, there should almost
+            // never be a case where the index catalog entry remains but the index table (identified
+            // by ident) has been removed.
+            //
+            // There is an exception to this due to the fact that we drop the index ident without a
+            // timestamp when restarting an index build for startup recovery. Then, if we experience
+            // an unclean shutdown before a checkpoint is taken, the subsequent startup recovery can
+            // see the now-dropped ident referenced by the old index catalog entry.
+            invariant(engineIdents.find(indexIdent) != engineIdents.end() ||
+                          lastShutdownState == LastShutdownState::kUnclean,
+                      str::stream() << "Failed to find an index data table matching " << indexIdent
+                                    << " for durable index catalog entry " << indexMetaData.spec
+                                    << " in collection " << nss.ns());
 
             // Any index build with a UUID is an unfinished two-phase build and must be restarted.
             // There are no special cases to handle on primaries or secondaries. An index build may
@@ -797,16 +777,19 @@ std::string StorageEngineImpl::getFilesystemPathForDb(
 }
 
 void StorageEngineImpl::cleanShutdown() {
-    _timestampMonitor.reset();
+    if (_timestampMonitor) {
+        _timestampMonitor->clearListeners();
+    }
 
     CollectionCatalog::write(getGlobalServiceContext(), [](CollectionCatalog& catalog) {
-        catalog.onCloseCatalog();
         catalog.deregisterAllCollectionsAndViews();
     });
 
     _catalog.reset();
     _catalogRecordStore.reset();
 
+    _timestampMonitor.reset();
+
     _engine->cleanShutdown();
     // intentionally not deleting _engine
 }
@@ -901,10 +884,8 @@ Status StorageEngineImpl::_dropCollectionsNoTimestamp(OperationContext* opCtx,
 
         // No need to remove the indexes from the IndexCatalog because eliminating the Collection
         // will have the same effect.
-        auto ii = coll->getIndexCatalog()->getIndexIterator(
-            opCtx,
-            IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished |
-                IndexCatalog::InclusionPolicy::kFrozen);
+        auto ii =
+            coll->getIndexCatalog()->getIndexIterator(opCtx, true /* includeUnfinishedIndexes */);
         while (ii->more()) {
             const IndexCatalogEntry* ice = ii->next();
 
@@ -1172,37 +1153,28 @@ void StorageEngineImpl::_dumpCatalog(OperationContext* opCtx) {
     opCtx->recoveryUnit()->abandonSnapshot();
 }
 
-void StorageEngineImpl::addDropPendingIdent(
-    const stdx::variant<Timestamp, StorageEngine::CheckpointIteration>& dropTime,
-    std::shared_ptr<Ident> ident,
-    DropIdentCallback&& onDrop) {
-    _dropPendingIdentReaper.addDropPendingIdent(dropTime, ident, std::move(onDrop));
+void StorageEngineImpl::addDropPendingIdent(const Timestamp& dropTimestamp,
+                                            std::shared_ptr<Ident> ident,
+                                            DropIdentCallback&& onDrop) {
+    _dropPendingIdentReaper.addDropPendingIdent(dropTimestamp, ident, std::move(onDrop));
 }
 
 void StorageEngineImpl::checkpoint() {
     _engine->checkpoint();
 }
 
-StorageEngine::CheckpointIteration StorageEngineImpl::getCheckpointIteration() const {
-    return _engine->getCheckpointIteration();
-}
-
-bool StorageEngineImpl::hasDataBeenCheckpointed(
-    StorageEngine::CheckpointIteration checkpointIteration) const {
-    return _engine->hasDataBeenCheckpointed(checkpointIteration);
-}
+void StorageEngineImpl::_onMinOfCheckpointAndOldestTimestampChanged(const Timestamp& timestamp) {
+    // No drop-pending idents present if getEarliestDropTimestamp() returns boost::none.
+    if (auto earliestDropTimestamp = _dropPendingIdentReaper.getEarliestDropTimestamp()) {
+        if (timestamp >= *earliestDropTimestamp) {
+            LOGV2(22260,
+                  "Removing drop-pending idents with drop timestamps before timestamp",
+                  "timestamp"_attr = timestamp);
+            auto opCtx = cc().getOperationContext();
+            invariant(opCtx);
 
-void StorageEngineImpl::_onMinOfCheckpointAndOldestTimestampChanged(OperationContext* opCtx,
-                                                                    const Timestamp& timestamp) {
-    if (_dropPendingIdentReaper.hasExpiredIdents(timestamp)) {
-        LOGV2(22260,
-              "Removing drop-pending idents with drop timestamps before timestamp",
-              "timestamp"_attr = timestamp);
-
-        _dropPendingIdentReaper.dropIdentsOlderThan(opCtx, timestamp);
-    } else {
-        LOGV2_DEBUG(
-            8097401, 1, "No drop-pending idents have expired", "timestamp"_attr = timestamp);
+            _dropPendingIdentReaper.dropIdentsOlderThan(opCtx, timestamp);
+        }
     }
 }
 
@@ -1213,6 +1185,8 @@ StorageEngineImpl::TimestampMonitor::TimestampMonitor(KVEngine* engine, Periodic
 
 StorageEngineImpl::TimestampMonitor::~TimestampMonitor() {
     LOGV2(22261, "Timestamp monitor shutting down");
+    stdx::lock_guard<Latch> lock(_monitorMutex);
+    invariant(_listeners.empty());
 }
 
 void StorageEngineImpl::TimestampMonitor::_startup() {
@@ -1236,12 +1210,12 @@ void StorageEngineImpl::TimestampMonitor::_startup() {
             }
 
             try {
-                auto uniqueOpCtx = client->makeOperationContext();
-                auto opCtx = uniqueOpCtx.get();
-
-                // The TimestampMonitor is an important background cleanup task for the storage
-                // engine and needs to be able to make progress to free up resources.
-                SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx);
+                auto opCtx = client->getOperationContext();
+                mongo::ServiceContext::UniqueOperationContext uOpCtx;
+                if (!opCtx) {
+                    uOpCtx = client->makeOperationContext();
+                    opCtx = uOpCtx.get();
+                }
 
                 Timestamp checkpoint;
                 Timestamp oldest;
@@ -1268,19 +1242,19 @@ void StorageEngineImpl::TimestampMonitor::_startup() {
                     stdx::lock_guard<Latch> lock(_monitorMutex);
                     for (const auto& listener : _listeners) {
                         if (listener->getType() == TimestampType::kCheckpoint) {
-                            listener->notify(opCtx, checkpoint);
+                            listener->notify(checkpoint);
                         } else if (listener->getType() == TimestampType::kOldest) {
-                            listener->notify(opCtx, oldest);
+                            listener->notify(oldest);
                         } else if (listener->getType() == TimestampType::kStable) {
-                            listener->notify(opCtx, stable);
+                            listener->notify(stable);
                         } else if (listener->getType() ==
                                    TimestampType::kMinOfCheckpointAndOldest) {
-                            listener->notify(opCtx, minOfCheckpointAndOldest);
+                            listener->notify(minOfCheckpointAndOldest);
                         } else if (stable == Timestamp::min()) {
                             // Special case notification of all listeners when writes do not have
                             // timestamps. This handles standalone mode and storage engines that
                             // don't support timestamps.
-                            listener->notify(opCtx, Timestamp::min());
+                            listener->notify(Timestamp::min());
                         }
                     }
                 }
@@ -1338,9 +1312,7 @@ int64_t StorageEngineImpl::sizeOnDiskForDb(OperationContext* opCtx,
     auto perCollectionWork = [&](const CollectionPtr& collection) {
         size += collection->getRecordStore()->storageSize(opCtx);
 
-        auto it = collection->getIndexCatalog()->getIndexIterator(
-            opCtx,
-            IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+        auto it = collection->getIndexCatalog()->getIndexIterator(opCtx, true);
         while (it->more()) {
             size += _engine->getIdentSize(opCtx, it->next()->getIdent());
         }
@@ -1350,8 +1322,10 @@ int64_t StorageEngineImpl::sizeOnDiskForDb(OperationContext* opCtx,
 
     if (opCtx->isLockFreeReadsOp()) {
         auto collectionCatalog = CollectionCatalog::get(opCtx);
-        for (auto&& coll : collectionCatalog->range(tenantDbName)) {
-            perCollectionWork(coll);
+        for (auto it = collectionCatalog->begin(opCtx, tenantDbName);
+             it != collectionCatalog->end(opCtx);
+             ++it) {
+            perCollectionWork(*it);
         }
     } else {
         catalog::forEachCollectionFromDb(opCtx, tenantDbName, MODE_IS, perCollectionWork);
@@ -1385,11 +1359,6 @@ const DurableCatalog* StorageEngineImpl::getCatalog() const {
     return _catalog.get();
 }
 
-BSONObj StorageEngineImpl::getSanitizedStorageOptionsForSecondaryReplication(
-    const BSONObj& options) const {
-    return _engine->getSanitizedStorageOptionsForSecondaryReplication(options);
-}
-
 void StorageEngineImpl::dump() const {
     _engine->dump();
 }