Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 49 insertions, 215 deletions

diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 9b750af05ed..453c17c6688 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -45,7 +45,6 @@
 #include "mongo/db/auth/action_type.h"
 #include "mongo/db/auth/authorization_manager.h"
 #include "mongo/db/auth/privilege.h"
-#include "mongo/db/catalog/backwards_compatible_collection_options_util.h"
 #include "mongo/db/catalog/capped_utils.h"
 #include "mongo/db/catalog/coll_mod.h"
 #include "mongo/db/catalog/collection.h"
@@ -56,7 +55,6 @@
 #include "mongo/db/catalog/drop_collection.h"
 #include "mongo/db/catalog/drop_database.h"
 #include "mongo/db/catalog/drop_indexes.h"
-#include "mongo/db/catalog/health_log_interface.h"
 #include "mongo/db/catalog/import_collection_oplog_entry_gen.h"
 #include "mongo/db/catalog/local_oplog_info.h"
 #include "mongo/db/catalog/multi_index_block.h"
@@ -64,9 +62,8 @@
 #include "mongo/db/client.h"
 #include "mongo/db/coll_mod_gen.h"
 #include "mongo/db/commands.h"
-#include "mongo/db/commands/create_gen.h"
 #include "mongo/db/commands/feature_compatibility_version_parser.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/dbhelpers.h"
@@ -120,7 +117,6 @@ using std::string;
 using std::stringstream;
 using std::unique_ptr;
 using std::vector;
-using namespace std::string_literals;
 
 using IndexVersion = IndexDescriptor::IndexVersion;
 
@@ -293,6 +289,23 @@ void writeToImageCollection(OperationContext* opCtx,
     DisableDocumentValidation documentValidationDisabler(
         opCtx, DocumentValidationSettings::kDisableInternalValidation);
 
+    BSONObj existingImageEntryBson;
+    Helpers::findOne(opCtx,
+                     autoColl.getCollection(),
+                     BSON("_id" << imageEntry.get_id().toBSON() << "ts" << imageEntry.getTs()),
+                     existingImageEntryBson);
+    if (!existingImageEntryBson.isEmpty()) {
+        auto existingImageEntry = repl::ImageEntry::parse(
+            IDLParserErrorContext("writeToImageCollection"), existingImageEntryBson);
+        uassert(
+            6652600,
+            str::stream()
+                << "Found an existing findAndModify image entry with unexpected content. Found: "
+                << existingImageEntry.toBSON() << ". Expected: " << imageEntry.toBSON(),
+            existingImageEntry.toBSON().woCompare(imageEntry.toBSON()) == 0);
+        return;
+    }
+
     UpdateRequest request;
     request.setNamespaceString(NamespaceString::kConfigImagesNamespace);
     request.setQuery(
@@ -793,21 +806,6 @@ NamespaceString extractNsFromUUIDorNs(OperationContext* opCtx,
     return ui ? extractNsFromUUID(opCtx, ui.get()) : extractNs(ns.db(), cmd);
 }
 
-BSONObj getObjWithSanitizedStorageEngineOptions(OperationContext* opCtx, const BSONObj& cmd) {
-    static_assert(
-        CreateCommand::kStorageEngineFieldName == IndexDescriptor::kStorageEngineFieldName,
-        "Expected storage engine options field to be the same for collections and indexes.");
-
-    if (auto storageEngineElem = cmd[IndexDescriptor::kStorageEngineFieldName]) {
-        auto storageEngine = opCtx->getServiceContext()->getStorageEngine();
-        auto engineObj = storageEngineElem.embeddedObject();
-        auto sanitizedObj =
-            storageEngine->getSanitizedStorageOptionsForSecondaryReplication(engineObj);
-        return cmd.addFields(BSON(IndexDescriptor::kStorageEngineFieldName << sanitizedObj));
-    }
-    return cmd;
-}
-
 using OpApplyFn = std::function<Status(
     OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode)>;
 
@@ -832,9 +830,7 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
     {"create",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
           const auto& ui = entry.getUuid();
-          // Sanitize storage engine options to remove options which might not apply to this node.
-          // See SERVER-68122.
-          const auto cmd = getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
+          const auto& cmd = entry.getObject();
           const NamespaceString nss(extractNs(entry.getNss().db(), cmd));
 
           // Mode SECONDARY steady state replication should not allow create collection to rename an
@@ -844,23 +840,6 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
           // complete.
           const bool allowRenameOutOfTheWay = (mode != repl::OplogApplication::Mode::kSecondary);
 
-          // Check whether there is an open but empty database where the name conflicts with the new
-          // collection's database name. It is possible for a secondary's in-memory database state
-          // to diverge from the primary's, if the primary rolls back the dropDatabase oplog entry
-          // after closing its own in-memory database state. In this case, the primary may accept
-          // creating a new database with a conflicting name to what the secondary still has open.
-          // It is okay to simply close the empty database on the secondary in this case.
-          auto duplicates = DatabaseHolder::get(opCtx)->getNamesWithConflictingCasing(
-              TenantDatabaseName(boost::none, nss.db()));
-          if (duplicates.size() == 1) {
-              auto dupDatabaseIt = duplicates.begin();
-              if (CollectionCatalog::get(opCtx)
-                      ->getAllCollectionUUIDsFromDb(*dupDatabaseIt)
-                      .size() == 0) {
-                  fassert(7727801, dropDatabaseForApplyOps(opCtx, dupDatabaseIt->dbName()).isOK());
-              }
-          }
-
           Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX);
           if (auto idIndexElem = cmd["idIndex"]) {
               // Remove "idIndex" field from command.
@@ -895,10 +874,7 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
       {ErrorCodes::NamespaceExists}}},
     {"createIndexes",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
-          // Sanitize storage engine options to remove options which might not apply to this node.
-          // See SERVER-68122.
-          const auto cmd = getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
-
+          const auto& cmd = entry.getObject();
           if (OplogApplication::Mode::kApplyOpsCmd == mode) {
               return {ErrorCodes::CommandNotSupported,
                       "The createIndexes operation is not supported in applyOps mode"};
@@ -933,12 +909,6 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
                   "Error parsing 'startIndexBuild' oplog entry");
           }
 
-          // Sanitize storage engine options to remove options which might not apply to this node.
-          // See SERVER-68122.
-          for (auto& spec : swOplogEntry.getValue().indexSpecs) {
-              spec = getObjWithSanitizedStorageEngineOptions(opCtx, spec);
-          }
-
           IndexBuildsCoordinator::ApplicationMode applicationMode =
               IndexBuildsCoordinator::ApplicationMode::kNormal;
           if (mode == OplogApplication::Mode::kInitialSync) {
@@ -989,8 +959,7 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
       {ErrorCodes::NamespaceNotFound}}},
     {"collMod",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
-          const auto cmd =
-              backwards_compatible_collection_options::parseCollModCmdFromOplogEntry(entry);
+          const auto& cmd = entry.getObject();
           auto opMsg = OpMsgRequest::fromDBAndBody(entry.getNss().db(), cmd);
           auto collModCmd = CollMod::parse(IDLParserErrorContext("collModOplogEntry"), opMsg);
           const auto nssOrUUID([&collModCmd, &entry, mode]() -> NamespaceStringOrUUID {
@@ -1141,8 +1110,6 @@ void writeChangeStreamPreImage(OperationContext* opCtx,
 
 constexpr StringData OplogApplication::kInitialSyncOplogApplicationMode;
 constexpr StringData OplogApplication::kRecoveringOplogApplicationMode;
-constexpr StringData OplogApplication::kStableRecoveringOplogApplicationMode;
-constexpr StringData OplogApplication::kUnstableRecoveringOplogApplicationMode;
 constexpr StringData OplogApplication::kSecondaryOplogApplicationMode;
 constexpr StringData OplogApplication::kApplyOpsCmdOplogApplicationMode;
 
@@ -1150,10 +1117,8 @@ StringData OplogApplication::modeToString(OplogApplication::Mode mode) {
     switch (mode) {
         case OplogApplication::Mode::kInitialSync:
             return OplogApplication::kInitialSyncOplogApplicationMode;
-        case OplogApplication::Mode::kUnstableRecovering:
-            return OplogApplication::kUnstableRecoveringOplogApplicationMode;
-        case OplogApplication::Mode::kStableRecovering:
-            return OplogApplication::kStableRecoveringOplogApplicationMode;
+        case OplogApplication::Mode::kRecovering:
+            return OplogApplication::kRecoveringOplogApplicationMode;
         case OplogApplication::Mode::kSecondary:
             return OplogApplication::kSecondaryOplogApplicationMode;
         case OplogApplication::Mode::kApplyOpsCmd:
@@ -1166,9 +1131,7 @@ StatusWith<OplogApplication::Mode> OplogApplication::parseMode(const std::string
     if (mode == OplogApplication::kInitialSyncOplogApplicationMode) {
         return OplogApplication::Mode::kInitialSync;
     } else if (mode == OplogApplication::kRecoveringOplogApplicationMode) {
-        // This only being used in applyOps command which is controlled by the client, so it should
-        // be unstable.
-        return OplogApplication::Mode::kUnstableRecovering;
+        return OplogApplication::Mode::kRecovering;
     } else if (mode == OplogApplication::kSecondaryOplogApplicationMode) {
         return OplogApplication::Mode::kSecondary;
     } else if (mode == OplogApplication::kApplyOpsCmdOplogApplicationMode) {
@@ -1180,76 +1143,6 @@ StatusWith<OplogApplication::Mode> OplogApplication::parseMode(const std::string
     MONGO_UNREACHABLE;
 }
 
-void OplogApplication::checkOnOplogFailureForRecovery(OperationContext* opCtx,
-                                                      const mongo::NamespaceString& nss,
-                                                      const mongo::BSONObj& oplogEntry,
-                                                      const std::string& errorMsg) {
-    const bool isReplicaSet =
-        repl::ReplicationCoordinator::get(opCtx->getServiceContext())->getReplicationMode() ==
-        repl::ReplicationCoordinator::modeReplSet;
-    // Relax the constraints of oplog application if the node is not a replica set member or the
-    // node is in the middle of a backup and restore process.
-    if (!isReplicaSet || storageGlobalParams.restore) {
-        return;
-    }
-
-    // During the recovery process, certain configuration collections such as
-    // 'config.image_collections' are handled differently, which may result in encountering oplog
-    // application failures in common scenarios, and therefore assert statements are not used.
-    if (nss.isConfigDB()) {
-        LOGV2_DEBUG(
-            5415002,
-            1,
-            "Error applying operation while recovering from stable checkpoint. This is related to "
-            "one of the configuration collections so this error might be benign.",
-            "oplogEntry"_attr = oplogEntry,
-            "error"_attr = errorMsg);
-    } else if (getTestCommandsEnabled()) {
-        // Only fassert in test environment.
-        LOGV2_FATAL(5415000,
-                    "Error applying operation while recovering from stable "
-                    "checkpoint. This can lead to data corruption.",
-                    "oplogEntry"_attr = oplogEntry,
-                    "error"_attr = errorMsg);
-    } else {
-        LOGV2_WARNING(5415001,
-                      "Error applying operation while recovering from stable "
-                      "checkpoint. This can lead to data corruption.",
-                      "oplogEntry"_attr = oplogEntry,
-                      "error"_attr = errorMsg);
-    }
-}
-
-// Logger for oplog constraint violations.
-OplogConstraintViolationLogger* oplogConstraintViolationLogger;
-
-MONGO_INITIALIZER(CreateOplogConstraintViolationLogger)(InitializerContext* context) {
-    oplogConstraintViolationLogger = new OplogConstraintViolationLogger();
-}
-
-void logOplogConstraintViolation(OperationContext* opCtx,
-                                 const NamespaceString& nss,
-                                 OplogConstraintViolationEnum type,
-                                 const std::string& operation,
-                                 const BSONObj& opObj,
-                                 boost::optional<Status> status) {
-    // Log the violation.
-    oplogConstraintViolationLogger->logViolationIfReady(type, opObj, status);
-
-    // Write a new entry to the health log.
-    HealthLogEntry entry;
-    entry.setNss(nss);
-    entry.setTimestamp(Date_t::now());
-    // Oplog constraint violations should always be marked as warning.
-    entry.setSeverity(SeverityEnum::Warning);
-    entry.setScope(ScopeEnum::Document);
-    entry.setMsg(toString(type));
-    entry.setOperation(operation);
-    entry.setData(opObj);
-
-    HealthLogInterface::get(opCtx->getServiceContext())->log(entry);
-}
-
 // @return failure status if an update should have happened and the document DNE.
 // See replset initial sync code.
 Status applyOperation_inlock(OperationContext* opCtx,
@@ -1286,22 +1179,11 @@ Status applyOperation_inlock(OperationContext* opCtx,
         return Status::OK();
     }
 
-    const bool inStableRecovery = mode == OplogApplication::Mode::kStableRecovering;
     NamespaceString requestNss;
     CollectionPtr collection = nullptr;
     if (auto uuid = op.getUuid()) {
         auto catalog = CollectionCatalog::get(opCtx);
         collection = catalog->lookupCollectionByUUID(opCtx, uuid.get());
-        if (!collection && inStableRecovery) {
-            repl::OplogApplication::checkOnOplogFailureForRecovery(
-                opCtx,
-                op.getNss(),
-                redact(op.toBSONForLogging()),
-                str::stream()
-                    << "(NamespaceNotFound): Failed to apply operation due to missing collection ("
-                    << uuid.value() << ")");
-        }
-
         uassert(ErrorCodes::NamespaceNotFound,
                 str::stream() << "Failed to apply operation due to missing collection ("
                               << uuid.get() << "): " << redact(opOrGroupedInserts.toBSON()),
@@ -1369,7 +1251,7 @@ Status applyOperation_inlock(OperationContext* opCtx,
                 case ReplicationCoordinator::modeNone: {
                     // Only assign timestamps on standalones during replication recovery when
                     // started with the 'recoverFromOplogAsStandalone' flag.
-                    return OplogApplication::inRecovering(mode);
+                    return mode == OplogApplication::Mode::kRecovering;
                 }
             }
         }
@@ -1388,7 +1270,8 @@ Status applyOperation_inlock(OperationContext* opCtx,
         // correct pre-image for them.
         return collection && collection->isChangeStreamPreAndPostImagesEnabled() &&
             isDataConsistent &&
-            (OplogApplication::inRecovering(mode) || mode == OplogApplication::Mode::kSecondary) &&
+            (mode == OplogApplication::Mode::kRecovering ||
+             mode == OplogApplication::Mode::kSecondary) &&
             !op.getFromMigrate().get_value_or(false) &&
             !requestNss.isTemporaryReshardingCollection();
     };
@@ -1528,23 +1411,10 @@ Status applyOperation_inlock(OperationContext* opCtx,
                             return status;
                         }
                         if (mode == OplogApplication::Mode::kSecondary) {
-                            const auto& opObj = redact(op.toBSONForLogging());
-
                             opCounters->gotInsertOnExistingDoc();
-                            logOplogConstraintViolation(
-                                opCtx,
-                                op.getNss(),
-                                OplogConstraintViolationEnum::kInsertOnExistingDoc,
-                                "insert",
-                                opObj,
-                                boost::none /* status */);
-
                             if (oplogApplicationEnforcesSteadyStateConstraints) {
                                 return status;
                             }
-                        } else if (inStableRecovery) {
-                            repl::OplogApplication::checkOnOplogFailureForRecovery(
-                                opCtx, op.getNss(), redact(op.toBSONForLogging()), redact(status));
                         }
                         // Continue to the next block to retry the operation as an upsert.
                         needToDoUpsert = true;
@@ -1619,13 +1489,13 @@ Status applyOperation_inlock(OperationContext* opCtx,
             request.setNamespaceString(requestNss);
             request.setQuery(updateCriteria);
             // If we are in steady state and the update is on a timeseries bucket collection, we can
-            // enable some optimizations in diff application. In some cases, like during tenant
-            // migration or $_internalApplyOplogUpdate update, we can for some reason generate
-            // entries for timeseries bucket collections which still rely on the idempotency
-            // guarantee, which then means we shouldn't apply these optimizations.
+            // enable some optimizations in diff application. In some cases, during tenant
+            // migration, we can for some reason generate entries for timeseries bucket collections
+            // which still rely on the idempotency guarantee, which then means we shouldn't apply
+            // these optimizations.
             write_ops::UpdateModification::DiffOptions options;
             if (mode == OplogApplication::Mode::kSecondary && collection->getTimeseriesOptions() &&
-                !op.getCheckExistenceForDiffInsert() && !op.getFromTenantMigration()) {
+                !op.getFromTenantMigration()) {
                 options.mustCheckExistenceForInsertOperations = false;
             }
             auto updateMod = write_ops::UpdateModification::parseFromOplogEntry(o, options);
@@ -1762,15 +1632,10 @@ Status applyOperation_inlock(OperationContext* opCtx,
                            !ur.upsertedId.isEmpty() && !(collection && collection->isCapped())) {
                     // This indicates we upconverted an update to an upsert, and it did indeed
                     // upsert.  In steady state mode this is unexpected.
-                    const auto& opObj = redact(op.toBSONForLogging());
-
+                    LOGV2_WARNING(2170001,
+                                  "update needed to be converted to upsert",
+                                  "op"_attr = redact(op.toBSONForLogging()));
                     opCounters->gotUpdateOnMissingDoc();
-                    logOplogConstraintViolation(opCtx,
-                                                op.getNss(),
-                                                OplogConstraintViolationEnum::kUpdateOnMissingDoc,
-                                                "update",
-                                                opObj,
-                                                boost::none /* status */);
 
                     // We shouldn't be doing upserts in secondary mode when enforcing steady state
                     // constraints.
@@ -1807,10 +1672,6 @@ Status applyOperation_inlock(OperationContext* opCtx,
             });
 
             if (!status.isOK()) {
-                if (inStableRecovery) {
-                    repl::OplogApplication::checkOnOplogFailureForRecovery(
-                        opCtx, op.getNss(), redact(op.toBSONForLogging()), redact(status));
-                }
                 return status;
             }
 
@@ -1892,22 +1753,16 @@ Status applyOperation_inlock(OperationContext* opCtx,
                     writeChangeStreamPreImage(opCtx, collection, op, *(result.requestedPreImage));
                 }
 
-                if (result.nDeleted == 0 && inStableRecovery) {
-                    repl::OplogApplication::checkOnOplogFailureForRecovery(
-                        opCtx,
-                        op.getNss(),
-                        redact(op.toBSONForLogging()),
-                        !collection ? str::stream()
-                                << "(NamespaceNotFound): Failed to apply operation due "
-                                   "to missing collection ("
-                                << requestNss << ")"
-                                    : "Applied a delete which did not delete anything."s);
-                }
                 // It is legal for a delete operation on the pre-images collection to delete zero
                 // documents - pre-image collections are not guaranteed to contain the same set of
                 // documents at all times.
                 if (result.nDeleted == 0 && mode == OplogApplication::Mode::kSecondary &&
                     !requestNss.isChangeStreamPreImagesCollection()) {
+                    LOGV2_WARNING(2170002,
+                                  "Applied a delete which did not delete anything in steady state "
+                                  "replication",
+                                  "op"_attr = redact(op.toBSONForLogging()));
+
                     // In FCV 4.4, each node is responsible for deleting the excess documents in
                     // capped collections. This implies that capped deletes may not be synchronized
                     // between nodes at times. When upgraded to FCV 5.0, the primary will generate
@@ -1920,25 +1775,11 @@ Status applyOperation_inlock(OperationContext* opCtx,
                     // capped collections when oplog application is enforcing steady state
                     // constraints.
                     bool isCapped = false;
-                    const auto& opObj = redact(op.toBSONForLogging());
                     if (collection) {
                         isCapped = collection->isCapped();
                         opCounters->gotDeleteWasEmpty();
-                        logOplogConstraintViolation(opCtx,
-                                                    op.getNss(),
-                                                    OplogConstraintViolationEnum::kDeleteWasEmpty,
-                                                    "delete",
-                                                    opObj,
-                                                    boost::none /* status */);
                     } else {
                         opCounters->gotDeleteFromMissingNamespace();
-                        logOplogConstraintViolation(
-                            opCtx,
-                            op.getNss(),
-                            OplogConstraintViolationEnum::kDeleteOnMissingNs,
-                            "delete",
-                            opObj,
-                            boost::none /* status */);
                     }
 
                     if (!isCapped) {
@@ -2057,7 +1898,7 @@ Status applyCommand_inlock(OperationContext* opCtx,
             case ReplicationCoordinator::modeNone: {
                 // Only assign timestamps on standalones during replication recovery when
                 // started with 'recoverFromOplogAsStandalone'.
-                return OplogApplication::inRecovering(mode);
+                return mode == OplogApplication::Mode::kRecovering;
             }
         }
         MONGO_UNREACHABLE;
@@ -2128,17 +1969,12 @@ Status applyCommand_inlock(OperationContext* opCtx,
                                 "aborting index build and retrying",
                                 logAttrs(ns));
                 } else {
-                    opCtx->recoveryUnit()->abandonSnapshot();
-
                     auto lockState = opCtx->lockState();
                     Locker::LockSnapshot lockSnapshot;
-                    bool canSaveState = lockState->canSaveLockState();
-                    if (canSaveState) {
-                        lockState->saveLockStateAndUnlock(&lockSnapshot);
-                    }
+                    auto locksReleased = lockState->saveLockStateAndUnlock(&lockSnapshot);
 
                     ScopeGuard guard{[&] {
-                        if (canSaveState) {
+                        if (locksReleased) {
                             invariant(!lockState->isLocked());
                             lockState->restoreLockState(lockSnapshot);
                         }
@@ -2154,6 +1990,7 @@ Status applyCommand_inlock(OperationContext* opCtx,
                     IndexBuildsCoordinator::get(opCtx)->awaitNoIndexBuildInProgressForCollection(
                         opCtx, swUUID.get());
 
+                    opCtx->recoveryUnit()->abandonSnapshot();
                     opCtx->checkForInterrupt();
 
                     LOGV2_DEBUG(
@@ -2191,15 +2028,12 @@ Status applyCommand_inlock(OperationContext* opCtx,
 
                 if (mode == OplogApplication::Mode::kSecondary &&
                     status.code() != ErrorCodes::IndexNotFound) {
-                    const auto& opObj = redact(entry.toBSONForLogging());
+                    LOGV2_WARNING(2170000,
+                                  "Acceptable error during oplog application",
+                                  "db"_attr = nss.db(),
+                                  "error"_attr = status,
+                                  "oplogEntry"_attr = redact(entry.toBSONForLogging()));
                     opCounters->gotAcceptableErrorInCommand();
-                    logOplogConstraintViolation(
-                        opCtx,
-                        entry.getNss(),
-                        OplogConstraintViolationEnum::kAcceptableErrorInCommand,
-                        "command",
-                        opObj,
-                        status);
                 } else {
                     LOGV2_DEBUG(51776,
                                 1,