New upstream version 6.0.20upstream

45 files changed, 542 insertions, 266 deletions

diff --git a/src/mongo/db/s/add_shard_cmd.cpp b/src/mongo/db/s/add_shard_cmd.cpp
index 3bfc1cc4a18..af64045ca6a 100644
--- a/src/mongo/db/s/add_shard_cmd.cpp
+++ b/src/mongo/db/s/add_shard_cmd.cpp
@@ -72,6 +72,14 @@ public:
             auto addShardCmd = request();
             auto shardIdUpsertCmd =
                 add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd);
+
+            // A request dispatched through a local client is served within the same thread that
+            // submits it (so that the opCtx needs to be used as the vehicle to pass the WC to the
+            // ServiceEntryPoint).
+            const auto originalWC = opCtx->getWriteConcern();
+            ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); });
+            opCtx->setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern);
+
             DBDirectClient localClient(opCtx);
             BSONObj res;
 
diff --git a/src/mongo/db/s/add_shard_util.cpp b/src/mongo/db/s/add_shard_util.cpp
index 957c2626e26..750dbf98d23 100644
--- a/src/mongo/db/s/add_shard_util.cpp
+++ b/src/mongo/db/s/add_shard_util.cpp
@@ -73,7 +73,6 @@ BSONObj createShardIdentityUpsertForAddShard(const AddShard& addShardCmd) {
 
         return updateOp;
     }());
-    request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON());
 
     return request.toBSON();
 }
diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp
index d194d900d29..caf1a8cbe3d 100644
--- a/src/mongo/db/s/balancer/balancer.cpp
+++ b/src/mongo/db/s/balancer/balancer.cpp
@@ -78,6 +78,7 @@ using std::vector;
 namespace {
 
 MONGO_FAIL_POINT_DEFINE(overrideBalanceRoundInterval);
+MONGO_FAIL_POINT_DEFINE(forceBalancerWarningChecks);
 
 const Milliseconds kBalanceRoundDefaultInterval(10 * 1000);
 
@@ -89,9 +90,6 @@ static constexpr StringData kBalancerPolicyStatusZoneViolation = "zoneViolation"
 static constexpr StringData kBalancerPolicyStatusChunksImbalance = "chunksImbalance"_sd;
 static constexpr StringData kBalancerPolicyStatusDefragmentingChunks = "defragmentingChunks"_sd;
 
-// Time interval between checks on draining shards.
-constexpr Minutes kDrainingShardsCheckInterval{10};
-
 /**
  * Utility class to generate timing and statistics for a single balancer round.
  */
@@ -237,6 +235,7 @@ std::vector<std::string> getDrainingShardNames(OperationContext* opCtx) {
 
     // Build the list of the draining shard names.
     std::vector<std::string> drainingShardNames;
+    drainingShardNames.reserve(drainingShardsDocs.size());
     std::transform(drainingShardsDocs.begin(),
                    drainingShardsDocs.end(),
                    std::back_inserter(drainingShardNames),
@@ -247,6 +246,105 @@ std::vector<std::string> getDrainingShardNames(OperationContext* opCtx) {
     return drainingShardNames;
 }
 
+class BalancerWarning {
+    // Time interval between checks on draining shards.
+    constexpr static Minutes kDrainingShardsCheckInterval{10};
+
+public:
+    BalancerWarning() = default;
+
+    void warnIfRequired(OperationContext* opCtx, BalancerSettingsType::BalancerMode balancerMode) {
+        if (Date_t::now() - _lastDrainingShardsCheckTime < kDrainingShardsCheckInterval &&
+            MONGO_likely(!forceBalancerWarningChecks.shouldFail())) {
+            return;
+        }
+        _lastDrainingShardsCheckTime = Date_t::now();
+
+        LOGV2(7977401, "Performing balancer warning checks");
+
+        const auto drainingShardNames{getDrainingShardNames(opCtx)};
+        if (drainingShardNames.empty()) {
+            return;
+        }
+
+        if (balancerMode == BalancerSettingsType::BalancerMode::kOff) {
+            LOGV2_WARNING(
+                6434000,
+                "Draining of removed shards cannot be completed because the balancer is disabled",
+                "shards"_attr = drainingShardNames);
+            return;
+        }
+
+        _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled(opCtx, drainingShardNames);
+    }
+
+private:
+    void _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled(
+        OperationContext* opCtx, const std::vector<std::string>& drainingShardNames) {
+        // Balancer is on, emit warning if balancer is disabled for collections which have chunks in
+        // shards in draining mode.
+        const auto catalogClient = Grid::get(opCtx)->catalogClient();
+        auto collections =
+            catalogClient->getCollections(opCtx,
+                                          {},
+                                          repl::ReadConcernLevel::kMajorityReadConcern,
+                                          BSON(CollectionType::kNssFieldName << 1));
+        if (collections.empty()) {
+            return;
+        }
+
+        // Construct BSONArray of draining shard names.
+        const auto drainingShardNameArray = [&]() {
+            BSONArrayBuilder shardNameArrayBuilder;
+            std::for_each(drainingShardNames.begin(),
+                          drainingShardNames.end(),
+                          [&shardNameArrayBuilder](const auto& shardName) {
+                              shardNameArrayBuilder.append(shardName);
+                          });
+            return shardNameArrayBuilder.arr();
+        }();
+
+        // For each collection, check if the collection has balancing disabled. If it is disabled,
+        // checks if the collection has any chunks in any of the draining shards. In which case a
+        // warning is emitted.
+        for (const auto& collType : collections) {
+            if (!collType.getAllowBalance() || !collType.getAllowMigrations() ||
+                !collType.getPermitMigrations()) {
+                const auto findQuery =
+                    BSON(ChunkType::collectionUUID() << collType.getUuid() << ChunkType::shard()
+                                                     << BSON("$in" << drainingShardNameArray));
+
+                auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+                auto findResponse = uassertStatusOK(configShard->exhaustiveFindOnConfig(
+                    opCtx,
+                    ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+                    repl::ReadConcernLevel::kMajorityReadConcern,
+                    ChunkType::ConfigNS,
+                    findQuery,
+                    BSONObj(),
+                    boost::none));
+
+                const auto& chunks = findResponse.docs;
+                if (!chunks.empty()) {
+                    stdx::unordered_set<std::string> shardsWithChunks;
+                    std::for_each(
+                        chunks.begin(), chunks.end(), [&shardsWithChunks](const BSONObj& chunkObj) {
+                            shardsWithChunks.emplace(chunkObj.getStringField(ChunkType::shard()));
+                        });
+                    LOGV2_WARNING(
+                        7977400,
+                        "Draining of removed shards cannot be completed because the balancer is "
+                        "disabled for a collection which has chunks in those shards",
+                        "uuid"_attr = collType.getUuid(),
+                        "nss"_attr = collType.getNss(),
+                        "shardsWithChunks"_attr = shardsWithChunks);
+                }
+            }
+        }
+    }
+
+    Date_t _lastDrainingShardsCheckTime{Date_t::fromMillisSinceEpoch(0)};
+};
 }  // namespace
 
 Balancer* Balancer::get(ServiceContext* serviceContext) {
@@ -271,24 +369,29 @@ Balancer::Balancer()
       _imbalancedCollectionsCache(std::make_unique<stdx::unordered_set<NamespaceString>>()) {}
 
 Balancer::~Balancer() {
-    // Terminate the balancer thread so it doesn't leak memory.
-    interruptBalancer();
-    waitForBalancerToStop();
+    onShutdown();
 }
 
 void Balancer::onStepUpBegin(OperationContext* opCtx, long long term) {
-    // Before starting step-up, ensure the balancer is ready to start. Specifically, that the
-    // balancer is actually stopped, because it may still be in the process of stopping if this
-    // node was previously primary.
-    waitForBalancerToStop();
+    // Before starting step-up, ensure the balancer is ready to start. Specifically, that there is
+    // not an outstanding termination sequence requested during a previous step down of this node.
+    joinTermination();
 }
 
 void Balancer::onStepUpComplete(OperationContext* opCtx, long long term) {
-    initiateBalancer(opCtx);
+    initiate(opCtx);
 }
 
 void Balancer::onStepDown() {
-    interruptBalancer();
+    // Asynchronously request to terminate all the worker threads and allow the stepdown sequence to
+    // continue.
+    requestTermination();
+}
+
+void Balancer::onShutdown() {
+    // Terminate the balancer thread so it doesn't leak memory.
+    requestTermination();
+    joinTermination();
 }
 
 void Balancer::onBecomeArbiter() {
@@ -297,11 +400,11 @@ void Balancer::onBecomeArbiter() {
     MONGO_UNREACHABLE;
 }
 
-void Balancer::initiateBalancer(OperationContext* opCtx) {
+void Balancer::initiate(OperationContext* opCtx) {
     stdx::lock_guard<Latch> scopedLock(_mutex);
     _imbalancedCollectionsCache->clear();
-    invariant(_state == kStopped);
-    _state = kRunning;
+    invariant(_threadSetState == ThreadSetState::Terminated);
+    _threadSetState = ThreadSetState::Running;
 
     invariant(!_thread.joinable());
     invariant(!_actionStreamConsumerThread.joinable());
@@ -309,13 +412,13 @@ void Balancer::initiateBalancer(OperationContext* opCtx) {
     _thread = stdx::thread([this] { _mainThread(); });
 }
 
-void Balancer::interruptBalancer() {
+void Balancer::requestTermination() {
     stdx::lock_guard<Latch> scopedLock(_mutex);
-    if (_state != kRunning) {
+    if (_threadSetState != ThreadSetState::Running) {
         return;
     }
 
-    _state = kStopping;
+    _threadSetState = ThreadSetState::Terminating;
 
     // Interrupt the balancer thread if it has been started. We are guaranteed that the operation
     // context of that thread is still alive, because we hold the balancer mutex.
@@ -328,9 +431,9 @@ void Balancer::interruptBalancer() {
     _defragmentationCondVar.notify_all();
 }
 
-void Balancer::waitForBalancerToStop() {
+void Balancer::joinTermination() {
     stdx::unique_lock<Latch> scopedLock(_mutex);
-    _joinCond.wait(scopedLock, [this] { return _state == kStopped; });
+    _joinCond.wait(scopedLock, [this] { return _threadSetState == ThreadSetState::Terminated; });
     if (_thread.joinable()) {
         _thread.join();
     }
@@ -529,10 +632,10 @@ void Balancer::_consumeActionStreamLoop() {
                 }
                 auto canConsumeStream = selectedStream != nullptr &&
                     _outstandingStreamingOps.load() <= kMaxOutstandingStreamingOperations;
-                return _state != kRunning ||
+                return _threadSetState != ThreadSetState::Running ||
                     (canConsumeStream && (!streamDrained || _newInfoOnStreamingActions.load()));
             });
-            if (_state != kRunning) {
+            if (_threadSetState != ThreadSetState::Running) {
                 break;
             }
         }
@@ -543,9 +646,23 @@ void Balancer::_consumeActionStreamLoop() {
                   "selectedStream"_attr = selectedStream->getName());
         }
 
-        _newInfoOnStreamingActions.store(false);
-        auto nextAction = selectedStream->getNextStreamingAction(opCtx.get());
-        if ((streamDrained = !nextAction.is_initialized())) {
+        boost::optional<DefragmentationAction> nextAction;
+        try {
+            _newInfoOnStreamingActions.store(false);
+            nextAction = selectedStream->getNextStreamingAction(opCtx.get());
+        } catch (const DBException& e) {
+            LOGV2_WARNING(7435001,
+                          "Failed to get next action from action stream",
+                          "error"_attr = redact(e),
+                          "stream"_attr = selectedStream->getName());
+
+            _newInfoOnStreamingActions.store(true);
+            continue;
+        }
+
+        if (!nextAction.is_initialized()) {
+            // No action was returned by this stream. This means that the stream is drained.
+            streamDrained = true;
             continue;
         }
 
@@ -636,8 +753,8 @@ void Balancer::_mainThread() {
     ON_BLOCK_EXIT([this] {
         {
             stdx::lock_guard<Latch> scopedLock(_mutex);
-            _state = kStopped;
-            LOGV2_DEBUG(21855, 1, "Balancer thread terminated");
+            _threadSetState = ThreadSetState::Terminated;
+            LOGV2_DEBUG(21855, 1, "Balancer thread set terminated");
         }
         _joinCond.notify_all();
     });
@@ -656,7 +773,7 @@ void Balancer::_mainThread() {
     const Seconds kInitBackoffInterval(10);
 
     auto balancerConfig = shardingContext->getBalancerConfiguration();
-    while (!_stopRequested()) {
+    while (!_terminationRequested()) {
         Status refreshStatus = balancerConfig->refreshAndCheck(opCtx.get());
         if (!refreshStatus.isOK()) {
             LOGV2_WARNING(
@@ -686,9 +803,9 @@ void Balancer::_mainThread() {
     LOGV2(6036606, "Balancer worker thread initialised. Entering main loop.");
 
     // Main balancer loop
-    auto lastDrainingShardsCheckTime{Date_t::fromMillisSinceEpoch(0)};
     auto lastMigrationTime = Date_t::fromMillisSinceEpoch(0);
-    while (!_stopRequested()) {
+    BalancerWarning balancerWarning;
+    while (!_terminationRequested()) {
         BalanceRoundDetails roundDetails;
 
         _beginRound(opCtx.get());
@@ -708,22 +825,11 @@ void Balancer::_mainThread() {
                 continue;
             }
 
-            if (!balancerConfig->shouldBalance() || _stopRequested() ||
-                _clusterChunksResizePolicy->isActive()) {
-
-                if (balancerConfig->getBalancerMode() == BalancerSettingsType::BalancerMode::kOff &&
-                    Date_t::now() - lastDrainingShardsCheckTime >= kDrainingShardsCheckInterval) {
-                    const auto drainingShardNames{getDrainingShardNames(opCtx.get())};
-                    if (!drainingShardNames.empty()) {
-                        LOGV2_WARNING(6434000,
-                                      "Draining of removed shards cannot be completed because the "
-                                      "balancer is disabled",
-                                      "shards"_attr = drainingShardNames);
-                    }
-
-                    lastDrainingShardsCheckTime = Date_t::now();
-                }
+            // Warn before we skip the iteration due to balancing being disabled.
+            balancerWarning.warnIfRequired(opCtx.get(), balancerConfig->getBalancerMode());
 
+            if (!balancerConfig->shouldBalance() || _terminationRequested() ||
+                _clusterChunksResizePolicy->isActive()) {
                 LOGV2_DEBUG(21859, 1, "Skipping balancing round because balancing is disabled");
                 _endRound(opCtx.get(), kBalanceRoundDefaultInterval);
                 continue;
@@ -863,7 +969,7 @@ void Balancer::_mainThread() {
 
     {
         stdx::lock_guard<Latch> scopedLock(_mutex);
-        invariant(_state == kStopping);
+        invariant(_threadSetState == ThreadSetState::Terminating);
     }
 
     _commandScheduler->stop();
@@ -894,9 +1000,9 @@ void Balancer::_applyDefragmentationActionResponseToPolicy(
     policy->applyActionResult(opCtx.get(), action, response);
 };
 
-bool Balancer::_stopRequested() {
+bool Balancer::_terminationRequested() {
     stdx::lock_guard<Latch> scopedLock(_mutex);
-    return (_state != kRunning);
+    return (_threadSetState != ThreadSetState::Running);
 }
 
 void Balancer::_beginRound(OperationContext* opCtx) {
@@ -919,7 +1025,9 @@ void Balancer::_endRound(OperationContext* opCtx, Milliseconds waitTimeout) {
 
 void Balancer::_sleepFor(OperationContext* opCtx, Milliseconds waitTimeout) {
     stdx::unique_lock<Latch> lock(_mutex);
-    _condVar.wait_for(lock, waitTimeout.toSystemDuration(), [&] { return _state != kRunning; });
+    _condVar.wait_for(lock, waitTimeout.toSystemDuration(), [&] {
+        return _threadSetState != ThreadSetState::Running;
+    });
 }
 
 bool Balancer::_checkOIDs(OperationContext* opCtx) {
@@ -931,7 +1039,7 @@ bool Balancer::_checkOIDs(OperationContext* opCtx) {
     map<int, ShardId> oids;
 
     for (const ShardId& shardId : all) {
-        if (_stopRequested()) {
+        if (_terminationRequested()) {
             return false;
         }
 
@@ -1044,7 +1152,7 @@ int Balancer::_moveChunks(OperationContext* opCtx,
     auto catalogClient = Grid::get(opCtx)->catalogClient();
 
     // If the balancer was disabled since we started this round, don't start new chunk moves
-    if (_stopRequested() || !balancerConfig->shouldBalance() ||
+    if (_terminationRequested() || !balancerConfig->shouldBalance() ||
         _clusterChunksResizePolicy->isActive()) {
         LOGV2_DEBUG(21870, 1, "Skipping balancing round because balancer was stopped");
         return 0;
@@ -1196,10 +1304,11 @@ void Balancer::_disableBalancer(OperationContext* opCtx, NamespaceString nss) {
         return updateOp;
     }());
 
-    updateRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON());
-
-    auto response = configShard->runBatchWriteCommand(
-        opCtx, Shard::kDefaultConfigCommandTimeout, updateRequest, Shard::RetryPolicy::kIdempotent);
+    auto response = configShard->runBatchWriteCommand(opCtx,
+                                                      Shard::kDefaultConfigCommandTimeout,
+                                                      updateRequest,
+                                                      ShardingCatalogClient::kMajorityWriteConcern,
+                                                      Shard::RetryPolicy::kIdempotent);
     uassertStatusOK(response.toStatus());
 }
 
diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h
index b84df5558bb..b9cdd2a6b6c 100644
--- a/src/mongo/db/s/balancer/balancer.h
+++ b/src/mongo/db/s/balancer/balancer.h
@@ -75,38 +75,40 @@ public:
 
     /**
      * Invoked when the config server primary enters the 'PRIMARY' state and is invoked while the
-     * caller is holding the global X lock. Kicks off the main balancer thread and returns
-     * immediately. Auto-balancing (if enabled) should commence shortly, and manual migrations will
-     * be processed and run.
+     * caller is holding the global X lock. Kicks off the main balancer thread (which will in turn
+     * instantiate a secondary worker and the CommandsScheduler) and returns immediately.
+     * Auto-balancing (if enabled) should commence shortly, and manual migrations will be processed
+     * and run.
      *
-     * Must only be called if the balancer is in the stopped state (i.e., just constructed or
-     * waitForBalancerToStop has been called before). Any code in this call must not try to acquire
-     * any locks or to wait on operations, which acquire locks.
+     * Must only be called if the balancer thread set is in the Terminated state (i.e., just
+     * constructed or joinTermination() has been called before).
+     * Any code in this call must not try to acquire any locks or to wait on operations, which
+     * acquire locks.
      */
-    void initiateBalancer(OperationContext* opCtx);
+    void initiate(OperationContext* opCtx);
 
     /**
      * Invoked when this node which is currently serving as a 'PRIMARY' steps down and is invoked
-     * while the global X lock is held. Requests the main balancer thread to stop and returns
-     * immediately without waiting for it to terminate. Once the balancer has stopped, manual
-     * migrations will be rejected.
+     * while the global X lock is held. Requests to the hierarchy of balancer threads to leave and
+     * returns immediately without waiting for them to terminate. (Once the termination is complete,
+     * manual migrations will be rejected).
      *
      * This method might be called multiple times in succession, which is what happens as a result
      * of incomplete transition to primary so it is resilient to that.
      *
-     * The waitForBalancerToStop method must be called afterwards in order to wait for the main
+     * The joinTermination() method must be called afterwards in order to wait for the main
      * balancer thread to terminate and to allow initiateBalancer to be called again.
      */
-    void interruptBalancer();
+    void requestTermination();
 
     /**
      * Invoked when a node on its way to becoming a primary finishes draining and is about to
-     * acquire the global X lock in order to allow writes. Waits for the balancer thread to
-     * terminate and primes the balancer so that initiateBalancer can be called.
+     * acquire the global X lock in order to allow writes. Waits for the hierarchy of balancer
+     * threads to terminate and primes the balancer so that initiateBalancer can be called.
      *
      * This must not be called while holding any locks!
      */
-    void waitForBalancerToStop();
+    void joinTermination();
 
     /**
      * Potentially blocking method, which will return immediately if the balancer is not running a
@@ -187,12 +189,20 @@ private:
     static constexpr int kMaxOutstandingStreamingOperations = 50;
 
     /**
-     * Possible runtime states of the balancer. The comments indicate the allowed next state.
+     * Possible runtime states of the set of threads instantiated by the balancer.
+     * The diagram below depicts the allowed transitions.
+     * Terminated --> Running --> Terminating
+     *    ^            /          /
+     *    |           /          /
+     *     \---------------------
      */
-    enum State {
-        kStopped,   // kRunning
-        kRunning,   // kStopping | kStopped
-        kStopping,  // kStopped
+    enum class ThreadSetState {
+        // There is no worker thread instantiated by the balancer
+        Terminated,
+        // The balancer is initiliasing its worker threads (or they are all already active)
+        Running,
+        // A request to terminate all the balancer worker threads is ongoing
+        Terminating,
     };
 
     /**
@@ -200,7 +210,7 @@ private:
      */
     void onStartup(OperationContext* opCtx) final {}
     void onInitialDataAvailable(OperationContext* opCtx, bool isMajorityDataAvailable) final {}
-    void onShutdown() final {}
+    void onShutdown() final;
     void onStepUpBegin(OperationContext* opCtx, long long term) final;
     void onStepUpComplete(OperationContext* opCtx, long long term) final;
     void onStepDown() final;
@@ -217,9 +227,9 @@ private:
     void _consumeActionStreamLoop();
 
     /**
-     * Checks whether the balancer main thread has been requested to stop.
+     * Checks whether the balancer is going through a termination sequence of its threads.
      */
-    bool _stopRequested();
+    bool _terminationRequested();
 
     /**
      * Signals the beginning and end of a balancing round.
@@ -277,8 +287,9 @@ private:
     // Protects the state below
     Mutex _mutex = MONGO_MAKE_LATCH("Balancer::_mutex");
 
-    // Indicates the current state of the balancer
-    State _state{kStopped};
+    // Indicates the current state of the worker threads instantiated by the balancer
+    // (_thread, _actionStreamConsumerThread and _commandScheduler)
+    ThreadSetState _threadSetState{ThreadSetState::Terminated};
 
     // The main balancer threads
     stdx::thread _thread;
diff --git a/src/mongo/db/s/balancer_stats_registry.cpp b/src/mongo/db/s/balancer_stats_registry.cpp
index 508648eb29f..0d262b2c299 100644
--- a/src/mongo/db/s/balancer_stats_registry.cpp
+++ b/src/mongo/db/s/balancer_stats_registry.cpp
@@ -284,9 +284,12 @@ void BalancerStatsRegistry::updateOrphansCount(const UUID& collectionUUID, long
         stats.numOrphanDocs += delta;
 
         if (stats.numOrphanDocs < 0) {
-            // This should happen only in case of direct manipulation of range deletion tasks
-            // documents or direct writes into orphaned ranges
-            LOGV2_ERROR(6419611,
+            // This could happen in case of direct manipulation of range deletion tasks documents or
+            // direct writes into orphaned ranges, but also in some other benign situations.
+            // numOrphanDocs is a best-effort counter, miscounting or even being negative in some
+            // scenarios is expected.
+            LOGV2_DEBUG(6419611,
+                        1,
                         "Cached orphan documents count became negative, resetting it to 0",
                         "collectionUUID"_attr = collectionUUID,
                         "numOrphanDocs"_attr = stats.numOrphanDocs,
@@ -334,7 +337,8 @@ void BalancerStatsRegistry::_loadOrphansCount(OperationContext* opCtx) {
             auto numRangeDeletionTasks = collObj[kNumRangeDeletionTasksLabel].exactNumberLong();
             invariant(numRangeDeletionTasks > 0);
             if (orphanCount < 0) {
-                LOGV2_ERROR(6419621,
+                LOGV2_DEBUG(6419621,
+                            1,
                             "Found negative orphan count in range deletion task documents",
                             "collectionUUID"_attr = collUUID,
                             "numOrphanDocs"_attr = orphanCount,
diff --git a/src/mongo/db/s/check_sharding_index_command.cpp b/src/mongo/db/s/check_sharding_index_command.cpp
index 7849ab169d3..dd5331e33f1 100644
--- a/src/mongo/db/s/check_sharding_index_command.cpp
+++ b/src/mongo/db/s/check_sharding_index_command.cpp
@@ -101,10 +101,8 @@ public:
                                                      keyPattern,
                                                      /*requireSingleKey=*/true,
                                                      &tmpErrMsg);
-        if (!shardKeyIdx) {
-            errmsg = tmpErrMsg;
-            return false;
-        }
+
+        uassert(ErrorCodes::InvalidOptions, str::stream() << tmpErrMsg, shardKeyIdx);
 
         return true;
     }
diff --git a/src/mongo/db/s/config/initial_split_policy.cpp b/src/mongo/db/s/config/initial_split_policy.cpp
index 15c5a345c59..0d623d81fba 100644
--- a/src/mongo/db/s/config/initial_split_policy.cpp
+++ b/src/mongo/db/s/config/initial_split_policy.cpp
@@ -179,7 +179,7 @@ StringMap<std::vector<ShardId>> buildTagsToShardIdsMap(OperationContext* opCtx,
 }  // namespace
 
 std::vector<BSONObj> InitialSplitPolicy::calculateHashedSplitPoints(
-    const ShardKeyPattern& shardKeyPattern, BSONObj prefix, int numInitialChunks) {
+    const ShardKeyPattern& shardKeyPattern, BSONObj prefix, size_t numInitialChunks) {
     invariant(shardKeyPattern.isHashedPattern());
     invariant(numInitialChunks > 0);
 
@@ -224,7 +224,7 @@ std::vector<BSONObj> InitialSplitPolicy::calculateHashedSplitPoints(
         current += intervalSize / 2;
     }
 
-    for (int i = 0; i < (numInitialChunks - 1) / 2; i++) {
+    for (size_t i = 0; i < (numInitialChunks - 1) / 2; i++) {
         splitPoints.push_back(buildSplitPoint(current));
         splitPoints.push_back(buildSplitPoint(-current));
         current += intervalSize;
diff --git a/src/mongo/db/s/config/initial_split_policy.h b/src/mongo/db/s/config/initial_split_policy.h
index e492e9c4cb2..7103b89d91b 100644
--- a/src/mongo/db/s/config/initial_split_policy.h
+++ b/src/mongo/db/s/config/initial_split_policy.h
@@ -104,7 +104,7 @@ public:
      */
     static std::vector<BSONObj> calculateHashedSplitPoints(const ShardKeyPattern& shardKeyPattern,
                                                            BSONObj prefix,
-                                                           int numInitialChunks);
+                                                           size_t numInitialChunks);
 
     /**
      * Produces the initial chunks that need to be written for an *empty* collection which is being
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.cpp b/src/mongo/db/s/config/sharding_catalog_manager.cpp
index bb4980b5eb0..d380fe58468 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager.cpp
@@ -44,6 +44,7 @@
 #include "mongo/db/internal_transactions_feature_flag_gen.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/ops/write_ops.h"
+#include "mongo/db/query/cursor_response.h"
 #include "mongo/db/query/query_request_helper.h"
 #include "mongo/db/repl/repl_client_info.h"
 #include "mongo/db/s/balancer/type_migration.h"
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp
index db1d21033bd..aa483a030a5 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp
@@ -460,11 +460,7 @@ TEST_F(AddShardTest, CreateShardIdentityUpsertForAddShard) {
                                 << shardName << "clusterId" << _clusterId
                                 << "configsvrConnectionString"
                                 << replicationCoordinator()->getConfigConnectionString().toString())
-                        << "multi" << false << "upsert" << true))
-             << "writeConcern"
-             << BSON("w"
-                     << "majority"
-                     << "wtimeout" << 60000));
+                        << "multi" << false << "upsert" << true)));
     auto addShardCmd = add_shard_util::createAddShardCmd(operationContext(), shardName);
     auto actualBSON = add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd);
     ASSERT_BSONOBJ_EQ(expectedBSON, actualBSON);
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
index 0c3a23b1582..d1bc8fcd343 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/logical_session_cache.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
+#include "mongo/db/query/cursor_response.h"
 #include "mongo/db/query/distinct_command_gen.h"
 #include "mongo/db/repl/repl_client_info.h"
 #include "mongo/db/s/sharding_logging.h"
@@ -645,6 +646,11 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit(
     newChunkBounds.push_back(range.getMax());
 
     if (isSplitAlreadyDone(opCtx, range, shardName, origChunk.getValue(), newChunkBounds)) {
+        // In case the request was already fullfilled, we still need to wait until the original
+        // request is majority written. The timestamp is not known, so we use the system's last
+        // optime. Otherwise the next RoutingInfo cache refresh from the shard may not see the
+        // newest information.
+        repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx);
         return buildChunkVersionBSON(collVersion);
     }
 
@@ -1404,10 +1410,12 @@ void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx,
             }()});
             return updateOp;
         }());
-        request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON());
 
-        auto response = configShard->runBatchWriteCommand(
-            opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent);
+        auto response = configShard->runBatchWriteCommand(opCtx,
+                                                          Shard::kDefaultConfigCommandTimeout,
+                                                          request,
+                                                          ShardingCatalogClient::kLocalWriteConcern,
+                                                          Shard::RetryPolicy::kIdempotent);
         uassertStatusOK(response.toStatus());
 
         uassert(ErrorCodes::Error(5760502),
@@ -2073,11 +2081,13 @@ bool ShardingCatalogManager::clearChunkEstimatedSize(OperationContext* opCtx, co
         }()});
         return updateOp;
     }());
-    request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON());
 
     auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-    auto response = configShard->runBatchWriteCommand(
-        opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent);
+    auto response = configShard->runBatchWriteCommand(opCtx,
+                                                      Shard::kDefaultConfigCommandTimeout,
+                                                      request,
+                                                      ShardingCatalogClient::kMajorityWriteConcern,
+                                                      Shard::RetryPolicy::kIdempotent);
 
     uassertStatusOK(response.toStatus());
     return response.getN() > 0;
diff --git a/src/mongo/db/s/create_collection_coordinator.cpp b/src/mongo/db/s/create_collection_coordinator.cpp
index e1f42dfdcfe..9daff7b07a5 100644
--- a/src/mongo/db/s/create_collection_coordinator.cpp
+++ b/src/mongo/db/s/create_collection_coordinator.cpp
@@ -334,16 +334,14 @@ void insertCollectionEntry(OperationContext* opCtx,
 
     BatchedCommandRequest insertRequest(
         write_ops::InsertCommandRequest(CollectionType::ConfigNS, {coll.toBSON()}));
-    insertRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON());
-
-    const BSONObj cmdObj = insertRequest.toBSON().addFields(osi.toBSON());
+    const auto cmdObj = CommandHelpers::appendMajorityWriteConcern(insertRequest.toBSON());
 
     BatchedCommandResponse unusedResponse;
     uassertStatusOK(Shard::CommandResponse::processBatchWriteResponse(
         configShard->runCommand(opCtx,
                                 ReadPreferenceSetting{ReadPreference::PrimaryOnly},
                                 CollectionType::ConfigNS.db().toString(),
-                                cmdObj,
+                                cmdObj.addFields(osi.toBSON()),
                                 Shard::kDefaultConfigCommandTimeout,
                                 Shard::RetryPolicy::kIdempotent),
         &unusedResponse));
@@ -705,7 +703,7 @@ void CreateCollectionCoordinator::_checkCommandArguments(OperationContext* opCtx
         const int maxNumInitialChunksForShards =
             Grid::get(opCtx)->shardRegistry()->getNumShardsNoReload() * shardutil::kMaxSplitPoints;
         const int maxNumInitialChunksTotal = 1000 * 1000;  // Arbitrary limit to memory consumption
-        int numChunks = _request.getNumInitialChunks().value();
+        const auto numChunks = _request.getNumInitialChunks().value();
         uassert(ErrorCodes::InvalidOptions,
                 str::stream() << "numInitialChunks cannot be more than either: "
                               << maxNumInitialChunksForShards << ", " << shardutil::kMaxSplitPoints
diff --git a/src/mongo/db/s/dist_lock_catalog_replset.cpp b/src/mongo/db/s/dist_lock_catalog_replset.cpp
index 5a0dd02d83d..15e5c76e492 100644
--- a/src/mongo/db/s/dist_lock_catalog_replset.cpp
+++ b/src/mongo/db/s/dist_lock_catalog_replset.cpp
@@ -377,38 +377,15 @@ Status DistLockCatalogImpl::unlockAll(OperationContext* opCtx,
         }()});
         return updateOp;
     }());
-    request.setWriteConcern(kLocalWriteConcern.toBSON());
-
-    BSONObj cmdObj = request.toBSON();
 
     auto const shardRegistry = Grid::get(opCtx)->shardRegistry();
-    auto response = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts(
-        opCtx,
-        ReadPreferenceSetting{ReadPreference::PrimaryOnly},
-        _locksNS.db().toString(),
-        cmdObj,
-        Shard::kDefaultConfigCommandTimeout,
-        Shard::RetryPolicy::kIdempotent);
+    auto batchResponse =
+        shardRegistry->getConfigShard()->runBatchWriteCommand(opCtx,
+                                                              Shard::kDefaultConfigCommandTimeout,
+                                                              request,
+                                                              kLocalWriteConcern,
+                                                              Shard::RetryPolicy::kIdempotent);
 
-    if (!response.isOK()) {
-        return response.getStatus();
-    }
-    if (!response.getValue().commandStatus.isOK()) {
-        return response.getValue().commandStatus;
-    }
-    if (!response.getValue().writeConcernStatus.isOK()) {
-        return response.getValue().writeConcernStatus;
-    }
-
-    BatchedCommandResponse batchResponse;
-    std::string errmsg;
-    if (!batchResponse.parseBSON(response.getValue().response, &errmsg)) {
-        return Status(ErrorCodes::FailedToParse,
-                      str::stream()
-                          << "Failed to parse config server response to batch request for "
-                             "unlocking existing distributed locks"
-                          << causedBy(errmsg));
-    }
     return batchResponse.toStatus();
 }
 
diff --git a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp
index a05f6db250d..63b9d91700c 100644
--- a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp
+++ b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp
@@ -1234,8 +1234,6 @@ TEST_F(DistLockCatalogReplSetTest, BasicUnlockAll) {
         const auto opMsgRequest(OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj));
         const auto commandRequest(BatchedCommandRequest::parseUpdate(opMsgRequest));
 
-        ASSERT_BSONOBJ_EQ(BSON("w" << 1 << "wtimeout" << 0), commandRequest.getWriteConcern());
-
         const auto& updateOp = commandRequest.getUpdateRequest();
         ASSERT_EQUALS(LocksType::ConfigNS, updateOp.getNamespace());
 
diff --git a/src/mongo/db/s/flush_resharding_state_change_command.cpp b/src/mongo/db/s/flush_resharding_state_change_command.cpp
index cc4a4093dd1..53b3f9953ff 100644
--- a/src/mongo/db/s/flush_resharding_state_change_command.cpp
+++ b/src/mongo/db/s/flush_resharding_state_change_command.cpp
@@ -109,7 +109,11 @@ public:
                     "Can't call _flushReshardingStateChange if in read-only mode",
                     !storageGlobalParams.readOnly);
 
-            ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getArbitraryExecutor())
+            // We use the fixed executor here since it may cause the thread to block. This would
+            // cause potential liveness issues since the arbitrary executor is a NetworkInterfaceTL
+            // executor in sharded clusters and that executor is one that executes networking
+            // operations.
+            ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getFixedExecutor())
                 .then([svcCtx = opCtx->getServiceContext(), nss = ns()] {
                     ThreadClient tc("FlushReshardingStateChange", svcCtx);
                     {
diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
index 807d7d0da0a..776e620f072 100644
--- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
+++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp
@@ -214,7 +214,8 @@ public:
         MONGO_UNREACHABLE;
     }
 
-    bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const override {
+    StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData(
+        const BSONObj& bucketsDoc) const override {
         return _coll->doesTimeseriesBucketsDocContainMixedSchemaData(bucketsDoc);
     }
 
diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp
index 840c7e7ee5e..bbe4f34bf19 100644
--- a/src/mongo/db/s/migration_destination_manager.cpp
+++ b/src/mongo/db/s/migration_destination_manager.cpp
@@ -1488,7 +1488,7 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx,
             runWithoutSession(outerOpCtx, [&] {
                 auto awaitReplicationResult =
                     repl::ReplicationCoordinator::get(opCtx)->awaitReplication(
-                        opCtx, lastOpApplied, _writeConcern);
+                        opCtx, lastOpApplied, WriteConcerns::kMajorityWriteConcernShardingTimeout);
                 uassertStatusOKWithContext(awaitReplicationResult.status,
                                            awaitReplicationResult.status.codeString());
             });
diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp
index c4f301d52c7..9ca15fe9fad 100644
--- a/src/mongo/db/s/migration_source_manager.cpp
+++ b/src/mongo/db/s/migration_source_manager.cpp
@@ -426,6 +426,12 @@ void MigrationSourceManager::startClone() {
         _state = kCloning;
     }
 
+    // Refreshing the collection routing information after starting the clone driver will give us a
+    // stable view on whether the recipient is owning other chunks of the collection (a condition
+    // that will be later evaluated).
+    uassertStatusOK(
+        Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfoWithRefresh(_opCtx, nss()));
+
     if (replEnabled) {
         auto const readConcernArgs = repl::ReadConcernArgs(
             replCoord->getMyLastAppliedOpTime(), repl::ReadConcernLevel::kLocalReadConcern);
@@ -471,11 +477,12 @@ void MigrationSourceManager::enterCriticalSection() {
     _stats.totalDonorChunkCloneTimeMillis.addAndFetch(_cloneAndCommitTimer.millis());
     _cloneAndCommitTimer.reset();
 
-    const auto& metadata = _getCurrentMetadataAndCheckEpoch();
+    const auto cm =
+        uassertStatusOK(Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfo(_opCtx, nss()));
 
     // Check that there are no chunks on the recepient shard. Write an oplog event for change
     // streams if this is the first migration to the recipient.
-    if (!metadata.getChunkManager()->getVersion(_args.getToShard()).isSet()) {
+    if (!cm.getVersion(_args.getToShard()).isSet()) {
         migrationutil::notifyChangeStreamsOnRecipientFirstChunk(
             _opCtx, nss(), _args.getFromShard(), _args.getToShard(), _collectionUUID);
     }
diff --git a/src/mongo/db/s/range_deletion_util.cpp b/src/mongo/db/s/range_deletion_util.cpp
index 9ca0a8f2518..54944dcb0bd 100644
--- a/src/mongo/db/s/range_deletion_util.cpp
+++ b/src/mongo/db/s/range_deletion_util.cpp
@@ -52,6 +52,8 @@
 #include "mongo/db/repl/repl_client_info.h"
 #include "mongo/db/repl/wait_for_majority_service.h"
 #include "mongo/db/s/migration_util.h"
+#include "mongo/db/s/operation_sharding_state.h"
+#include "mongo/db/s/shard_filtering_metadata_refresh.h"
 #include "mongo/db/s/shard_key_index_util.h"
 #include "mongo/db/s/sharding_runtime_d_params_gen.h"
 #include "mongo/db/s/sharding_statistics.h"
@@ -668,45 +670,59 @@ void setOrphanCountersOnRangeDeletionTasks(OperationContext* opCtx) {
         opCtx,
         BSONObj(),
         [opCtx, &store, &setNumOrphansOnTask](const RangeDeletionTask& deletionTask) {
-            AutoGetCollection collection(opCtx, deletionTask.getNss(), MODE_IX);
-            if (!collection || collection->uuid() != deletionTask.getCollectionUuid()) {
-                // The deletion task is referring to a collection that has been dropped
-                setNumOrphansOnTask(deletionTask, 0);
-                return true;
-            }
+            // The operation context is not bound to any specific namespace; acquire the shard role
+            // to ensure that the collection key pattern may be retrieved through the
+            // AutoGetCollection object.
+            ScopedSetShardRole scopedRole(
+                opCtx, deletionTask.getNss(), ChunkVersion::IGNORED(), boost::none);
+            while (true) {
+                try {
+                    AutoGetCollection collection(opCtx, deletionTask.getNss(), MODE_IX);
+                    if (!collection || collection->uuid() != deletionTask.getCollectionUuid()) {
+                        // The deletion task is referring to a collection that has been dropped
+                        setNumOrphansOnTask(deletionTask, 0);
+                        return true;
+                    }
 
-            KeyPattern keyPattern;
-            uassertStatusOK(deletionTask.getRange().extractKeyPattern(&keyPattern));
-            auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx,
-                                                         *collection,
-                                                         collection->getIndexCatalog(),
-                                                         keyPattern.toBSON(),
-                                                         /*requireSingleKey=*/false);
-
-            uassert(ErrorCodes::IndexNotFound,
-                    str::stream() << "couldn't find index over shard key " << keyPattern.toBSON()
-                                  << " for collection " << deletionTask.getNss()
-                                  << " (uuid: " << deletionTask.getCollectionUuid() << ")",
-                    shardKeyIdx);
-
-            const auto& range = deletionTask.getRange();
-            auto forwardIdxScanner =
-                InternalPlanner::shardKeyIndexScan(opCtx,
-                                                   &(*collection),
-                                                   *shardKeyIdx,
-                                                   range.getMin(),
-                                                   range.getMax(),
-                                                   BoundInclusion::kIncludeStartKeyOnly,
-                                                   PlanYieldPolicy::YieldPolicy::YIELD_AUTO,
-                                                   InternalPlanner::FORWARD);
-            int64_t numOrphansInRange = 0;
-            BSONObj indexEntry;
-            while (forwardIdxScanner->getNext(&indexEntry, nullptr) != PlanExecutor::IS_EOF) {
-                ++numOrphansInRange;
-            }
 
-            setNumOrphansOnTask(deletionTask, numOrphansInRange);
-            return true;
+                    const auto keyPattern = collection.getCollection().getShardKeyPattern();
+                    auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx,
+                                                                 *collection,
+                                                                 collection->getIndexCatalog(),
+                                                                 keyPattern,
+                                                                 /*requireSingleKey=*/false);
+
+                    uassert(ErrorCodes::IndexNotFound,
+                            str::stream() << "couldn't find index over shard key " << keyPattern
+                                          << " for collection " << deletionTask.getNss()
+                                          << " (uuid: " << deletionTask.getCollectionUuid() << ")",
+                            shardKeyIdx);
+
+                    const auto& range = deletionTask.getRange();
+                    auto forwardIdxScanner =
+                        InternalPlanner::shardKeyIndexScan(opCtx,
+                                                           &(*collection),
+                                                           *shardKeyIdx,
+                                                           range.getMin(),
+                                                           range.getMax(),
+                                                           BoundInclusion::kIncludeStartKeyOnly,
+                                                           PlanYieldPolicy::YieldPolicy::YIELD_AUTO,
+                                                           InternalPlanner::FORWARD);
+                    int64_t numOrphansInRange = 0;
+                    BSONObj indexEntry;
+                    while (forwardIdxScanner->getNext(&indexEntry, nullptr) !=
+                           PlanExecutor::IS_EOF) {
+                        ++numOrphansInRange;
+                    }
+
+                    setNumOrphansOnTask(deletionTask, numOrphansInRange);
+                    return true;
+
+                } catch (const ExceptionFor<ErrorCodes::StaleConfig>& e) {
+                    onShardVersionMismatchNoExcept(opCtx, e->getNss(), e->getVersionReceived())
+                        .ignore();
+                }
+            }
         });
 }
 
diff --git a/src/mongo/db/s/range_deletion_util_test.cpp b/src/mongo/db/s/range_deletion_util_test.cpp
index 1a8154dffd8..c45eeb53e82 100644
--- a/src/mongo/db/s/range_deletion_util_test.cpp
+++ b/src/mongo/db/s/range_deletion_util_test.cpp
@@ -32,6 +32,7 @@
 #include "mongo/db/catalog/create_collection.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/dbdirectclient.h"
+#include "mongo/db/hasher.h"
 #include "mongo/db/persistent_task_store.h"
 #include "mongo/db/repl/wait_for_majority_service.h"
 #include "mongo/db/s/collection_sharding_runtime.h"
@@ -51,7 +52,7 @@ namespace {
 
 const NamespaceString kNss = NamespaceString("foo", "bar");
 const std::string kShardKey = "_id";
-const BSONObj kShardKeyPattern = BSON(kShardKey << 1);
+const BSONObj kRangeBasedShardKeyPattern = BSON(kShardKey << 1);
 
 class RangeDeleterTest : public ShardServerTestFixture {
 public:
@@ -96,13 +97,14 @@ public:
         ShardServerTestFixture::tearDown();
     }
 
-    void setFilteringMetadataWithUUID(const UUID& uuid) {
+    void setFilteringMetadataWithUUID(const UUID& uuid,
+                                      const BSONObj& shardKeyPattern = kRangeBasedShardKeyPattern) {
         const OID epoch = OID::gen();
 
         auto rt = RoutingTableHistory::makeNew(
             kNss,
             uuid,
-            kShardKeyPattern,
+            shardKeyPattern,
             nullptr,
             false,
             epoch,
@@ -228,7 +230,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -257,7 +259,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -280,7 +282,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeInsertsDocumentToNotifySecondarie
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -313,7 +315,7 @@ TEST_F(
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -346,7 +348,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -378,7 +380,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -407,7 +409,7 @@ TEST_F(RangeDeleterTest,
                                kNss,
                                // Use a different UUID from the collection UUID.
                                UUID::gen(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -429,7 +431,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeThrowsErrorWhenCollectionDoesNotE
                                std::move(queriesComplete),
                                NamespaceString("someFake", "namespace"),
                                UUID::gen(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                ChunkRange(BSON(kShardKey << 0), BSON(kShardKey << 10)),
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -472,7 +474,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeLeavesDocumentsWhenTaskDocumentDo
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                UUID::gen(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -522,7 +524,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationAfterDeletingS
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -572,7 +574,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationOnlyOnceAfterS
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -618,7 +620,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotWaitForReplicationIfErrorD
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/);
@@ -648,7 +650,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnWriteConflictException)
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -679,7 +681,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnUnexpectedError) {
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -720,7 +722,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsDelayInBetweenBatches) {
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                task.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -752,7 +754,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsOrphanCleanupDelay) {
                                                   std::move(queriesComplete),
                                                   kNss,
                                                   uuid(),
-                                                  kShardKeyPattern,
+                                                  kRangeBasedShardKeyPattern,
                                                   range,
                                                   task.getId(),
                                                   orphanCleanupDelay);
@@ -790,7 +792,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRemovesRangeDeletionTaskOnSuccess
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -820,7 +822,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                fakeUuid,
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -857,7 +859,7 @@ TEST_F(RangeDeleterTest,
                                std::move(queriesComplete),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -888,7 +890,7 @@ DEATH_TEST_F(RangeDeleterTest, RemoveDocumentsInRangeCrashesIfInputFutureHasErro
                                std::move((queriesCompletePf.future)).semi(),
                                kNss,
                                uuid(),
-                               kShardKeyPattern,
+                               kRangeBasedShardKeyPattern,
                                range,
                                t.getId(),
                                Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */);
@@ -1025,8 +1027,9 @@ TEST_F(RenameRangeDeletionsTest, IdempotentRenameRangeDeletionsTest) {
     ASSERT_EQ(0, forRenameStore.count(_opCtx, BSONObj()));
 }
 
-TEST_F(RangeDeleterTest,
-       setOrphanCountersOnRangeDeletionTasksUpdatesTaskWithExpectedNumberOfOrphans) {
+TEST_F(
+    RangeDeleterTest,
+    setOrphanCountersOnRangeDeletionTasksUpdatesTaskForCollectionWithRangeShardKeyWithExpectedNumberOfOrphans) {
     const auto numOrphansInRange = 5;
     setFilteringMetadataWithUUID(uuid());
 
@@ -1046,6 +1049,44 @@ TEST_F(RangeDeleterTest,
         1);
 }
 
+TEST_F(
+    RangeDeleterTest,
+    setOrphanCountersOnRangeDeletionTasksUpdatesTaskForCollectionWithHashedShardKeyWithExpectedNumberOfOrphans) {
+    const BSONObj kHashedShardKeyPattern = BSON(kShardKey << "hashed");
+
+    DBDirectClient dbClient(_opCtx);
+    dbClient.createIndex(kNss.ns(),
+                         BSON("_id"
+                              << "hashed"));
+
+    setFilteringMetadataWithUUID(uuid(), kHashedShardKeyPattern);
+
+    const auto orphanedRangeLowerBoud = std::numeric_limits<int64_t>::max() / 2;
+    const ChunkRange orphansRange(BSON(kShardKey << orphanedRangeLowerBoud),
+                                  BSON(kShardKey << MAXKEY));
+
+    auto t = insertRangeDeletionTask(_opCtx, uuid(), orphansRange);
+    const auto numDocInserted = 10;
+    auto numOrphansInRange = 0;
+    for (auto i = 0; i < numDocInserted; ++i) {
+        dbClient.insert(kNss.toString(), BSON(kShardKey << i));
+        const auto hashedDocId = BSONElementHasher::hash64(BSON("_id" << i).firstElement(),
+                                                           BSONElementHasher::DEFAULT_HASH_SEED);
+        if (hashedDocId >= orphanedRangeLowerBoud) {
+            ++numOrphansInRange;
+        }
+    }
+
+    ASSERT(numOrphansInRange > 0);
+
+    setOrphanCountersOnRangeDeletionTasks(_opCtx);
+
+    PersistentTaskStore<RangeDeletionTask> store(NamespaceString::kRangeDeletionNamespace);
+    ASSERT_EQ(
+        store.count(_opCtx, BSON(RangeDeletionTask::kNumOrphanDocsFieldName << numOrphansInRange)),
+        1);
+}
+
 TEST_F(RangeDeleterTest, setOrphanCountersOnRangeDeletionTasksAddsZeroValueWhenNamespaceNotFound) {
     NamespaceString unexistentCollection("foo", "iDontExist");
     auto collUuid = UUID::gen();
diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp
index 75cb7be7049..2695ed842c0 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp
+++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp
@@ -101,8 +101,7 @@ StageConstraints DocumentSourceReshardingAddResumeId::constraints(
                             ChangeStreamRequirement::kDenylist);
 }
 
-Value DocumentSourceReshardingAddResumeId::serialize(
-    boost::optional<ExplainOptions::Verbosity> explain) const {
+Value DocumentSourceReshardingAddResumeId::serialize(const SerializationOptions& opts) const {
     return Value(Document{{kStageName, Value(Document{})}});
 }
 
diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h
index 31cbd97c694..4fb27980c68 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h
+++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h
@@ -53,7 +53,7 @@ public:
 
     DocumentSource::GetModPathsReturn getModifiedPaths() const final;
 
-    Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const;
+    Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override;
 
     StageConstraints constraints(Pipeline::SplitState pipeState) const final;
 
diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp
index 8075111e3af..8260cf3e4cc 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp
+++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp
@@ -112,7 +112,7 @@ StageConstraints DocumentSourceReshardingIterateTransaction::constraints(
 }
 
 Value DocumentSourceReshardingIterateTransaction::serialize(
-    boost::optional<ExplainOptions::Verbosity> explain) const {
+    const SerializationOptions& opts) const {
     return Value(
         Document{{kStageName,
                   Value(Document{{kIncludeCommitTransactionTimestampFieldName,
diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h
index 9589cb64a08..8c792116e6c 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h
+++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h
@@ -66,7 +66,7 @@ public:
 
     DocumentSource::GetModPathsReturn getModifiedPaths() const final;
 
-    Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const;
+    Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override;
 
     StageConstraints constraints(Pipeline::SplitState pipeState) const final;
 
diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp
index 3144723bf2d..157876391d3 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp
+++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp
@@ -90,12 +90,11 @@ StageConstraints DocumentSourceReshardingOwnershipMatch::constraints(
                             ChangeStreamRequirement::kDenylist);
 }
 
-Value DocumentSourceReshardingOwnershipMatch::serialize(
-    boost::optional<ExplainOptions::Verbosity> explain) const {
+Value DocumentSourceReshardingOwnershipMatch::serialize(const SerializationOptions& opts) const {
     return Value{Document{{kStageName,
                            DocumentSourceReshardingOwnershipMatchSpec(
                                _recipientShardId, _reshardingKey.getKeyPattern())
-                               .toBSON()}}};
+                               .toBSON(opts)}}};
 }
 
 DepsTracker::State DocumentSourceReshardingOwnershipMatch::getDependencies(
diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h
index 7a6db2bc125..b7da07a5a57 100644
--- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h
+++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h
@@ -58,7 +58,7 @@ public:
 
     DocumentSource::GetModPathsReturn getModifiedPaths() const final;
 
-    Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final;
+    Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override;
 
     StageConstraints constraints(Pipeline::SplitState pipeState) const final;
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp
index 5700f0326ae..c87cb3667dd 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp
@@ -91,17 +91,19 @@ CoordinatorCommitMonitor::CoordinatorCommitMonitor(
     std::vector<ShardId> recipientShards,
     CoordinatorCommitMonitor::TaskExecutorPtr executor,
     CancellationToken cancelToken,
+    int delayBeforeInitialQueryMillis,
     Milliseconds maxDelayBetweenQueries)
     : _ns(std::move(ns)),
       _recipientShards(std::move(recipientShards)),
       _executor(std::move(executor)),
       _cancelToken(std::move(cancelToken)),
       _threshold(Milliseconds(gRemainingReshardingOperationTimeThresholdMillis.load())),
+      _delayBeforeInitialQueryMillis(Milliseconds(delayBeforeInitialQueryMillis)),
       _maxDelayBetweenQueries(maxDelayBetweenQueries) {}
 
 
 SemiFuture<void> CoordinatorCommitMonitor::waitUntilRecipientsAreWithinCommitThreshold() const {
-    return _makeFuture()
+    return _makeFuture(_delayBeforeInitialQueryMillis)
         .onError([](Status status) {
             if (ErrorCodes::isCancellationError(status.code()) ||
                 ErrorCodes::isInterruption(status.code())) {
@@ -195,9 +197,16 @@ CoordinatorCommitMonitor::queryRemainingOperationTimeForRecipients() const {
     return {minRemainingTime, maxRemainingTime};
 }
 
-ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture() const {
+ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture(Milliseconds delayBetweenQueries) const {
     return ExecutorFuture<void>(_executor)
-        .then([this] { return queryRemainingOperationTimeForRecipients(); })
+        // Start waiting so that we have a more time to calculate a more realistic remaining time
+        // estimate.
+        .then([this, anchor = shared_from_this(), delayBetweenQueries] {
+            return _executor->sleepFor(delayBetweenQueries, _cancelToken)
+                .then([this, anchor = std::move(anchor)] {
+                    return queryRemainingOperationTimeForRecipients();
+                });
+        })
         .onError([this](Status status) {
             if (_cancelToken.isCanceled()) {
                 // Do not retry on cancellation errors.
@@ -233,12 +242,10 @@ ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture() const {
             // The following ensures that the monitor would never sleep for more than a predefined
             // maximum delay between querying recipient shards. Thus, it can handle very large,
             // and potentially inaccurate estimates of the remaining operation time.
-            auto sleepTime = std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries);
-            return _executor->sleepFor(sleepTime, _cancelToken)
-                .then([this, anchor = std::move(anchor)] {
-                    // We are not canceled yet, so schedule new queries against recipient shards.
-                    return _makeFuture();
-                });
+            auto delayBetweenQueries =
+                std::min(remainingTimes.max - _threshold, _maxDelayBetweenQueries);
+
+            return _makeFuture(delayBetweenQueries);
         });
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h
index 64544981ae5..be722fb11bd 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h
+++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.h
@@ -72,6 +72,7 @@ public:
                              std::vector<ShardId> recipientShards,
                              TaskExecutorPtr executor,
                              CancellationToken cancelToken,
+                             int delayBeforeInitialQueryMillis,
                              Milliseconds maxDelayBetweenQueries = kMaxDelayBetweenQueries);
 
     SemiFuture<void> waitUntilRecipientsAreWithinCommitThreshold() const;
@@ -88,7 +89,7 @@ public:
     RemainingOperationTimes queryRemainingOperationTimeForRecipients() const;
 
 private:
-    ExecutorFuture<void> _makeFuture() const;
+    ExecutorFuture<void> _makeFuture(Milliseconds delayBetweenQueries) const;
 
     static constexpr auto kDiagnosticLogLevel = 0;
     static constexpr auto kMaxDelayBetweenQueries = Seconds(30);
@@ -98,6 +99,8 @@ private:
     const TaskExecutorPtr _executor;
     const CancellationToken _cancelToken;
     const Milliseconds _threshold;
+
+    const Milliseconds _delayBeforeInitialQueryMillis;
     const Milliseconds _maxDelayBetweenQueries;
 
     TaskExecutorPtr _networkExecutor;
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp
index 2fe3075f1fc..0804565201c 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp
@@ -152,7 +152,7 @@ void CoordinatorCommitMonitorTest::setUp() {
 
     _cancellationSource = std::make_unique<CancellationSource>();
     _commitMonitor = std::make_shared<CoordinatorCommitMonitor>(
-        _ns, _recipientShards, _futureExecutor, _cancellationSource->token(), Milliseconds(0));
+        _ns, _recipientShards, _futureExecutor, _cancellationSource->token(), 0, Milliseconds(0));
     _commitMonitor->setNetworkExecutorForTest(executor());
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
index ea3451f1c4a..988360d6799 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
@@ -98,6 +98,7 @@ MONGO_FAIL_POINT_DEFINE(pauseBeforeInsertCoordinatorDoc);
 MONGO_FAIL_POINT_DEFINE(pauseBeforeCTHolderInitialization);
 
 const std::string kReshardingCoordinatorActiveIndexName = "ReshardingCoordinatorActiveIndex";
+const int kReshardingNumInitialChunksDefault = 90;
 const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max());
 const WriteConcernOptions kMajorityWriteConcern{
     WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, Seconds(0)};
@@ -879,7 +880,7 @@ ReshardingCoordinatorExternalStateImpl::calculateParticipantShardsAndChunks(
     } else {
         int numInitialChunks = coordinatorDoc.getNumInitialChunks()
             ? *coordinatorDoc.getNumInitialChunks()
-            : cm.numChunks();
+            : kReshardingNumInitialChunksDefault;
 
         ShardKeyPattern shardKey(coordinatorDoc.getReshardingKey());
         const auto tempNs = coordinatorDoc.getTempReshardingNss();
@@ -1714,7 +1715,8 @@ void ReshardingCoordinatorService::ReshardingCoordinator::_startCommitMonitor(
         _coordinatorDoc.getSourceNss(),
         extractShardIdsFromParticipantEntries(_coordinatorDoc.getRecipientShards()),
         **executor,
-        _ctHolder->getCommitMonitorToken());
+        _ctHolder->getCommitMonitorToken(),
+        resharding::gReshardingDelayBeforeRemainingOperationTimeQueryMillis.load());
 
     _commitMonitorQuiesced = _commitMonitor->waitUntilRecipientsAreWithinCommitThreshold()
                                  .thenRunOn(**executor)
diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp
index d0b27f00c3c..8635e389cf4 100644
--- a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp
+++ b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp
@@ -282,7 +282,8 @@ void updateSessionRecord(OperationContext* opCtx,
                          BSONObj o2Field,
                          std::vector<StmtId> stmtIds,
                          boost::optional<repl::OpTime> preImageOpTime,
-                         boost::optional<repl::OpTime> postImageOpTime) {
+                         boost::optional<repl::OpTime> postImageOpTime,
+                         NamespaceString sourceNss) {
     invariant(opCtx->getLogicalSessionId());
     invariant(opCtx->getTxnNumber());
 
@@ -296,7 +297,7 @@ void updateSessionRecord(OperationContext* opCtx,
     oplogEntry.setOpType(repl::OpTypeEnum::kNoop);
     oplogEntry.setObject(SessionCatalogMigration::kSessionOplogTag);
     oplogEntry.setObject2(std::move(o2Field));
-    oplogEntry.setNss({});
+    oplogEntry.setNss(std::move(sourceNss));
     oplogEntry.setSessionId(sessionId);
     oplogEntry.setTxnNumber(txnNumber);
     oplogEntry.setStatementIds(stmtIds);
diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.h b/src/mongo/db/s/resharding/resharding_data_copy_util.h
index b51cfc250a5..af24152aaf2 100644
--- a/src/mongo/db/s/resharding/resharding_data_copy_util.h
+++ b/src/mongo/db/s/resharding/resharding_data_copy_util.h
@@ -144,7 +144,8 @@ void updateSessionRecord(OperationContext* opCtx,
                          BSONObj o2Field,
                          std::vector<StmtId> stmtIds,
                          boost::optional<repl::OpTime> preImageOpTime,
-                         boost::optional<repl::OpTime> postImageOpTime);
+                         boost::optional<repl::OpTime> postImageOpTime,
+                         NamespaceString sourceNss);
 
 /**
  * Calls and returns the value from the supplied lambda function.
diff --git a/src/mongo/db/s/resharding/resharding_donor_service.cpp b/src/mongo/db/s/resharding/resharding_donor_service.cpp
index 4594c4018a6..2e75751ba01 100644
--- a/src/mongo/db/s/resharding/resharding_donor_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_donor_service.cpp
@@ -47,6 +47,7 @@
 #include "mongo/db/persistent_task_store.h"
 #include "mongo/db/repl/repl_client_info.h"
 #include "mongo/db/repl/wait_for_majority_service.h"
+#include "mongo/db/s/collection_sharding_runtime.h"
 #include "mongo/db/s/recoverable_critical_section_service.h"
 #include "mongo/db/s/resharding/resharding_change_event_o2_field_gen.h"
 #include "mongo/db/s/resharding/resharding_data_copy_util.h"
@@ -179,8 +180,9 @@ public:
         }
     }
 
-    void clearFilteringMetadata(OperationContext* opCtx) {
-        resharding::clearFilteringMetadata(opCtx, true /* scheduleAsyncRefresh */);
+    void refreshCollectionPlacementInfo(OperationContext* opCtx,
+                                        const NamespaceString& sourceNss) override {
+        onShardVersionMismatch(opCtx, sourceNss, boost::none);
     }
 };
 
@@ -369,8 +371,15 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin
 
                {
                    auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc());
-
-                   _externalState->clearFilteringMetadata(opCtx.get());
+                   std::initializer_list<NamespaceString> namespacesToRefresh{
+                       _metadata.getSourceNss(), _metadata.getTempReshardingNss()};
+
+                   // Clear filtering metadata for the source and temp resharding nss.
+                   for (const auto& nss : namespacesToRefresh) {
+                       AutoGetCollection autoColl(opCtx.get(), nss, MODE_IX);
+                       CollectionShardingRuntime::get(opCtx.get(), nss)
+                           ->clearFilteringMetadata(opCtx.get());
+                   }
 
                    RecoverableCriticalSectionService::get(opCtx.get())
                        ->releaseRecoverableCriticalSection(
@@ -380,6 +389,13 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin
                            ShardingCatalogClient::kLocalWriteConcern);
 
                    _metrics()->leaveCriticalSection(getCurrentTime());
+
+                   // We force a refresh to make sure that the placement information is updated in
+                   // cache after abort decision before the donor state document is deleted.
+                   for (const auto& nss : namespacesToRefresh) {
+                       _externalState->refreshCollectionPlacementInfo(opCtx.get(), nss);
+                       _externalState->waitForCollectionFlush(opCtx.get(), nss);
+                   }
                }
 
                auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc());
diff --git a/src/mongo/db/s/resharding/resharding_donor_service.h b/src/mongo/db/s/resharding/resharding_donor_service.h
index b50c88b6af5..7b5331f93ac 100644
--- a/src/mongo/db/s/resharding/resharding_donor_service.h
+++ b/src/mongo/db/s/resharding/resharding_donor_service.h
@@ -298,7 +298,8 @@ public:
                                            const BSONObj& query,
                                            const BSONObj& update) = 0;
 
-    virtual void clearFilteringMetadata(OperationContext* opCtx) = 0;
+    virtual void refreshCollectionPlacementInfo(OperationContext* opCtx,
+                                                const NamespaceString& sourceNss) = 0;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
index cb358c3f508..16e1507be1c 100644
--- a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp
@@ -84,7 +84,8 @@ public:
                                    const BSONObj& query,
                                    const BSONObj& update) override {}
 
-    void clearFilteringMetadata(OperationContext* opCtx) override {}
+    void refreshCollectionPlacementInfo(OperationContext* opCtx,
+                                        const NamespaceString& sourceNss) override {}
 };
 
 class DonorOpObserverForTest : public OpObserverForTest {
diff --git a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp
index 80338642587..95c29d0825c 100644
--- a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp
+++ b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp
@@ -108,6 +108,7 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp
     invariant(op.getTxnNumber());
     invariant(op.get_id());
 
+    auto sourceNss = op.getNss();
     auto lsid = *op.getSessionId();
     if (isInternalSessionForNonRetryableWrite(lsid)) {
         // Skip internal sessions for non-retryable writes since they only support transactions
@@ -157,7 +158,8 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp
                                                        std::move(o2Field),
                                                        std::move(stmtIds),
                                                        std::move(preImageOpTime),
-                                                       std::move(postImageOpTime));
+                                                       std::move(postImageOpTime),
+                                                       std::move(sourceNss));
         });
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_server_parameters.idl b/src/mongo/db/s/resharding/resharding_server_parameters.idl
index daaedfc9ea4..7508d30c772 100644
--- a/src/mongo/db/s/resharding/resharding_server_parameters.idl
+++ b/src/mongo/db/s/resharding/resharding_server_parameters.idl
@@ -151,6 +151,21 @@ server_parameters:
         validator:
             gte: 0
     
+    reshardingDelayBeforeRemainingOperationTimeQueryMillis:
+        description: >-
+            Initial delay before querying for remaining operation time from recipient shards.
+            The delay allows for applying more oplog entries before calculating time remaining, giving
+            a more accurate value.
+            Note we will have this delay every time we happen to have a failover occur.
+        set_at: [startup, runtime]
+        cpp_vartype: AtomicWord<int>
+        cpp_varname: gReshardingDelayBeforeRemainingOperationTimeQueryMillis
+        default:
+            expr: 0
+        validator:
+            gte: 0
+        redact: false
+    
     reshardingCriticalSectionTimeoutMillis:
         description: >-
             The upper limit on how long to wait to hear back from recipient shards reaching strict
diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp
index 764a32d68d5..96a08c210fa 100644
--- a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp
+++ b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp
@@ -190,7 +190,8 @@ boost::optional<SharedSemiFuture<void>> ReshardingTxnCloner::doOneRecord(
                                                        TransactionParticipant::kDeadEndSentinel,
                                                        {kIncompleteHistoryStmtId},
                                                        boost::none /* preImageOpTime */,
-                                                       boost::none /* postImageOpTime */);
+                                                       boost::none /* postImageOpTime */,
+                                                       {});
         });
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
index 3a10e01306c..b77db34ef11 100644
--- a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp
@@ -40,6 +40,7 @@
 #include "mongo/db/logical_session_cache_noop.h"
 #include "mongo/db/persistent_task_store.h"
 #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h"
+#include "mongo/db/query/cursor_response.h"
 #include "mongo/db/repl/storage_interface_impl.h"
 #include "mongo/db/repl/wait_for_majority_service.h"
 #include "mongo/db/s/resharding/resharding_server_parameters_gen.h"
diff --git a/src/mongo/db/s/set_allow_migrations_coordinator.cpp b/src/mongo/db/s/set_allow_migrations_coordinator.cpp
index 1cf2edac166..aee3052a8ee 100644
--- a/src/mongo/db/s/set_allow_migrations_coordinator.cpp
+++ b/src/mongo/db/s/set_allow_migrations_coordinator.cpp
@@ -122,12 +122,12 @@ ExecutorFuture<void> SetAllowMigrationsCoordinator::_runImpl(
                 return updateOp;
             }());
 
-            updateRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON());
-
-            auto response = configShard->runBatchWriteCommand(opCtx,
-                                                              Shard::kDefaultConfigCommandTimeout,
-                                                              updateRequest,
-                                                              Shard::RetryPolicy::kIdempotent);
+            auto response =
+                configShard->runBatchWriteCommand(opCtx,
+                                                  Shard::kDefaultConfigCommandTimeout,
+                                                  updateRequest,
+                                                  ShardingCatalogClient::kMajorityWriteConcern,
+                                                  Shard::RetryPolicy::kIdempotent);
 
             uassertStatusOK(response.toStatus());
         })
diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp
index 34cd5ea7cd3..ebb4497260a 100644
--- a/src/mongo/db/s/shard_key_util.cpp
+++ b/src/mongo/db/s/shard_key_util.cpp
@@ -292,7 +292,15 @@ void ValidationBehaviorsShardCollection::verifyUsefulNonMultiKeyIndex(
         "admin",
         BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey),
         res);
-    uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success);
+
+    // checkShardingIndex may return UnknownError if a compatible shard key index cannot be
+    // found when the command is executed on a node with an old binary. In this case, we should
+    // return InvalidOptions to correspond with the shardCollection behavior.
+    const auto status = getStatusFromCommandResult(res);
+    if (status == ErrorCodes::UnknownError) {
+        uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success);
+    }
+    uassertStatusOK(status);
 }
 
 void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
@@ -342,7 +350,7 @@ std::vector<BSONObj> ValidationBehaviorsRefineShardKey::loadIndexes(
 
 void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex(
     const NamespaceString& nss, const BSONObj& proposedKey) const {
-    auto checkShardingIndexRes = uassertStatusOK(_indexShard->runCommand(
+    auto res = uassertStatusOK(_indexShard->runCommand(
         _opCtx,
         ReadPreferenceSetting(ReadPreference::PrimaryOnly),
         "admin",
@@ -350,13 +358,14 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex(
             BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey),
             _cm.getVersion(_indexShard->getId())),
         Shard::RetryPolicy::kIdempotent));
-    if (checkShardingIndexRes.commandStatus == ErrorCodes::UnknownError) {
-        // CheckShardingIndex returns UnknownError if a compatible shard key index cannot be found,
-        // but we return InvalidOptions to correspond with the shardCollection behavior.
-        uasserted(ErrorCodes::InvalidOptions, checkShardingIndexRes.response["errmsg"].str());
+
+    // checkShardingIndex may return UnknownError if a compatible shard key index cannot be
+    // found when the command is executed on a node with an old binary. In this case, we should
+    // return InvalidOptions to correspond with the shardCollection behavior.
+    if (res.commandStatus == ErrorCodes::UnknownError) {
+        uasserted(ErrorCodes::InvalidOptions, res.response["errmsg"].str());
     }
-    // Rethrow any other error to allow retries on retryable errors.
-    uassertStatusOK(checkShardingIndexRes.commandStatus);
+    uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(res));
 }
 
 void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(const NamespaceString& nss,
diff --git a/src/mongo/db/s/shard_local.cpp b/src/mongo/db/s/shard_local.cpp
index fec64ebd8f2..229a61e8480 100644
--- a/src/mongo/db/s/shard_local.cpp
+++ b/src/mongo/db/s/shard_local.cpp
@@ -227,4 +227,24 @@ Status ShardLocal::runAggregation(
     return _rsLocalClient.runAggregation(opCtx, aggRequest, callback);
 }
 
+BatchedCommandResponse ShardLocal::runBatchWriteCommand(OperationContext* opCtx,
+                                                        const Milliseconds maxTimeMS,
+                                                        const BatchedCommandRequest& batchRequest,
+                                                        const WriteConcernOptions& writeConcern,
+                                                        RetryPolicy retryPolicy) {
+    // A request dispatched through a local client is served within the same thread that submits it
+    // (so that the opCtx needs to be used as the vehicle to pass the WC to the ServiceEntryPoint).
+    const auto originalWC = opCtx->getWriteConcern();
+    ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); });
+    opCtx->setWriteConcern(writeConcern);
+
+    const auto dbName = batchRequest.getNS().db();
+    const BSONObj cmdObj = [&] {
+        BSONObjBuilder cmdObjBuilder;
+        batchRequest.serialize(&cmdObjBuilder);
+        return cmdObjBuilder.obj();
+    }();
+
+    return _submitBatchWriteCommand(opCtx, cmdObj, dbName, maxTimeMS, retryPolicy);
+}
 }  // namespace mongo
diff --git a/src/mongo/db/s/shard_local.h b/src/mongo/db/s/shard_local.h
index e7c6e74b000..f82e3f7bb96 100644
--- a/src/mongo/db/s/shard_local.h
+++ b/src/mongo/db/s/shard_local.h
@@ -81,6 +81,12 @@ public:
         std::function<bool(const std::vector<BSONObj>& batch,
                            const boost::optional<BSONObj>& postBatchResumeToken)> callback);
 
+    BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx,
+                                                Milliseconds maxTimeMS,
+                                                const BatchedCommandRequest& batchRequest,
+                                                const WriteConcernOptions& writeConcern,
+                                                RetryPolicy retryPolicy) final;
+
 private:
     StatusWith<Shard::CommandResponse> _runCommand(OperationContext* opCtx,
                                                    const ReadPreferenceSetting& unused,
diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp
index a374159bc87..91aa5b9564b 100644
--- a/src/mongo/db/s/shard_server_op_observer.cpp
+++ b/src/mongo/db/s/shard_server_op_observer.cpp
@@ -587,9 +587,16 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx,
                                                const BSONObj& idIndex,
                                                const OplogSlot& createOpTime,
                                                bool fromMigrate) {
-    // Only the shard primay nodes control the collection creation and secondaries just follow
-    // Secondaries CSR will be the defaulted one (UNKNOWN in most of the cases)
+    // Only the shard primary nodes control the collection creation.
     if (!opCtx->writesAreReplicated()) {
+        // On secondaries node of sharded cluster we force the cleanup of the filtering metadata in
+        // order to remove anything that was left from any previous collection instance. This could
+        // happen by first having an UNSHARDED version for a collection that didn't exist followed
+        // by a movePrimary to the current shard.
+        if (ShardingState::get(opCtx)->enabled()) {
+            CollectionShardingRuntime::get(opCtx, collectionName)->clearFilteringMetadata(opCtx);
+        }
+
         return;
     }
 
diff --git a/src/mongo/db/s/sharding_ddl_util.cpp b/src/mongo/db/s/sharding_ddl_util.cpp
index 14c18e46923..ba13de4abbd 100644
--- a/src/mongo/db/s/sharding_ddl_util.cpp
+++ b/src/mongo/db/s/sharding_ddl_util.cpp
@@ -77,11 +77,14 @@ void updateTags(OperationContext* opCtx,
         }()});
         return updateOp;
     }());
-    request.setWriteConcern(writeConcern.toBSON());
 
     auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-    auto response = configShard->runBatchWriteCommand(
-        opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
+    auto response =
+        configShard->runBatchWriteCommand(opCtx,
+                                          Milliseconds::max(),
+                                          request,
+                                          writeConcern,
+                                          Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
 
     uassertStatusOK(response.toStatus());
 }
@@ -105,11 +108,13 @@ void deleteChunks(OperationContext* opCtx,
         return deleteOp;
     }());
 
-    request.setWriteConcern(writeConcern.toBSON());
-
     auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-    auto response = configShard->runBatchWriteCommand(
-        opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
+    auto response =
+        configShard->runBatchWriteCommand(opCtx,
+                                          Milliseconds::max(),
+                                          request,
+                                          writeConcern,
+                                          Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
 
     uassertStatusOK(response.toStatus());
 }
@@ -336,11 +341,13 @@ void removeTagsMetadataFromConfig_notIdempotent(OperationContext* opCtx,
         return deleteOp;
     }());
 
-    request.setWriteConcern(writeConcern.toBSON());
-
     auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-    auto response = configShard->runBatchWriteCommand(
-        opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
+    auto response =
+        configShard->runBatchWriteCommand(opCtx,
+                                          Milliseconds::max(),
+                                          request,
+                                          writeConcern,
+                                          Shard::RetryPolicy::kIdempotentOrCursorInvalidated);
 
     uassertStatusOK(response.toStatus());
 }