SERVER-87061 Check that the sharding catalog and local catalog agree on the collection UUIDr7.3.0-rc4

GitOrigin-RevId: a503ec734ac25df2dc4feb8612b7861581cedfd0

10 files changed, 191 insertions, 14 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_csrs_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_csrs_continuous_config_stepdown.yml
index 78a4f40e08b..7e4dc129f1a 100644
--- a/buildscripts/resmokeconfig/suites/sharding_csrs_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_csrs_continuous_config_stepdown.yml
@@ -104,6 +104,7 @@ selector:
   - jstests/sharding/move_chunk_insert_with_write_retryability.js
   - jstests/sharding/move_chunk_remove_with_write_retryability.js
   - jstests/sharding/move_chunk_update_with_write_retryability.js
+  - jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js  # sets a failpoint on the config primary
   - jstests/sharding/refine_collection_shard_key_atomic.js  # sets a failpoint on the config primary
   - jstests/sharding/restart_transactions.js
   - jstests/sharding/txn_two_phase_commit_commands_basic_requirements.js
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 12730f5eae9..936e92585e8 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -394,7 +394,7 @@ last-continuous:
   - test_file: jstests/sharding/query/sharded_lookup_execution.js
     ticket: SERVER-77427
   - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
-    ticket: SERVER-84723
+    ticket: SERVER-87061
   - test_file: jstests/sharding/fsync_lock_unlock.js
     ticket: SERVER-73685
   - test_file: jstests/sharding/fsync_lock_ddl_lock.js
@@ -1019,7 +1019,7 @@ last-lts:
   - test_file: jstests/sharding/query/sharded_lookup_execution.js
     ticket: SERVER-77427
   - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
-    ticket: SERVER-84723
+    ticket: SERVER-87061
   - test_file: jstests/sharding/fsync_lock_unlock.js
     ticket: SERVER-73685
   - test_file: jstests/sharding/fsync_lock_ddl_lock.js
diff --git a/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js b/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
index 3e03f448129..fa507919a89 100644
--- a/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
+++ b/jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js
@@ -3,6 +3,7 @@
  * TransientTransactionError label) if a collection or database placement changes have occurred
  * later than the transaction data snapshot timestamp.
  */
+import {configureFailPoint} from "jstests/libs/fail_point_util.js";
 
 const st = new ShardingTest({mongos: 1, shards: 2});
 
@@ -244,6 +245,81 @@ const st = new ShardingTest({mongos: 1, shards: 2});
             runTest(readConcern, command);
         }));
     }
+
+    // Test transaction concurrent with reshardCollection.
+    {
+        function runTest(readConcernLevel, command) {
+            jsTest.log("Running transaction + resharding test with read concern " +
+                       readConcernLevel + " and command " + command);
+
+            // Setup initial state:
+            assert.commandWorked(st.s.getDB(dbName).dropDatabase());
+            st.adminCommand({enableSharding: dbName, primaryShard: st.shard0.shardName});
+
+            st.adminCommand({shardCollection: ns1, key: {x: 1}});
+            assert.commandWorked(st.splitAt(ns1, {x: 0}));
+            assert.commandWorked(st.moveChunk(ns1, {x: -1}, st.shard0.shardName));
+            assert.commandWorked(st.moveChunk(ns1, {x: 1}, st.shard1.shardName));
+
+            assert.commandWorked(coll1.insertMany([{x: -1, y: 0}, {x: 1, y: 0}]));
+
+            assert.commandWorked(coll2.insertOne({a: 1}));
+
+            // Set fp to block resharding after commit on configsvr but before commit on shards.
+            // We seek to test an interleaving where the transaction executes at a logical timestamp
+            // that falls between the timestamp at which resharding commits on the configsvr and the
+            // timestamp at which the temporary resharding collections are renamed to the original
+            // nss on the shards.
+            const fp = configureFailPoint(st.configRS.getPrimary(),
+                                          "reshardingPauseBeforeTellingParticipantsToCommit");
+
+            // On parallel shell, start resharding
+            const joinResharding = startParallelShell(() => {
+                assert.commandWorked(db.adminCommand({reshardCollection: 'test.foo', key: {y: 1}}));
+            }, st.s.port);
+
+            // Await configsvr to have done its part of the commit.
+            fp.wait();
+
+            const session = st.s.startSession();
+            const sessionDB = session.getDatabase(dbName);
+            const sessionColl1 = sessionDB.getCollection(collName1);
+            const sessionColl2 = sessionDB.getCollection(collName2);
+
+            // Make sure the session knows of a clusterTime inclusive of the resharding operation up
+            // to the commit on the configsvr.
+            assert.commandWorked(sessionColl2.insert({a: 2}));
+
+            // Start txn.
+            session.startTransaction({readConcern: {level: readConcernLevel}});
+            assert.eq(2, sessionColl2.find().itcount());
+
+            // Unset fp and wait for resharding to finish.
+            fp.off();
+            joinResharding();
+
+            // Make sure the router is aware of the new (post-resharding) routing table for test.foo
+            assert.eq(2, coll1.find({y: 0}).itcount());
+
+            // Now operate on coll1 within the transaction and expect to get a conflict.
+            let err = assert.throwsWithCode(() => {
+                if (command === 'find') {
+                    sessionColl1.find().itcount();
+                } else if (command === 'aggregate') {
+                    sessionColl1.aggregate().itcount();
+                } else if (command === 'update') {
+                    assert.commandWorked(sessionColl1.updateMany({}, {$set: {c: 1}}));
+                }
+            }, [ErrorCodes.WriteConflict, ErrorCodes.SnapshotUnavailable]);
+            assert.contains("TransientTransactionError", err.errorLabels, tojson(err));
+
+            session.abortTransaction();
+        }
+
+        readConcerns.forEach((readConcern) => commands.forEach((command) => {
+            runTest(readConcern, command);
+        }));
+    }
 }
 
 st.stop();
diff --git a/src/mongo/db/catalog_raii.cpp b/src/mongo/db/catalog_raii.cpp
index d226e8f628c..ab7ce48f8d2 100644
--- a/src/mongo/db/catalog_raii.cpp
+++ b/src/mongo/db/catalog_raii.cpp
@@ -379,6 +379,11 @@ AutoGetCollection::AutoGetCollection(OperationContext* opCtx,
                 opCtx, *catalog, _coll, _resolvedNss);
         }
 
+        if (receivedShardVersion) {
+            shard_role_details::checkShardingAndLocalCatalogCollectionUUIDMatch(
+                opCtx, _resolvedNss, *receivedShardVersion, collDesc, _coll);
+        }
+
         return;
     }
 
diff --git a/src/mongo/db/db_raii.cpp b/src/mongo/db/db_raii.cpp
index 3fedd36b5b5..b774e2a3f35 100644
--- a/src/mongo/db/db_raii.cpp
+++ b/src/mongo/db/db_raii.cpp
@@ -452,6 +452,11 @@ AutoGetCollectionForRead::AutoGetCollectionForRead(OperationContext* opCtx,
 
         checkCollectionUUIDMismatch(opCtx, *catalog, _resolvedNss, _coll, options._expectedUUID);
 
+        if (receivedShardVersion) {
+            shard_role_details::checkShardingAndLocalCatalogCollectionUUIDMatch(
+                opCtx, _resolvedNss, *receivedShardVersion, collDesc, _coll);
+        }
+
         return;
     }
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
index 733fbe3d655..d1c59f0438e 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
@@ -140,6 +140,7 @@ MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeInitializing);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCloning);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeBlockingWrites);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeDecisionPersisted);
+MONGO_FAIL_POINT_DEFINE(reshardingPauseBeforeTellingParticipantsToCommit);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeRemovingStateDoc);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCompletion);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeStartingErrorFlow);
@@ -2754,6 +2755,12 @@ void ReshardingCoordinator::_tellAllDonorsToRefresh(
 
 void ReshardingCoordinator::_tellAllParticipantsToCommit(
     const NamespaceString& nss, const std::shared_ptr<executor::ScopedTaskExecutor>& executor) {
+    {
+        auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc());
+        reshardingPauseBeforeTellingParticipantsToCommit.pauseWhileSetAndNotCanceled(
+            opCtx.get(), _ctHolder->getAbortToken());
+    }
+
     auto opts = createShardsvrCommitReshardCollectionOptions(
         nss, _coordinatorDoc.getReshardingUUID(), **executor, _ctHolder->getStepdownToken(), {});
     opts->cmd.setDbName(DatabaseName::kAdmin);
diff --git a/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp b/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp
index 52bac27be96..cfce85e7a30 100644
--- a/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp
@@ -279,7 +279,7 @@ protected:
                             env.version.placementVersion().epoch(),
                             env.version.placementVersion().getTimestamp(),
                             Date_t::now(),
-                            UUID::gen(),
+                            env.sourceUuid,
                             BSON(kShardKey << 1));
         coll.setAllowMigrations(false);
 
diff --git a/src/mongo/db/s/scoped_collection_metadata.h b/src/mongo/db/s/scoped_collection_metadata.h
index 11b460bd279..2e0d512ca46 100644
--- a/src/mongo/db/s/scoped_collection_metadata.h
+++ b/src/mongo/db/s/scoped_collection_metadata.h
@@ -102,6 +102,10 @@ public:
         return _impl->get().uuidMatches(uuid);
     }
 
+    const UUID& getUUID() const {
+        return _impl->get().getUUID();
+    }
+
     const boost::optional<TypeCollectionReshardingFields>& getReshardingFields() const {
         return _impl->get().getReshardingFields();
     }
diff --git a/src/mongo/db/shard_role.cpp b/src/mongo/db/shard_role.cpp
index cad506c8d74..8f5b36d01d2 100644
--- a/src/mongo/db/shard_role.cpp
+++ b/src/mongo/db/shard_role.cpp
@@ -35,6 +35,7 @@
 #include <boost/optional.hpp>
 #include <boost/optional/optional.hpp>
 #include <boost/utility/in_place_factory.hpp>  // IWYU pragma: keep
+#include <fmt/format.h>
 #include <iterator>
 #include <list>
 #include <map>
@@ -86,6 +87,8 @@
 
 namespace mongo {
 
+using namespace fmt::literals;
+
 using TransactionResources = shard_role_details::TransactionResources;
 
 namespace {
@@ -356,21 +359,32 @@ CollectionOrViewAcquisitions acquireResolvedCollectionsOrViewsWithoutTakingLocks
         const bool isCollection =
             holds_alternative<CollectionPtr>(snapshotedServices.collectionPtrOrView);
 
-        if (holds_alternative<PlacementConcern>(prerequisites.placementConcern)) {
-            const auto& placementConcern = get<PlacementConcern>(prerequisites.placementConcern);
-
-            if (placementConcern.shardVersion == ShardVersion::UNSHARDED()) {
-                shard_role_details::checkLocalCatalogIsValidForUnshardedShardVersion(
-                    opCtx,
-                    catalog,
-                    isCollection ? get<CollectionPtr>(snapshotedServices.collectionPtrOrView)
-                                 : CollectionPtr::null,
-                    prerequisites.nss);
-            }
+        const boost::optional<ShardVersion> placementConcernShardVersion =
+            holds_alternative<PlacementConcern>(prerequisites.placementConcern)
+            ? get<PlacementConcern>(prerequisites.placementConcern).shardVersion
+            : boost::none;
+
+        if (placementConcernShardVersion == ShardVersion::UNSHARDED()) {
+            shard_role_details::checkLocalCatalogIsValidForUnshardedShardVersion(
+                opCtx,
+                catalog,
+                isCollection ? get<CollectionPtr>(snapshotedServices.collectionPtrOrView)
+                             : CollectionPtr::null,
+                prerequisites.nss);
         }
 
         if (isCollection) {
             const auto& collectionPtr = get<CollectionPtr>(snapshotedServices.collectionPtrOrView);
+
+            if (placementConcernShardVersion && snapshotedServices.collectionDescription) {
+                shard_role_details::checkShardingAndLocalCatalogCollectionUUIDMatch(
+                    opCtx,
+                    prerequisites.nss,
+                    *placementConcernShardVersion,
+                    *snapshotedServices.collectionDescription,
+                    collectionPtr);
+            }
+
             invariant(!prerequisites.uuid || prerequisites.uuid == collectionPtr->uuid());
             if (!prerequisites.uuid && collectionPtr) {
                 // If the uuid wasn't originally set on the AcquisitionRequest, set it now on the
@@ -1665,4 +1679,59 @@ void shard_role_details::checkLocalCatalogIsValidForUnshardedShardVersion(
     }
 }
 
+void shard_role_details::checkShardingAndLocalCatalogCollectionUUIDMatch(
+    OperationContext* opCtx,
+    const NamespaceString& nss,
+    const ShardVersion& requestedShardVersion,
+    const ScopedCollectionDescription& shardingCollectionDescription,
+    const CollectionPtr& collectionPtr) {
+    // Skip the check if the requested shard version corresponds to an untracked collection or
+    // corresponds to this shard not own any chunk. Also skip the check if the router attached
+    // ShardVersion::IGNORED, since in this case the router broadcasts request to shards that may
+    // not even own the collection at all (so they won't have any uuid on their local catalog).
+    if (requestedShardVersion == ShardVersion::UNSHARDED() ||
+        !requestedShardVersion.placementVersion().isSet() ||
+        ShardVersion::isPlacementVersionIgnored(requestedShardVersion)) {
+        return;
+    }
+
+    // Skip checking resharding temporary collections. The reason is that resharding registers the
+    // temporary collections on the sharding catalog before creating them on the shards, without
+    // holding any critical section.
+    // TODO: SERVER-87235 Remove this when resharding creates the temporary collections under a
+    // critical section.
+    if (nss.isTemporaryReshardingCollection()) {
+        return;
+    }
+
+    // Check that the collection uuid in the sharding catalog and the one on the local catalog
+    // match.
+    if (shardingCollectionDescription.hasRoutingTable() &&
+        (!collectionPtr || !shardingCollectionDescription.uuidMatches(collectionPtr->uuid()))) {
+        if ((opCtx->inMultiDocumentTransaction() ||
+             repl::ReadConcernArgs::get(opCtx).getArgsAtClusterTime())) {
+            // If in multi-document transaction or snapshot read, throw SnapshotUnavailable so that
+            // the transaction can be retried. This situation is known to be possible when a
+            // collection undergoes resharding, due to the resharding commit protocol. See
+            // SERVER-87061.
+            // TODO: SERVER-87235: Remove this condition and leave only the tassert below also for
+            // transaction and snapshot reads.
+            uasserted(
+                ErrorCodes::SnapshotUnavailable,
+                "Sharding catalog and local catalog collection uuid do not match. Nss: '{}', sharding uuid: '{}', local uuid: '{}'"_format(
+                    nss.toStringForErrorMsg(),
+                    shardingCollectionDescription.getUUID().toString(),
+                    collectionPtr ? collectionPtr->uuid().toString() : ""));
+
+        } else {
+            tasserted(
+                8706100,
+                "Sharding catalog and local catalog collection uuid do not match. Nss: '{}', sharding uuid: '{}', local uuid: '{}'"_format(
+                    nss.toStringForErrorMsg(),
+                    shardingCollectionDescription.getUUID().toString(),
+                    collectionPtr ? collectionPtr->uuid().toString() : ""));
+        }
+    }
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/shard_role.h b/src/mongo/db/shard_role.h
index 2b79d39c8a4..b83e79a87a9 100644
--- a/src/mongo/db/shard_role.h
+++ b/src/mongo/db/shard_role.h
@@ -607,5 +607,15 @@ void checkLocalCatalogIsValidForUnshardedShardVersion(OperationContext* opCtx,
                                                       const CollectionPtr& collectionPtr,
                                                       const NamespaceString& nss);
 
+/*
+ * Check that the collection uuid on the sharding catalog and the local catalog match.
+ */
+void checkShardingAndLocalCatalogCollectionUUIDMatch(
+    OperationContext* opCtx,
+    const NamespaceString& nss,
+    const ShardVersion& requestedShardVersion,
+    const ScopedCollectionDescription& shardingCollectionDescription,
+    const CollectionPtr& collectionPtr);
+
 }  // namespace shard_role_details
 }  // namespace mongo