SERVER-62987 Secondary refreshes should be interrupted on replication rollback

(cherry picked from commit 2196021e412bb0ad1c470f4ba664551a9bbb56fe)

9 files changed, 41 insertions, 0 deletions

diff --git a/src/mongo/db/s/read_only_catalog_cache_loader.h b/src/mongo/db/s/read_only_catalog_cache_loader.h
index a0b10834f4c..a8e0ca5b431 100644
--- a/src/mongo/db/s/read_only_catalog_cache_loader.h
+++ b/src/mongo/db/s/read_only_catalog_cache_loader.h
@@ -46,6 +46,7 @@ public:
     void initializeReplicaSetRole(bool isPrimary) override {}
     void onStepDown() override {}
     void onStepUp() override {}
+    void onReplicationRollback() override {}
     void shutDown() override;
     void notifyOfCollectionPlacementVersionUpdate(const NamespaceString& nss) override {}
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
index 875fbc30b1a..d1a65b5f622 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
@@ -420,6 +420,13 @@ void ShardServerCatalogCacheLoader::onStepUp() {
     _role = ReplicaSetRole::Primary;
 }
 
+void ShardServerCatalogCacheLoader::onReplicationRollback() {
+    // No need to increment the term since this interruption is only to prevent the secondary
+    // refresh thread from getting stuck or waiting on an incorrect opTime.
+    stdx::lock_guard<Latch> lg(_mutex);
+    _contexts.interrupt(ErrorCodes::Interrupted);
+}
+
 void ShardServerCatalogCacheLoader::shutDown() {
     {
         stdx::lock_guard<Latch> lg(_mutex);
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.h b/src/mongo/db/s/shard_server_catalog_cache_loader.h
index 446de371751..07d18b749d3 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader.h
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader.h
@@ -72,6 +72,13 @@ public:
     void shutDown() override;
 
     /**
+     * Interrupts ongoing refreshes to prevent secondaries from waiting for opTimes from wrong terms
+     * in case of rollback. Primaries must step down before going through rollback, so this should
+     * only be run on secondaries.
+     */
+    void onReplicationRollback();
+
+    /**
      * Sets any notifications waiting for this version to arrive and invalidates the catalog cache's
      * chunk metadata for collection 'nss' so that the next caller provokes a refresh.
      */
diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp
index a6453d81ec0..bb1afdbf24f 100644
--- a/src/mongo/db/s/shard_server_op_observer.cpp
+++ b/src/mongo/db/s/shard_server_op_observer.cpp
@@ -809,6 +809,17 @@ void ShardServerOpObserver::onCollMod(OperationContext* opCtx,
 void ShardServerOpObserver::_onReplicationRollback(OperationContext* opCtx,
                                                    const RollbackObserverInfo& rbInfo) {
     ShardingRecoveryService::get(opCtx)->recoverStates(opCtx, rbInfo.rollbackNamespaces);
+
+    // If writes to config.cache.collections or config.cache.* have been rolled back, interrupt the
+    // SSCCL to ensure secondary waits for replication do not use incorrect opTimes.
+    if (std::any_of(rbInfo.rollbackNamespaces.begin(),
+                    rbInfo.rollbackNamespaces.end(),
+                    [](const NamespaceString& nss) {
+                        return nss == NamespaceString::kShardConfigCollectionsNamespace ||
+                            nss.isConfigDotCacheDotChunks();
+                    })) {
+        CatalogCacheLoader::get(opCtx).onReplicationRollback();
+    }
 }
 
 
diff --git a/src/mongo/s/catalog_cache_loader.h b/src/mongo/s/catalog_cache_loader.h
index 053e72b4175..992a906e09e 100644
--- a/src/mongo/s/catalog_cache_loader.h
+++ b/src/mongo/s/catalog_cache_loader.h
@@ -120,6 +120,11 @@ public:
     virtual void onStepUp() = 0;
 
     /**
+     * Interrupts ongoing refreshes on rollback.
+     */
+    virtual void onReplicationRollback() = 0;
+
+    /**
      * Transitions into shut down and cleans up state. Once this transitions to shut down, should
      * not be able to transition back to normal. Should be safe to be called more than once.
      */
diff --git a/src/mongo/s/catalog_cache_loader_mock.cpp b/src/mongo/s/catalog_cache_loader_mock.cpp
index 2c190ce6530..369fa7af3a7 100644
--- a/src/mongo/s/catalog_cache_loader_mock.cpp
+++ b/src/mongo/s/catalog_cache_loader_mock.cpp
@@ -61,6 +61,10 @@ void CatalogCacheLoaderMock::onStepUp() {
     MONGO_UNREACHABLE;
 }
 
+void CatalogCacheLoaderMock::onReplicationRollback() {
+    MONGO_UNREACHABLE;
+}
+
 void CatalogCacheLoaderMock::shutDown() {}
 
 void CatalogCacheLoaderMock::notifyOfCollectionPlacementVersionUpdate(const NamespaceString& nss) {
diff --git a/src/mongo/s/catalog_cache_loader_mock.h b/src/mongo/s/catalog_cache_loader_mock.h
index b846b4caf94..882d1265214 100644
--- a/src/mongo/s/catalog_cache_loader_mock.h
+++ b/src/mongo/s/catalog_cache_loader_mock.h
@@ -52,6 +52,7 @@ public:
     void initializeReplicaSetRole(bool isPrimary) override;
     void onStepDown() override;
     void onStepUp() override;
+    void onReplicationRollback() override;
     void shutDown() override;
     void notifyOfCollectionPlacementVersionUpdate(const NamespaceString& nss) override;
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;
diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp
index a9706b68a67..d9348c19766 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.cpp
+++ b/src/mongo/s/config_server_catalog_cache_loader.cpp
@@ -104,6 +104,10 @@ void ConfigServerCatalogCacheLoader::onStepUp() {
     MONGO_UNREACHABLE;
 }
 
+void ConfigServerCatalogCacheLoader::onReplicationRollback() {
+    MONGO_UNREACHABLE;
+}
+
 void ConfigServerCatalogCacheLoader::shutDown() {
     _executor->shutdown();
     _executor->join();
diff --git a/src/mongo/s/config_server_catalog_cache_loader.h b/src/mongo/s/config_server_catalog_cache_loader.h
index 04c3471e77e..bb0f84962c3 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.h
+++ b/src/mongo/s/config_server_catalog_cache_loader.h
@@ -46,6 +46,7 @@ public:
     void initializeReplicaSetRole(bool isPrimary) override;
     void onStepDown() override;
     void onStepUp() override;
+    void onReplicationRollback() override;
     void shutDown() override;
     void notifyOfCollectionPlacementVersionUpdate(const NamespaceString& nss) override;
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;