diff options
Diffstat (limited to 'src/mongo/db/storage')
23 files changed, 488 insertions, 93 deletions
diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index 144cfa386bb..616e7c09185 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -90,6 +90,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', ], ) @@ -389,6 +390,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/bson/dotted_path_support', '$BUILD_DIR/mongo/db/server_options_core', ], @@ -486,6 +488,7 @@ env.Library( env.CppUnitTest( target='db_storage_test', source=[ + 'backup_block_test.cpp', 'flow_control_test.cpp', 'historical_ident_tracker_test.cpp', 'index_entry_comparison_test.cpp', @@ -521,6 +524,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/executor/network_interface_factory', '$BUILD_DIR/mongo/executor/network_interface_mock', '$BUILD_DIR/mongo/util/periodic_runner_factory', + 'backup_block', 'flow_control', 'flow_control_parameters', 'historical_ident_tracker', @@ -645,6 +649,7 @@ env.CppLibfuzzerTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/bson/bson_validate', 'key_string', ], ) diff --git a/src/mongo/db/storage/backup_block.cpp b/src/mongo/db/storage/backup_block.cpp index 3cc2b0b0cf3..9b8dc29f209 100644 --- a/src/mongo/db/storage/backup_block.cpp +++ b/src/mongo/db/storage/backup_block.cpp @@ -49,22 +49,38 @@ const std::set<std::string> kRequiredMDBFiles = {"_mdb_catalog.wt", "sizeStorer. } // namespace +namespace details { + +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath) { + // Remove the dbpath prefix to the identAbsolutePath. + boost::filesystem::path identWithExtension = boost::filesystem::relative( + identAbsolutePath, boost::filesystem::path(storageGlobalParams.dbpath)); + + // Remove the file extension and convert to generic form (i.e. replace "\" with "/" + // on windows, no-op on unix). + return boost::filesystem::change_extension(identWithExtension, "").generic_string(); +} + +} // namespace details + BackupBlock::BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional<Timestamp> checkpointTimestamp, std::uint64_t offset, std::uint64_t length, std::uint64_t fileSize) - : _filePath(filePath), _offset(offset), _length(length), _fileSize(fileSize) { - boost::filesystem::path path(filePath); - _filenameStem = path.stem().string(); + : _fileAbsolutePath(fileAbsolutePath), _offset(offset), _length(length), _fileSize(fileSize) { + boost::filesystem::path absolutePath(fileAbsolutePath); + _ident = details::extractIdentFromPath(boost::filesystem::path(storageGlobalParams.dbpath), + absolutePath); _initialize(opCtx, identToNamespaceAndUUIDMap, checkpointTimestamp); } bool BackupBlock::isRequired() const { // Extract the filename from the path. - boost::filesystem::path path(_filePath); + boost::filesystem::path path(_fileAbsolutePath); const std::string filename = path.filename().string(); // Check whether this is a required WiredTiger file. @@ -121,7 +137,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, } // Fetch the latest values for the ident. - auto it = identToNamespaceAndUUIDMap.find(_filenameStem); + auto it = identToNamespaceAndUUIDMap.find(_ident); if (it != identToNamespaceAndUUIDMap.end()) { _uuid = it->second.second; _setNamespaceString(it->second.first); @@ -134,7 +150,7 @@ void BackupBlock::_initialize(OperationContext* opCtx, // Check if the ident had a different value at the checkpoint timestamp. If so, we want to use // that instead as that will be the ident's value when restoring from the backup. boost::optional<std::pair<NamespaceString, UUID>> historicalEntry = - HistoricalIdentTracker::get(opCtx).lookup(_filenameStem, checkpointTimestamp.get()); + HistoricalIdentTracker::get(opCtx).lookup(_ident, checkpointTimestamp.value()); if (historicalEntry) { _uuid = historicalEntry->second; _setNamespaceString(historicalEntry->first); diff --git a/src/mongo/db/storage/backup_block.h b/src/mongo/db/storage/backup_block.h index fd96b8e5e2e..30a640190a6 100644 --- a/src/mongo/db/storage/backup_block.h +++ b/src/mongo/db/storage/backup_block.h @@ -39,6 +39,11 @@ namespace mongo { +namespace details { +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); +} + /** * Represents the file blocks returned by the storage engine during both full and incremental * backups. In the case of a full backup, each block is an entire file with offset=0 and @@ -57,7 +62,7 @@ public: stdx::unordered_map<std::string, std::pair<NamespaceString, UUID>>; explicit BackupBlock(OperationContext* opCtx, - std::string filePath, + std::string fileAbsolutePath, const IdentToNamespaceAndUUIDMap& identToNamespaceAndUUIDMap, boost::optional<Timestamp> checkpointTimestamp, std::uint64_t offset = 0, @@ -67,7 +72,7 @@ public: ~BackupBlock() = default; std::string filePath() const { - return _filePath; + return _fileAbsolutePath; } std::string ns() const { @@ -110,12 +115,12 @@ private: boost::optional<Timestamp> checkpointTimestamp); void _setNamespaceString(const NamespaceString& nss); - const std::string _filePath; + const std::string _fileAbsolutePath; const std::uint64_t _offset; const std::uint64_t _length; const std::uint64_t _fileSize; - std::string _filenameStem; + std::string _ident; NamespaceString _nss; boost::optional<UUID> _uuid; }; diff --git a/src/mongo/db/storage/backup_block_test.cpp b/src/mongo/db/storage/backup_block_test.cpp new file mode 100644 index 00000000000..d98a1e63650 --- /dev/null +++ b/src/mongo/db/storage/backup_block_test.cpp @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2023-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include <boost/filesystem/path.hpp> + +#include "mongo/db/storage/backup_block.h" +#include "mongo/unittest/unittest.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest + + +namespace mongo { +namespace details { +namespace { + +TEST(BackupBlockTest, ExtractIdentFromPath) { + boost::filesystem::path dbpath = "/data/db"; + boost::filesystem::path identAbsolutePathDefault = + "/data/db/collection-9-11733751379908443489.wt"; + std::string identDefault = "collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDefault), identDefault); + + boost::filesystem::path identAbsolutePathDirectoryPerDb = + "/data/db/test/collection-9-11733751379908443489.wt"; + std::string identDirectoryPerDb = "test/collection-9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathDirectoryPerDb), + identDirectoryPerDb); + + boost::filesystem::path identAbsolutePathWiredTigerDirectoryForIndexes = + "/data/db/collection/9-11733751379908443489.wt"; + std::string identWiredTigerDirectoryForIndexes = "collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath(dbpath, identAbsolutePathWiredTigerDirectoryForIndexes), + identWiredTigerDirectoryForIndexes); + + boost::filesystem::path identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes = + "/data/db/test/collection/9-11733751379908443489.wt"; + std::string identDirectoryPerDbWiredTigerDirectoryForIndexes = + "test/collection/9-11733751379908443489"; + + ASSERT_EQ(details::extractIdentFromPath( + dbpath, identAbsolutePathDirectoryPerDbAndWiredTigerDirectoryForIndexes), + identDirectoryPerDbWiredTigerDirectoryForIndexes); +} + +} // namespace +} // namespace details +} // namespace mongo diff --git a/src/mongo/db/storage/control/storage_control.cpp b/src/mongo/db/storage/control/storage_control.cpp index 11222448eba..3f5af49c729 100644 --- a/src/mongo/db/storage/control/storage_control.cpp +++ b/src/mongo/db/storage/control/storage_control.cpp @@ -79,8 +79,8 @@ void startStorageControls(ServiceContext* serviceContext, bool forTestOnly) { std::unique_ptr<JournalFlusher> journalFlusher = std::make_unique<JournalFlusher>( /*disablePeriodicFlushes*/ forTestOnly || (!storageEngine->isDurable() && !storageEngine->isEphemeral())); - journalFlusher->go(); JournalFlusher::set(serviceContext, std::move(journalFlusher)); + JournalFlusher::get(serviceContext)->go(); } if (!storageEngine->isEphemeral() && !storageGlobalParams.readOnly) { diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript index 67f6904300a..8e53d62525f 100644 --- a/src/mongo/db/storage/kv/SConscript +++ b/src/mongo/db/storage/kv/SConscript @@ -11,7 +11,7 @@ env.Library( LIBDEPS=[ '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/write_unit_of_work', ], ) diff --git a/src/mongo/db/storage/kv/storage_engine_test.cpp b/src/mongo/db/storage/kv/storage_engine_test.cpp index fe85863c7b5..0a7fb957003 100644 --- a/src/mongo/db/storage/kv/storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/storage_engine_test.cpp @@ -649,15 +649,17 @@ TEST_F(TimestampKVEngineTest, TimestampMonitorNotifiesListeners) { _storageEngine->getTimestampMonitor()->addListener(&fourth); // Wait until all 4 listeners get notified at least once. - stdx::unique_lock<Latch> lk(mutex); - cv.wait(lk, [&] { - for (auto const& change : changes) { - if (!change) { - return false; + { + stdx::unique_lock<Latch> lk(mutex); + cv.wait(lk, [&] { + for (auto const& change : changes) { + if (!change) { + return false; + } } - } - return true; - }); + return true; + }); + }; _storageEngine->getTimestampMonitor()->clearListeners(); } diff --git a/src/mongo/db/storage/record_store.h b/src/mongo/db/storage/record_store.h index 1742cddba5e..ccbc8d582b9 100644 --- a/src/mongo/db/storage/record_store.h +++ b/src/mongo/db/storage/record_store.h @@ -477,15 +477,6 @@ public: } /** - * If compact() supports online compaction. - * - * Only called if compactSupported() returns true. - */ - virtual bool supportsOnlineCompaction() const { - MONGO_UNREACHABLE; - } - - /** * Attempt to reduce the storage space used by this RecordStore. * * Only called if compactSupported() returns true. diff --git a/src/mongo/db/storage/recovery_unit.h b/src/mongo/db/storage/recovery_unit.h index eb032f73d4e..7b867906535 100644 --- a/src/mongo/db/storage/recovery_unit.h +++ b/src/mongo/db/storage/recovery_unit.h @@ -77,6 +77,22 @@ enum class PrepareConflictBehavior { }; /** + * DataCorruptionDetectionMode determines how we handle the discovery of evidence of data + * corruption. + */ +enum class DataCorruptionDetectionMode { + /** + * Always throw a DataCorruptionDetected error when evidence of data corruption is detected. + */ + kThrow, + /** + * When evidence of data corruption is decected, log an entry to the health log and the server + * logs, but do not throw an error. Continue attempting to return results. + */ + kLogAndContinue, +}; + +/** * A RecoveryUnit is responsible for ensuring that data is persisted. * All on-disk information must be mutated through this interface. */ @@ -711,6 +727,14 @@ public: return _noEvictionAfterRollback; } + void setDataCorruptionDetectionMode(DataCorruptionDetectionMode mode) { + _dataCorruptionDetectionMode = mode; + } + + DataCorruptionDetectionMode getDataCorruptionDetectionMode() const { + return _dataCorruptionDetectionMode; + } + /** * Returns true if this is an instance of RecoveryUnitNoop. */ @@ -718,6 +742,13 @@ public: return false; } + /** + * Sets a maximum timeout that the storage engine will block an operation when the cache is + * under pressure. + * If not set (default 0) then the storage engine will block indefinitely. + */ + virtual void setCacheMaxWaitTimeout(Milliseconds) {} + protected: RecoveryUnit(); @@ -770,6 +801,8 @@ protected: AbandonSnapshotMode _abandonSnapshotMode = AbandonSnapshotMode::kAbort; + DataCorruptionDetectionMode _dataCorruptionDetectionMode = DataCorruptionDetectionMode::kThrow; + private: // Sets the snapshot associated with this RecoveryUnit to a new globally unique id number. void assignNextSnapshotId(); diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 26a7fc9dae0..21f04d4445a 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -41,6 +41,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/multitenancy.h" #include "mongo/db/operation_context.h" @@ -1235,12 +1236,12 @@ void StorageEngineImpl::TimestampMonitor::_startup() { } try { - auto opCtx = client->getOperationContext(); - mongo::ServiceContext::UniqueOperationContext uOpCtx; - if (!opCtx) { - uOpCtx = client->makeOperationContext(); - opCtx = uOpCtx.get(); - } + auto uniqueOpCtx = client->makeOperationContext(); + auto opCtx = uniqueOpCtx.get(); + + // The TimestampMonitor is an important background cleanup task for the storage + // engine and needs to be able to make progress to free up resources. + SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx); Timestamp checkpoint; Timestamp oldest; diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 8ce4c135cfc..1b7790e61c6 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -53,11 +53,11 @@ wtEnv.Library( '$BUILD_DIR/mongo/db/catalog/collection', '$BUILD_DIR/mongo/db/catalog/collection_options', '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/global_settings', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/prepare_conflict_tracker', + '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/repl_settings', @@ -80,6 +80,7 @@ wtEnv.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/database_holder', + '$BUILD_DIR/mongo/db/catalog/health_log_interface', '$BUILD_DIR/mongo/db/commands/server_status', '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/db_raii', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 09b1603361e..0a498ead248 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -38,12 +38,16 @@ #include <set> #include "mongo/base/checked_cast.h" +#include "mongo/base/string_data.h" +#include "mongo/db/catalog/health_log.h" +#include "mongo/db/catalog/health_log_gen.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/catalog/validate_results.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/global_settings.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/json.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/service_context.h" #include "mongo/db/stats/resource_consumption_metrics.h" @@ -59,6 +63,7 @@ #include "mongo/util/assert_util.h" #include "mongo/util/fail_point.h" #include "mongo/util/hex.h" +#include "mongo/util/stacktrace.h" #include "mongo/util/str.h" #include "mongo/util/testing_proctor.h" @@ -84,8 +89,53 @@ namespace { MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY); MONGO_FAIL_POINT_DEFINE(WTIndexPauseAfterSearchNear); MONGO_FAIL_POINT_DEFINE(WTValidateIndexStructuralDamage); +MONGO_FAIL_POINT_DEFINE(WTIndexUassertDuplicateRecordForKeyOnIdUnindex); static const WiredTigerItem emptyItem(nullptr, 0); + +/** + * Add a data corruption entry to the health log. + */ +void addDataCorruptionEntryToHealthLog(OperationContext* opCtx, + const NamespaceString& nss, + StringData operation, + StringData message, + const BSONObj& key, + StringData indexName, + StringData uri) { + HealthLogEntry entry; + entry.setNss(nss); + entry.setTimestamp(Date_t::now()); + entry.setSeverity(SeverityEnum::Error); + entry.setScope(ScopeEnum::Index); + entry.setOperation(operation); + entry.setMsg(message); + + BSONObjBuilder bob; + bob.append("key", key); + bob.append("indexName", indexName); + bob.append("uri", uri); + bob.appendElements(getStackTrace().getBSONRepresentation()); + entry.setData(bob.obj()); + + HealthLog::get(opCtx)->log(entry); +} + +/** + * Returns the logv2::LogOptions controlling the behaviour after logging a data corruption + * error. When the TestingProctor is enabled we will fatally assert. When the testing proctor is + * disabled or when 'forceUassert' is specified (for instance because a failpoint is enabled), + * we should log and throw DataCorruptionDetected. + */ +logv2::LogOptions getLogOptionsForDataCorruption(RecoveryUnit& ru, bool forceUassert = false) { + if (ru.getDataCorruptionDetectionMode() == DataCorruptionDetectionMode::kThrow || + MONGO_unlikely(forceUassert)) { + return logv2::LogOptions{logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}; + } else { + return logv2::LogOptions(logv2::LogComponent::kAutomaticDetermination); + } +} + } // namespace void WiredTigerIndex::setKey(WT_CURSOR* cursor, const WT_ITEM* item) { @@ -240,7 +290,7 @@ WiredTigerIndex::WiredTigerIndex(OperationContext* ctx, bool isLogged, bool isReadOnly) : SortedDataInterface(ident, - _handleVersionInfo(ctx, uri, desc, isLogged, isReadOnly), + _handleVersionInfo(ctx, uri, ident, desc, isLogged, isReadOnly), Ordering::make(desc->keyPattern()), rsKeyFormat), _uri(uri), @@ -711,8 +761,45 @@ StatusWith<bool> WiredTigerIndex::_checkDups(OperationContext* opCtx, _collation); } +void WiredTigerIndex::_repairDataFormatVersion(OperationContext* opCtx, + const std::string& uri, + StringData ident, + const IndexDescriptor* desc) { + auto indexVersion = desc->version(); + auto isIndexVersion1 = indexVersion == IndexDescriptor::IndexVersion::kV1; + auto isIndexVersion2 = indexVersion == IndexDescriptor::IndexVersion::kV2; + auto isDataFormat6 = _dataFormatVersion == kDataFormatV1KeyStringV0IndexVersionV1; + auto isDataFormat8 = _dataFormatVersion == kDataFormatV2KeyStringV1IndexVersionV2; + auto isDataFormat13 = _dataFormatVersion == kDataFormatV5KeyStringV0UniqueIndexVersionV1; + auto isDataFormat14 = _dataFormatVersion == kDataFormatV6KeyStringV1UniqueIndexVersionV2; + // Only fixes the index data format when it could be from an edge case when converting the + // uniqueness of the index. Specifically: + // * The index is a secondary unique index, but the data format version is 6 (v1) or 8 (v2). + // * The index is a non-unique index, but the data format version is 13 (v1) or 14 (v2). + if ((!desc->isIdIndex() && desc->unique() && + ((isIndexVersion1 && isDataFormat6) || (isIndexVersion2 && isDataFormat8))) || + (!desc->unique() && + ((isIndexVersion1 && isDataFormat13) || (isIndexVersion2 && isDataFormat14)))) { + auto engine = opCtx->getServiceContext()->getStorageEngine(); + engine->getEngine()->alterIdentMetadata( + opCtx, ident, desc, /* isForceUpdateMetadata */ false); + auto prevVersion = _dataFormatVersion; + // The updated data format is guaranteed to be within the supported version range. + _dataFormatVersion = WiredTigerUtil::checkApplicationMetadataFormatVersion( + opCtx, uri, kMinimumIndexVersion, kMaximumIndexVersion) + .getValue(); + LOGV2_WARNING(6818600, + "Fixing index metadata data format version", + "namespace"_attr = desc->getEntry()->getNSSFromCatalog(opCtx), + "indexName"_attr = desc->indexName(), + "prevVersion"_attr = prevVersion, + "newVersion"_attr = _dataFormatVersion); + } +} + KeyString::Version WiredTigerIndex::_handleVersionInfo(OperationContext* ctx, const std::string& uri, + StringData ident, const IndexDescriptor* desc, bool isLogged, bool isReadOnly) { @@ -730,6 +817,8 @@ KeyString::Version WiredTigerIndex::_handleVersionInfo(OperationContext* ctx, } _dataFormatVersion = version.getValue(); + _repairDataFormatVersion(ctx, uri, ident, desc); + if (!desc->isIdIndex() && desc->unique() && (_dataFormatVersion < kDataFormatV3KeyStringV0UniqueIndexVersionV1 || _dataFormatVersion > kDataFormatV6KeyStringV1UniqueIndexVersionV2)) { @@ -1510,14 +1599,24 @@ private: _typeBits.resetFromBuffer(&br); if (!br.atEof()) { - LOGV2_FATAL(28608, - "Unique index cursor seeing multiple records for key {key} in index " - "{index} ({uri}) belonging to collection {collection}", - "Unique index cursor seeing multiple records for key in index", - "key"_attr = redact(curr(kWantKey)->key), - "index"_attr = _idx.indexName(), - "uri"_attr = _idx.uri(), - "collection"_attr = _idx.getCollectionNamespace(_opCtx)); + const auto bsonKey = redact(curr(kWantKey)->key); + const auto collectionNamespace = _idx.getCollectionNamespace(_opCtx); + addDataCorruptionEntryToHealthLog( + _opCtx, + collectionNamespace, + "WiredTigerIndexUniqueCursor::_updateIdAndTypeBitsFromValue", + "Unique index cursor seeing multiple records for key in index", + bsonKey, + _idx.indexName(), + _idx.uri()); + + LOGV2_ERROR_OPTIONS(7623202, + getLogOptionsForDataCorruption(*_opCtx->recoveryUnit()), + "Unique index cursor seeing multiple records for key in index", + "key"_attr = bsonKey, + "index"_attr = _idx.indexName(), + "uri"_attr = _idx.uri(), + logAttrs(collectionNamespace)); } } }; @@ -1545,12 +1644,25 @@ public: _typeBits.resetFromBuffer(&br); if (!br.atEof()) { - LOGV2_FATAL(5176200, - "Index cursor seeing multiple records for key in _id index", - "key"_attr = redact(curr(kWantKey)->key), - "index"_attr = _idx.indexName(), - "uri"_attr = _idx.uri(), - "collection"_attr = _idx.getCollectionNamespace(_opCtx)); + const auto bsonKey = redact(curr(kWantKey)->key); + const auto collectionNamespace = _idx.getCollectionNamespace(_opCtx); + + addDataCorruptionEntryToHealthLog( + _opCtx, + collectionNamespace, + "WiredTigerIdIndexCursor::updateIdAndTypeBits", + "Index cursor seeing multiple records for key in _id index", + bsonKey, + _idx.indexName(), + _idx.uri()); + + LOGV2_ERROR_OPTIONS(5176200, + getLogOptionsForDataCorruption(*_opCtx->recoveryUnit()), + "Index cursor seeing multiple records for key in _id index", + "key"_attr = bsonKey, + "index"_attr = _idx.indexName(), + "uri"_attr = _idx.uri(), + logAttrs(collectionNamespace)); } } }; @@ -1773,9 +1885,11 @@ void WiredTigerIdIndex::_unindex(OperationContext* opCtx, WiredTigerItem keyItem(keyString.getBuffer(), sizeWithoutRecordId); setKey(c, keyItem.Get()); + const auto failWithDataCorruptionForTest = + WTIndexUassertDuplicateRecordForKeyOnIdUnindex.shouldFail(); // On the _id index, the RecordId is stored in the value of the index entry. If the dupsAllowed // flag is not set, we blindly delete using only the key without checking the RecordId. - if (!dupsAllowed) { + if (!dupsAllowed && MONGO_likely(!failWithDataCorruptionForTest)) { int ret = WT_OP_CHECK(wiredTigerCursorRemove(opCtx, c)); if (ret == WT_NOTFOUND) { return; @@ -1807,14 +1921,26 @@ void WiredTigerIdIndex::_unindex(OperationContext* opCtx, RecordId idInIndex = KeyString::decodeRecordIdLong(&br); KeyString::TypeBits typeBits = KeyString::TypeBits::fromBuffer(getKeyStringVersion(), &br); - if (!br.atEof()) { + if (!br.atEof() || MONGO_unlikely(failWithDataCorruptionForTest)) { auto bsonKey = KeyString::toBson(keyString, _ordering); - LOGV2_FATAL(5176201, - "Un-index seeing multiple records for key", - "key"_attr = bsonKey, - "index"_attr = _desc->indexName(), - "uri"_attr = _uri, - "collection"_attr = getCollectionNamespace(opCtx)); + const auto collectionNamespace = getCollectionNamespace(opCtx); + + addDataCorruptionEntryToHealthLog(opCtx, + collectionNamespace, + "WiredTigerIdIndex::_unindex", + "Un-index seeing multiple records for key", + bsonKey, + _indexName, + _uri); + + LOGV2_ERROR_OPTIONS( + 5176201, + getLogOptionsForDataCorruption(*opCtx->recoveryUnit(), failWithDataCorruptionForTest), + "Un-index seeing multiple records for key", + "key"_attr = bsonKey, + "index"_attr = _indexName, + "uri"_attr = _uri, + logAttrs(collectionNamespace)); } // The RecordId matches, so remove the entry. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 8fccc7d8c24..faea4ca6cac 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -234,10 +234,20 @@ protected: */ KeyString::Version _handleVersionInfo(OperationContext* ctx, const std::string& uri, + StringData ident, const IndexDescriptor* desc, bool isLogged, bool isReadOnly); + /* + * Attempts to repair the data format version in the index table metadata if there is a mismatch + * to the index type during startup. + */ + void _repairDataFormatVersion(OperationContext* opCtx, + const std::string& uri, + StringData ident, + const IndexDescriptor* desc); + RecordId _decodeRecordIdAtEnd(const void* buffer, size_t size); class BulkBuilder; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 97bf1fc8557..0de1c43284f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1173,11 +1173,22 @@ private: int wtRet; bool fileUnchangedFlag = false; if (!_wtBackup->dupCursor) { - wtRet = (_session)->open_cursor( - _session, nullptr, _wtBackup->cursor, config.c_str(), &_wtBackup->dupCursor); - if (wtRet != 0) { - return wtRCToStatus(wtRet, _session); - } + size_t attempt = 0; + do { + wtRet = _session->open_cursor( + _session, nullptr, _wtBackup->cursor, config.c_str(), &_wtBackup->dupCursor); + + if (wtRet == EBUSY) { + logAndBackoff(8927900, + ::mongo::logv2::LogComponent::kStorage, + logv2::LogSeverity::Debug(1), + ++attempt, + "Opening duplicate backup cursor returned EBUSY, retrying", + "config"_attr = config); + } else if (wtRet != 0) { + return wtRCToStatus(wtRet, _session); + } + } while (wtRet == EBUSY); fileUnchangedFlag = true; } @@ -1310,12 +1321,15 @@ WiredTigerKVEngine::beginNonBlockingBackup(OperationContext* opCtx, for (const DurableCatalog::Entry& e : catalogEntries) { // Populate the collection ident with its namespace and UUID. UUID uuid = catalog->getMetaData(opCtx, e.catalogId)->options.uuid.get(); - _wtBackup.identToNamespaceAndUUIDMap.emplace(e.ident, std::make_pair(e.nss, uuid)); + std::string collectionIdent = e.ident; + _wtBackup.identToNamespaceAndUUIDMap.emplace(collectionIdent, + std::make_pair(e.nss, uuid)); // Populate the collection's index idents with the collection's namespace and UUID. std::vector<std::string> idxIdents = catalog->getIndexIdents(opCtx, e.catalogId); - for (const std::string& idxIdent : idxIdents) { - _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdent, std::make_pair(e.nss, uuid)); + for (const std::string& idxIdentFull : idxIdents) { + _wtBackup.identToNamespaceAndUUIDMap.emplace(idxIdentFull, + std::make_pair(e.nss, uuid)); } } } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 24af566c560..891c9c91bea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -60,6 +60,26 @@ class WiredTigerSessionCache; class WiredTigerSizeStorer; class WiredTigerEngineRuntimeConfigParameter; +/** + * With the absolute path to an ident and the parent dbpath, return the ident. + * + * Note that the ident can have 4 different forms depending on the combination + * of server parameters present (directoryperdb / wiredTigerDirectoryForIndexes). + * With any one of these server parameters enabled, a directory could be included + * in the returned ident. + * See the unit test WiredTigerKVEngineTest::ExtractIdentFromPath for example usage. + * + * Note (2) idents use unix-style separators (always, see + * durable_catalog.cpp:generateUniqueIdent) but ident paths are platform-dependant. + * This method returns the unix-style "/" separators always. + */ +std::string extractIdentFromPath(const boost::filesystem::path& dbpath, + const boost::filesystem::path& identAbsolutePath); + + +Status validateExtraDiagnostics(const std::vector<std::string>& value, + const boost::optional<TenantId>& tenantId); + struct WiredTigerFileVersion { // MongoDB 4.4+ will not open on datafiles left behind by 4.2.5 and earlier. MongoDB 4.4 // shutting down in FCV 4.2 will leave data files that 4.2.6+ will understand diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index ba8861d15a6..547753bebf9 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -677,6 +677,15 @@ public: // On destruction, we must always handle freeing the underlying raw WT_CURSOR pointer. _saveStorageCursorOnDetachFromOperationContext = false; + // Shutdown does not wait for any threads running queries to be interrupted and exit. + // In addition, the RandomCursor destructor doesn't hold any global lock so we need to + // check if the server is shutting down to avoid calling into the storage engine, whose + // connection may have already been closed. + Status interruptStatus = _opCtx->checkForInterruptNoAssert(); + if (interruptStatus.code() == ErrorCodes::InterruptedAtShutdown) { + return; + } + detachFromOperationContext(); } } @@ -1769,6 +1778,8 @@ Status WiredTigerRecordStore::doCompact(OperationContext* opCtx) { dassert(opCtx->lockState()->isWriteLocked()); WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache(); + const std::string uri(getURI()); + if (!cache->isEphemeral()) { WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(); opCtx->recoveryUnit()->abandonSnapshot(); @@ -1779,7 +1790,7 @@ Status WiredTigerRecordStore::doCompact(OperationContext* opCtx) { if (ret == EBUSY) { return Status(ErrorCodes::Interrupted, - str::stream() << "Compaction interrupted on " << getURI().c_str() + str::stream() << "Compaction interrupted on " << uri << " due to cache eviction pressure"); } invariantWTOK(ret, s); @@ -2232,9 +2243,18 @@ boost::optional<Record> WiredTigerRecordStoreCursorBase::next() { invariant(!TestingProctor::instance().isEnabled(), "cursor returned out-of-order keys"); } - // uassert with 'DataCorruptionDetected' after logging. + auto options = [&] { + if (_opCtx->recoveryUnit()->getDataCorruptionDetectionMode() == + DataCorruptionDetectionMode::kThrow) { + // uassert with 'DataCorruptionDetected' after logging. + return logv2::LogOptions{ + logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}; + } else { + return logv2::LogOptions(logv2::LogComponent::kAutomaticDetermination); + } + }(); LOGV2_ERROR_OPTIONS(22406, - {logv2::UserAssertAfterLog(ErrorCodes::DataCorruptionDetected)}, + options, "WT_Cursor::next -- returned out-of-order keys", "forward"_attr = _forward, "next"_attr = id, diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index 119b907f7a7..1afb4919fef 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -183,9 +183,6 @@ public: virtual bool compactSupported() const { return !_isEphemeral; } - virtual bool supportsOnlineCompaction() const { - return true; - } virtual Timestamp getPinnedOplog() const final; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index 3da6bb97782..f8d9c410ef3 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -91,6 +91,13 @@ WiredTigerRecoveryUnit::WiredTigerRecoveryUnit(WiredTigerSessionCache* sc, WiredTigerRecoveryUnit::~WiredTigerRecoveryUnit() { invariant(!_inUnitOfWork(), toString(_getState())); _abort(); + + // If the session has non zero timeout then reset it back to 0 before returning the session back + // to the cache. + if (durationCount<Milliseconds>(_cacheMaxWaitTimeout)) { + auto wtSession = getSessionNoTxn()->getSession(); + invariantWTOK(wtSession->reconfigure(wtSession, "cache_max_wait_ms=0"), wtSession); + } } void WiredTigerRecoveryUnit::_commit() { @@ -421,6 +428,12 @@ void WiredTigerRecoveryUnit::_txnClose(bool commit) { _isOplogReader = false; _oplogVisibleTs = boost::none; _orderedCommit = true; // Default value is true; we assume all writes are ordered. + // Reset the kLastApplied read source back to the default of kNoTimestamp. Any reader requiring + // kLastApplied will set the read source again before reading. Resetting this read source + // simplifies the handling when stepup happens concurrently with read operations. + if (_timestampReadSource == ReadSource::kLastApplied) { + _timestampReadSource = ReadSource::kNoTimestamp; + } } Status WiredTigerRecoveryUnit::majorityCommittedSnapshotAvailable() const { @@ -598,11 +611,12 @@ void WiredTigerRecoveryUnit::_beginTransactionAtLastAppliedTimestamp(WT_SESSION* RoundUpReadTimestamp::kRound); auto status = txnOpen.setReadSnapshot(_readAtTimestamp); fassert(4847501, status); - txnOpen.done(); // We might have rounded to oldest between calling getLastApplied and setReadSnapshot. We // need to get the actual read timestamp we used. - _readAtTimestamp = _getTransactionReadTimestamp(session); + auto actualTimestamp = _getTransactionReadTimestamp(session); + txnOpen.done(); + _readAtTimestamp = actualTimestamp; } Timestamp WiredTigerRecoveryUnit::_beginTransactionAtNoOverlapTimestamp(WT_SESSION* session) { @@ -659,11 +673,11 @@ Timestamp WiredTigerRecoveryUnit::_beginTransactionAtNoOverlapTimestamp(WT_SESSI RoundUpReadTimestamp::kRound); auto status = txnOpen.setReadSnapshot(readTimestamp); fassert(51066, status); - txnOpen.done(); // We might have rounded to oldest between calling getAllDurable and setReadSnapshot. We // need to get the actual read timestamp we used. readTimestamp = _getTransactionReadTimestamp(session); + txnOpen.done(); return readTimestamp; } @@ -925,4 +939,15 @@ void WiredTigerRecoveryUnit::storeWriteContextForDebugging(const BSONObj& info) _writeContextForDebugging.push_back(info); } +void WiredTigerRecoveryUnit::setCacheMaxWaitTimeout(Milliseconds timeout) { + _cacheMaxWaitTimeout = timeout; + + auto wtSession = getSessionNoTxn()->getSession(); + invariantWTOK( + wtSession->reconfigure( + wtSession, + fmt::format("cache_max_wait_ms={}", durationCount<Milliseconds>(_cacheMaxWaitTimeout)) + .c_str()), + wtSession); +} } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h index e0b26282779..f1f692043b4 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h @@ -147,6 +147,8 @@ public: _multiTimestampConstraintTracker.ignoreAllMultiTimestampConstraints = true; } + void setCacheMaxWaitTimeout(Milliseconds) override; + // ---- WT STUFF WiredTigerSession* getSession(); @@ -283,6 +285,8 @@ private: std::vector<BSONObj> _writeContextForDebugging; WiredTigerStats _sessionStatsAfterLastOperation; + + Milliseconds _cacheMaxWaitTimeout{0}; }; } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 26e9824cc57..19e749bb445 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -461,7 +461,7 @@ bool WiredTigerSessionCache::isEphemeral() { UniqueWiredTigerSession WiredTigerSessionCache::getSession() { // We should never be able to get here after _shuttingDown is set, because no new // operations should be allowed to start. - invariant(!(_shuttingDown.loadRelaxed() & kShuttingDownMask)); + invariant(!(_shuttingDown.load() & kShuttingDownMask)); { stdx::lock_guard<Latch> lock(_cacheLock); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp index 942d6ce8592..de55e539056 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_stats_test.cpp @@ -27,7 +27,12 @@ * it in the license file. */ +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kWiredTiger + #include "mongo/db/storage/wiredtiger/wiredtiger_stats.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/logv2/log.h" +#include "mongo/unittest/log_test.h" #include "mongo/unittest/temp_dir.h" #include "mongo/unittest/unittest.h" #include <memory> @@ -164,10 +169,23 @@ protected: }; TEST_F(WiredTigerStatsTest, EmptySession) { + // Increase log component verbosity for WiredTiger + auto verbosityGuard = unittest::MinimumLoggedSeverityGuard{logv2::LogComponent::kWiredTiger, + logv2::LogSeverity::Debug(5)}; + auto verboseConfig = WiredTigerUtil::generateWTVerboseConfiguration(); + ASSERT_OK(wtRCToStatus(_conn->reconfigure(_conn, verboseConfig.c_str()), nullptr)); + // Read and write statistics should be empty. Check "data" field does not exist. "wait" fields // such as the schemaLock might have some value. auto statsBson = WiredTigerStats{_session}.toBSON(); - ASSERT_FALSE(statsBson.hasField("data")); + + { + BSONObjBuilder bob; + ASSERT_OK(WiredTigerUtil::exportTableToBSON(_session, "statistics:", "", &bob)); + LOGV2(9032000, "Connection statistics", "stats"_attr = bob.obj()); + } + + ASSERT_FALSE(statsBson.hasField("data")) << statsBson; } TEST_F(WiredTigerStatsTest, SessionWithWrite) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 7fa7416d58f..1afdb79babc 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -602,9 +602,12 @@ logv2::LogSeverity getWTLOGV2SeverityLevel(const BSONObj& obj) { return logv2::LogSeverity::Info(); case WT_VERBOSE_INFO: return logv2::LogSeverity::Log(); - case WT_VERBOSE_DEBUG: - return logv2::LogSeverity::Debug(1); default: + // MongoDB enables some WT debug compnonents by default. If performed a 1:1 + // translation from WT log severity levels, MongoDB would not log anything + // below default level Log, even if a Debug message came through the message + // handler. To solve this, we upgrade all Debug messages to the Log level + // to ensure they are seen. return logv2::LogSeverity::Log(); } } @@ -1191,6 +1194,30 @@ std::string WiredTigerUtil::generateWTVerboseConfiguration() { return cfg; } +// static +boost::optional<std::string> WiredTigerUtil::getConfigStringFromStorageOptions( + const BSONObj& options) { + if (auto wtElem = options[kWiredTigerEngineName]) { + BSONObj wtObj = wtElem.Obj(); + if (auto configStringElem = wtObj.getField(kConfigStringField)) { + return configStringElem.String(); + } + } + + return boost::none; +} + +// static +BSONObj WiredTigerUtil::setConfigStringToStorageOptions(const BSONObj& options, + const std::string& configString) { + // Storage options may contain settings for non-WiredTiger storage engines (e.g. inMemory). + // We should leave these settings intact. + auto wtElem = options[kWiredTigerEngineName]; + auto wtObj = wtElem ? wtElem.Obj() : BSONObj(); + return options.addFields( + BSON(kWiredTigerEngineName << wtObj.addFields(BSON(kConfigStringField << configString)))); +} + void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) { static const StaticImmortal<pcrecpp::RE> encryptionOptsRegex(R"re(encryption=\([^\)]*\),?)re"); encryptionOptsRegex->GlobalReplace("", configString); @@ -1198,20 +1225,13 @@ void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) // static BSONObj WiredTigerUtil::getSanitizedStorageOptionsForSecondaryReplication(const BSONObj& options) { - // Storage options may contain settings for non-WiredTiger storage engines (e.g. inMemory). - // We should leave these settings intact. - if (auto wtElem = options[kWiredTigerEngineName]) { - BSONObj wtObj = wtElem.Obj(); - if (auto configStringElem = wtObj.getField(kConfigStringField)) { - auto configString = configStringElem.String(); - removeEncryptionFromConfigString(&configString); - // Return a new BSONObj with the configString field sanitized. - return options.addFields(BSON(kWiredTigerEngineName << wtObj.addFields( - BSON(kConfigStringField << configString)))); - } + auto configString = getConfigStringFromStorageOptions(options); + if (!configString) { + return options; } - return options; + removeEncryptionFromConfigString(configString.get_ptr()); + return setConfigStringToStorageOptions(options, *configString); } } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 57f8113e3c9..a70c050b2ea 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -335,6 +335,17 @@ public: static T castStatisticsValue(uint64_t statisticsValue); /** + * Gets the WiredTiger configuration string from storage engine collection options. + */ + static boost::optional<std::string> getConfigStringFromStorageOptions(const BSONObj& options); + + /** + * Sets the WiredTiger configuration string to storage engine collection options. + */ + static BSONObj setConfigStringToStorageOptions(const BSONObj& options, + const std::string& configString); + + /** * Removes encryption configuration from a config string. Should only be applied on custom * config strings on secondaries. Fixes an issue where encryption configuration might be * replicated to non-encrypted nodes, or nodes with different encryption options, causing |