diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
| commit | 4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch) | |
| tree | 1682a647d4463397c119183369ae6f750d5fdcff /src/mongo/db/catalog | |
| parent | aa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff) | |
| parent | 8f0827553e09872941945a093b647a4211a9db7f (diff) | |
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'src/mongo/db/catalog')
84 files changed, 1227 insertions, 3918 deletions
diff --git a/src/mongo/db/catalog/README.md b/src/mongo/db/catalog/README.md index 5fe1a00de68..ddb5d479da2 100644 --- a/src/mongo/db/catalog/README.md +++ b/src/mongo/db/catalog/README.md @@ -432,30 +432,6 @@ See [wtRcToStatus](https://github.com/mongodb/mongo/blob/c799851554dc01493d35b43701416e9c78b3665c/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp#L178-L183) where we throw the exception in WiredTiger. See [TemporarilyUnavailableException](https://github.com/mongodb/mongo/blob/c799851554dc01493d35b43701416e9c78b3665c/src/mongo/db/concurrency/temporarily_unavailable_exception.h#L39-L45). - -## TransactionTooLargeForCacheException - -A TransactionTooLargeForCacheException may be thrown inside the server to indicate that an operation -was rolled-back and is unlikely to ever complete because the storage engine cache is insufficient, -even in the absence of concurrent operations. This is determined by a simple heuristic wherein, -after a rollback, a threshold on the proportion of total dirty cache bytes the running transaction -can represent and still be considered fullfillable is checked. The threshold can be tuned with the -`transactionTooLargeForCacheThreshold` parameter. Setting this threshold to its maximum value (1.0) -causes the check to be skipped and TransactionTooLargeForCacheException to be disabled. - -On replica sets, if an operation succeeds on a primary, it should also succeed on a secondary. It -would be possible to convert to both TemporarilyUnavailableException and WriteConflictException, -as if TransactionTooLargeForCacheException was disabled. But on secondaries the only -difference between the two is the rate at which the operation is retried. Hence, -TransactionTooLargeForCacheException is always converted to a WriteConflictException, which retries -faster, to avoid stalling replication longer than necessary. - -Prior to 6.3, or when TransactionTooLargeForCacheException is disabled, multi-document -transactions always return a WriteConflictException, which may result in drivers retrying an -operation indefinitely. For non-multi-document operations, there is a limited number of retries on -TemporarilyUnavailableException, but it might still be beneficial to not retry operations which are -unlikely to complete and are disruptive for concurrent operations. - ## Collection and Index Writes Collection write operations (inserts, updates, and deletes) perform storage engine writes to both diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index e20adc6cf10..b116e2ad284 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -38,7 +38,6 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/pipeline/change_stream_pre_and_post_images_options', - '$BUILD_DIR/mongo/db/query/query_shape/query_shape', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', @@ -101,29 +100,17 @@ env.Library( ) env.Library( - target='health_log_interface', - source=[ - 'health_log.idl', - 'health_log_interface.cpp', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/service_context', - ], -) - -env.Library( target='health_log', source=[ - 'health_log.cpp', + "health_log.cpp", + "health_log.idl", ], LIBDEPS_PRIVATE=[ "$BUILD_DIR/mongo/base", "$BUILD_DIR/mongo/db/concurrency/deferred_writer", - "$BUILD_DIR/mongo/db/namespace_string", "$BUILD_DIR/mongo/db/service_context", "$BUILD_DIR/mongo/idl/idl_parser", 'collection_options', - 'health_log_interface', ], ) @@ -134,9 +121,9 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog_raii', - '$BUILD_DIR/mongo/db/concurrency/exception_util', + '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/key_string', 'validate_state', ] @@ -150,9 +137,9 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog_raii', - '$BUILD_DIR/mongo/db/concurrency/exception_util', + '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/query/query_knobs', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/db/storage/storage_repair_observer', 'index_repair', @@ -182,7 +169,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/common', - '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/index/index_descriptor', + '$BUILD_DIR/mongo/db/index/key_generator', '$BUILD_DIR/mongo/db/index_names', '$BUILD_DIR/mongo/db/matcher/expressions', '$BUILD_DIR/mongo/db/query/collation/collator_factory_interface', @@ -222,6 +210,7 @@ env.Library( '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/index/index_descriptor', '$BUILD_DIR/mongo/db/index_names', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/ttl_collection_cache', @@ -264,9 +253,9 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog_raii', - '$BUILD_DIR/mongo/db/concurrency/exception_util', + '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/resumable_index_builds_idl', '$BUILD_DIR/mongo/db/service_context', @@ -294,6 +283,7 @@ env.Library( 'views_for_database.cpp', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/profile_filter', @@ -316,7 +306,6 @@ env.Benchmark( LIBDEPS=[ '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/multitenancy', - 'collection', 'collection_catalog', ], ) @@ -331,8 +320,6 @@ env.Library( '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/rebuild_indexes', '$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range', - 'catalog_stats', 'collection', 'collection_catalog', 'database_holder', @@ -370,15 +357,16 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/commands/server_status_core', '$BUILD_DIR/mongo/db/concurrency/lock_manager', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/index/index_access_method', + '$BUILD_DIR/mongo/db/index/index_access_method_factory', + '$BUILD_DIR/mongo/db/index/index_access_methods', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/op_observer', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', '$BUILD_DIR/mongo/db/repl/oplog', @@ -395,7 +383,6 @@ env.Library( '$BUILD_DIR/mongo/db/storage/storage_util', '$BUILD_DIR/mongo/db/system_index', '$BUILD_DIR/mongo/db/timeseries/timeseries_conversion_util', - '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range', '$BUILD_DIR/mongo/db/transaction', '$BUILD_DIR/mongo/db/ttl_collection_cache', '$BUILD_DIR/mongo/db/vector_clock', @@ -408,14 +395,13 @@ env.Library( 'collection_catalog', 'collection_options', 'database_holder', - 'health_log_interface', + 'health_log', 'index_build_block', 'index_catalog', 'index_catalog_entry', 'index_key_validate', 'index_repair', 'local_oplog_info', - 'storage_engine_collection_options_flags_parser', 'throttle_cursor', 'validate_idl', 'validate_state', @@ -445,20 +431,17 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/bson/bson_validate', '$BUILD_DIR/mongo/bson/util/bson_column', - '$BUILD_DIR/mongo/db/concurrency/exception_util', + '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/index_names', '$BUILD_DIR/mongo/db/multi_key_path_tracker', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/record_id_helpers', "$BUILD_DIR/mongo/db/service_context", '$BUILD_DIR/mongo/db/storage/execution_context', '$BUILD_DIR/mongo/db/storage/key_string', - '$BUILD_DIR/mongo/db/timeseries/bucket_catalog', - '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/idl/basic_types', 'catalog_impl', 'collection_options', @@ -475,8 +458,8 @@ env.Library( 'throttle_cursor.cpp', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/index/index_access_method', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/util/fail_point', 'validate_idl', ], @@ -489,7 +472,6 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/logical_time', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/repl/optime', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/storage/flow_control', @@ -502,7 +484,6 @@ env.Library( env.Library( target='catalog_helpers', source=[ - 'backwards_compatible_collection_options_util.cpp', 'capped_utils.cpp', 'collection_catalog_helper.cpp', 'coll_mod.cpp', @@ -518,7 +499,6 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/index_builds_coordinator_interface', @@ -564,8 +544,8 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/collection_index_usage_tracker', '$BUILD_DIR/mongo/db/concurrency/lock_manager', + '$BUILD_DIR/mongo/db/curop', '$BUILD_DIR/mongo/db/fts/base_fts', - '$BUILD_DIR/mongo/db/query/query_stats/query_stats', '$BUILD_DIR/mongo/db/s/sharding_api_d', '$BUILD_DIR/mongo/db/service_context', 'index_catalog', @@ -620,21 +600,7 @@ env.Library( ) if wiredtiger: - wtEnv = env.Clone() - wtEnv.InjectThirdParty(libraries=["wiredtiger"]) - - wtEnv.Library( - target="storage_engine_collection_options_flags_parser", - source=[ - "storage_engine_collection_options_flags_parser.cpp", - ], - LIBDEPS_PRIVATE=[ - "$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger", - "$BUILD_DIR/third_party/shim_pcrecpp", - ], - ) - - wtEnv.CppUnitTest( + env.CppUnitTest( target='db_catalog_test', source=[ 'capped_utils_test.cpp', @@ -644,7 +610,6 @@ if wiredtiger: 'collection_test.cpp', 'collection_validation_test.cpp', 'collection_writer_test.cpp', - 'coll_mod_test.cpp', 'commit_quorum_options_test.cpp', 'create_collection_test.cpp', 'database_test.cpp', @@ -656,17 +621,14 @@ if wiredtiger: 'index_spec_validate_test.cpp', 'multi_index_block_test.cpp', 'rename_collection_test.cpp', - 'storage_engine_collection_options_flags_parser_test.cpp', 'throttle_cursor_test.cpp', 'validate_state_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', - '$BUILD_DIR/mongo/db/commands/create_command', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/db_raii', - '$BUILD_DIR/mongo/db/dbhelpers', '$BUILD_DIR/mongo/db/index_builds_coordinator_mongod', '$BUILD_DIR/mongo/db/matcher/expressions', '$BUILD_DIR/mongo/db/multitenancy', @@ -677,7 +639,6 @@ if wiredtiger: '$BUILD_DIR/mongo/db/query/query_test_service_context', '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', '$BUILD_DIR/mongo/db/repl/oplog', - '$BUILD_DIR/mongo/db/repl/oplog_application', '$BUILD_DIR/mongo/db/repl/optime', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replmocks', @@ -686,7 +647,6 @@ if wiredtiger: '$BUILD_DIR/mongo/db/service_context_d_test_fixture', '$BUILD_DIR/mongo/db/service_context_test_fixture', '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger', - '$BUILD_DIR/mongo/db/timeseries/timeseries_collmod', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/unittest/unittest', '$BUILD_DIR/mongo/util/clock_source_mock', @@ -705,7 +665,6 @@ if wiredtiger: 'index_builds_manager', 'index_key_validate', 'multi_index_block', - 'storage_engine_collection_options_flags_parser', 'throttle_cursor', 'validate_idl', 'validate_state', diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp b/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp deleted file mode 100644 index d966d6108c6..00000000000 --- a/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Copyright (C) 2024-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -// TODO SERVER-92265 evaluate getting rid of this util - -#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" -#include "mongo/bson/bsonobj.h" -#include "mongo/bson/simple_bsonobj_comparator.h" -#include "mongo/db/repl/oplog_entry.h" - -namespace mongo { -namespace backwards_compatible_collection_options { -std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd) { - const BSONObj collModCmdStrippedBackwardsIncompatibleParams = - collModCmd.removeFields(kBackwardsCompatibleCollectionOptions); - if (SimpleBSONObjComparator::kInstance.evaluate(collModCmdStrippedBackwardsIncompatibleParams == - collModCmd)) { - return {collModCmd, BSONObj()}; - } - - const BSONObj backwardsIncompatibleFields = [&]() { - BSONObjBuilder bob; - for (auto [fieldName, elem] : collModCmd) { - if (kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) { - bob.append(elem); - } - } - return bob.obj(); - }(); - - return {collModCmdStrippedBackwardsIncompatibleParams, backwardsIncompatibleFields}; -} - -BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry) { - uassert(ErrorCodes::IllegalOperation, - str::stream() << "Can't extract `collMod` command from non-collMod oplog entry: " - << entry.toBSONForLogging(), - entry.getCommandType() == repl::OplogEntry::CommandType::kCollMod); - - if (!entry.getObject2()) { - return entry.getObject(); - } - - BSONObj incompatibleFields = entry.getObject2()->getObjectField(additionalCollModO2Field); - if (incompatibleFields.isEmpty()) { - return entry.getObject(); - } - - // Only consider backwards incompatible fields supported in the current [sub-]version - for (auto [fieldName, elem] : incompatibleFields) { - if (!kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) { - incompatibleFields = incompatibleFields.removeField(fieldName); - } - } - - return entry.getObject().addFields(incompatibleFields); -} - -} // namespace backwards_compatible_collection_options -} // namespace mongo diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.h b/src/mongo/db/catalog/backwards_compatible_collection_options_util.h deleted file mode 100644 index d5c47b45e6d..00000000000 --- a/src/mongo/db/catalog/backwards_compatible_collection_options_util.h +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright (C) 2024-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -// TODO SERVER-92265 evaluate getting rid of this file - -#include "mongo/db/repl/oplog_entry.h" - -namespace mongo { - -/** - * - * This utility is providing primitives to manage catalog parameters for which the actual value may - * have been missing or incorrect in previous mongod [sub-]versions. It is only meant to fix catalog - * issues in existing versions and must NOT be used for implementing new features. - * - * The functions under this namespace are offering an abstraction to work around the following - * limitations: - * - collMod command is strict (so can't simply add a parameter to previous mongod [sub-]versions) - * - Collection options are strict (so can't simply add an option to previous mongod [sub-]versions) - * - */ -namespace backwards_compatible_collection_options { - -const std::string kTimeseriesBucketsMayHaveMixedSchemaData = - "timeseriesBucketsMayHaveMixedSchemaData"; - -const std::string additionalCollModO2Field = "backwardsIncompatibleCollModParameters"; - -/** - * Backwards incompatible catalog parameters for which the actual value may have been missing or - * incorrect in previous mongod [sub-]versions. - */ -const std::set<std::string> kBackwardsCompatibleCollectionOptions{ - kTimeseriesBucketsMayHaveMixedSchemaData}; - -/** - * Strips backwards incompatible fields from a collMod command and places them into a - * different BSON object. - * - * Returns two BSON objects: - * - A backwards compatible collMod oplog entry (not to generate crashes when applied by - * incompatible mongod [sub-]versions). - * - A field meant to be added to the `o2` sub-object (parsable by new mongod [sub-]versions). - * - * Example: - * - * - Original command: - * {"collMod":"testdb.system.buckets.testcoll", "timeseriesBucketsMayHaveMixedSchemaData":true } - * - * - Expected oplog entry with `timeseriesBucketsMayHaveMixedSchemaData` backwards incompatible - * collMod parameter. - * - * { - * "oplogEntry":{ - * "op":"c", - * "ns":"testdb.$cmd", - * "ui":"UUID(""7302d025-cb9c-4a16-9222-0d5aeefbc039"")", - * "o":{ - * "collMod":"system.buckets.testcoll" - * }, - * "o2":{ - * "collectionOptions_old":{ - * "uuid": UUID("7302d025-cb9c-4a16-9222-0d5aeefbc039"), - * "Validator":{ "...REDACTED..." }, - * "clusteredIndex":true, - * "timeseries":{ - * "timeField":"t", - * “granularity":"seconds", - * "bucketMaxSpanSeconds":3600 - * } - * }, - * "backwardsIncompatibleCollModParameters":{ - * "timeseriesBucketsMayHaveMixedSchemaData":true - * } - * }, - * "ts":Timestamp(1720003401,4), - * "t":1, - * "v":2, - * "wall":new Date(1720003401165) - * } - * } - * - */ -std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd); - -/** - * Rebuilds a collMod command from an oplog entry. - * - * Returns a bson object: - * - A collMod command inclusive of potential backwards incompatible fields present in the oplog - * entry's `o2` sub-object. - * - * Example: collMod command parsed from the sample oplog entry documented above. - * - * { - * "collMod":"system.buckets.testcoll", - * "timeseriesBucketsMayHaveMixedSchemaData":true - * } - * - */ -BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry); - -} // namespace backwards_compatible_collection_options -} // namespace mongo diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp index 77ccf110f97..945881e58c5 100644 --- a/src/mongo/db/catalog/capped_utils.cpp +++ b/src/mongo/db/catalog/capped_utils.cpp @@ -43,7 +43,7 @@ #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/namespace_string.h" @@ -245,7 +245,8 @@ void cloneCollectionAsCapped(OperationContext* opCtx, insertStmt.oplogSlot = oplogSlots.front(); } - uassertStatusOK(toCollection->insertDocument(opCtx, insertStmt, nullOpDebug, true)); + uassertStatusOK(toCollection->insertDocument( + opCtx, InsertStatement(objToClone), nullOpDebug, true)); wunit.commit(); // Go to the next document @@ -253,7 +254,7 @@ void cloneCollectionAsCapped(OperationContext* opCtx, } catch (const WriteConflictException&) { CurOp::get(opCtx)->debug().additiveMetrics.incrementWriteConflicts(1); retries++; // logAndBackoff expects this to be 1 on first call. - logWriteConflictAndBackoff(retries, "cloneCollectionAsCapped", fromNss.ns()); + WriteConflictException::logAndBackoff(retries, "cloneCollectionAsCapped", fromNss.ns()); // Can't use writeConflictRetry since we need to save/restore exec around call to // abandonSnapshot. diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp index 19f413fc1b6..dadfed598c0 100644 --- a/src/mongo/db/catalog/catalog_control.cpp +++ b/src/mongo/db/catalog/catalog_control.cpp @@ -35,7 +35,6 @@ #include "mongo/db/catalog/catalog_control.h" -#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/database.h" @@ -44,18 +43,17 @@ #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/namespace_string.h" #include "mongo/db/rebuild_indexes.h" -#include "mongo/db/repl/oplog.h" #include "mongo/db/tenant_database_name.h" -#include "mongo/db/timeseries/timeseries_extended_range.h" #include "mongo/logv2/log.h" namespace mongo { namespace catalog { namespace { -void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx, - StorageEngine* storageEngine, - const PreviousCatalogState& previousCatalogState, - Timestamp stableTimestamp) { +void reopenAllDatabasesAndReloadCollectionCatalog( + OperationContext* opCtx, + StorageEngine* storageEngine, + const MinVisibleTimestampMap& minVisibleTimestampMap, + Timestamp stableTimestamp) { // Open all databases and repopulate the CollectionCatalog. LOGV2(20276, "openCatalog: reopening all databases"); @@ -82,7 +80,7 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx, str::stream() << "failed to get valid collection pointer for namespace " << collNss); - if (previousCatalogState.minVisibleTimestampMap.count(collection->uuid()) > 0) { + if (minVisibleTimestampMap.count(collection->uuid()) > 0) { // After rolling back to a stable timestamp T, the minimum visible timestamp for // each collection must be reset to (at least) its value at T. Additionally, there // cannot exist a minimum visible timestamp greater than lastApplied. This allows us @@ -92,31 +90,14 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx, // bound the minimum visible timestamp (where necessary) to the stable timestamp. // The benefit of fine grained tracking is assumed to be low-value compared to the // cost/effort. - auto minVisible = std::min( - stableTimestamp, - previousCatalogState.minVisibleTimestampMap.find(collection->uuid())->second); + auto minVisible = std::min(stableTimestamp, + minVisibleTimestampMap.find(collection->uuid())->second); auto writableCollection = catalogWriter.get()->lookupCollectionByUUIDForMetadataWrite(opCtx, collection->uuid()); writableCollection->setMinimumVisibleSnapshot(minVisible); } - if (collection->getTimeseriesOptions()) { - bool extendedRangeSetting; - if (auto it = previousCatalogState.requiresTimestampExtendedRangeSupportMap.find( - collection->uuid()); - it != previousCatalogState.requiresTimestampExtendedRangeSupportMap.end()) { - extendedRangeSetting = it->second; - } else { - extendedRangeSetting = - timeseries::collectionMayRequireExtendedRangeSupport(opCtx, collection); - } - - if (extendedRangeSetting) { - collection->setRequiresTimeseriesExtendedRangeSupport(opCtx); - } - } - // If this is the oplog collection, re-establish the replication system's cached pointer // to the oplog. if (collNss.isOplog()) { @@ -133,25 +114,28 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx, // Opening CollectionCatalog: The collection catalog is now in sync with the storage engine // catalog. Clear the pre-closing state. - CollectionCatalog::write(opCtx, [](CollectionCatalog& catalog) { catalog.onOpenCatalog(); }); + CollectionCatalog::write(opCtx, + [&](CollectionCatalog& catalog) { catalog.onOpenCatalog(opCtx); }); opCtx->getServiceContext()->incrementCatalogGeneration(); LOGV2(20278, "openCatalog: finished reloading collection catalog"); } } // namespace -PreviousCatalogState closeCatalog(OperationContext* opCtx) { +MinVisibleTimestampMap closeCatalog(OperationContext* opCtx) { invariant(opCtx->lockState()->isW()); IndexBuildsCoordinator::get(opCtx)->assertNoIndexBuildInProgress(); - PreviousCatalogState previousCatalogState; + MinVisibleTimestampMap minVisibleTimestampMap; std::vector<TenantDatabaseName> allDbs = opCtx->getServiceContext()->getStorageEngine()->listDatabases(); auto databaseHolder = DatabaseHolder::get(opCtx); auto catalog = CollectionCatalog::get(opCtx); for (auto&& tenantDbName : allDbs) { - for (auto&& coll : catalog->range(tenantDbName)) { + for (auto collIt = catalog->begin(opCtx, tenantDbName); collIt != catalog->end(opCtx); + ++collIt) { + auto coll = *collIt; if (!coll) { break; } @@ -166,12 +150,7 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) { "coll_ns"_attr = coll->ns(), "uuid"_attr = coll->uuid(), "minVisible"_attr = minVisible); - previousCatalogState.minVisibleTimestampMap[coll->uuid()] = *minVisible; - } - - if (coll->getTimeseriesOptions()) { - previousCatalogState.requiresTimestampExtendedRangeSupportMap[coll->uuid()] = - coll->getRequiresTimeseriesExtendedRangeSupport(); + minVisibleTimestampMap[coll->uuid()] = *minVisible; } } } @@ -179,13 +158,14 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) { // Need to mark the CollectionCatalog as open if we our closeAll fails, dismissed if successful. ScopeGuard reopenOnFailure([opCtx] { CollectionCatalog::write(opCtx, - [](CollectionCatalog& catalog) { catalog.onOpenCatalog(); }); + [&](CollectionCatalog& catalog) { catalog.onOpenCatalog(opCtx); }); }); // Closing CollectionCatalog: only lookupNSSByUUID will fall back to using pre-closing state to // allow authorization for currently unknown UUIDs. This is needed because authorization needs // to work before acquiring locks, and might otherwise spuriously regard a UUID as unknown // while reloading the catalog. - CollectionCatalog::write(opCtx, [](CollectionCatalog& catalog) { catalog.onCloseCatalog(); }); + CollectionCatalog::write(opCtx, + [&](CollectionCatalog& catalog) { catalog.onCloseCatalog(opCtx); }); LOGV2_DEBUG(20270, 1, "closeCatalog: closing collection catalog"); @@ -197,16 +177,12 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) { LOGV2(20272, "closeCatalog: closing storage engine catalog"); opCtx->getServiceContext()->getStorageEngine()->closeCatalog(opCtx); - // Reset the stats counter for extended range time-series collections. This is maintained - // outside the catalog itself. - catalog_stats::requiresTimeseriesExtendedRangeSupport.store(0); - reopenOnFailure.dismiss(); - return previousCatalogState; + return minVisibleTimestampMap; } void openCatalog(OperationContext* opCtx, - const PreviousCatalogState& previousCatalogState, + const MinVisibleTimestampMap& minVisibleTimestampMap, Timestamp stableTimestamp) { invariant(opCtx->lockState()->isW()); @@ -278,7 +254,7 @@ void openCatalog(OperationContext* opCtx, opCtx, reconcileResult.indexBuildsToRestart, reconcileResult.indexBuildsToResume); reopenAllDatabasesAndReloadCollectionCatalog( - opCtx, storageEngine, previousCatalogState, stableTimestamp); + opCtx, storageEngine, minVisibleTimestampMap, stableTimestamp); } diff --git a/src/mongo/db/catalog/catalog_control.h b/src/mongo/db/catalog/catalog_control.h index f7bc3fbbdba..182f073c307 100644 --- a/src/mongo/db/catalog/catalog_control.h +++ b/src/mongo/db/catalog/catalog_control.h @@ -36,11 +36,6 @@ namespace catalog { using MinVisibleTimestamp = Timestamp; using MinVisibleTimestampMap = std::map<UUID, MinVisibleTimestamp>; -using RequiresTimestampExtendedRangeSupportMap = std::map<UUID, bool>; -struct PreviousCatalogState { - MinVisibleTimestampMap minVisibleTimestampMap; - RequiresTimestampExtendedRangeSupportMap requiresTimestampExtendedRangeSupportMap; -}; /** * Closes the catalog, destroying all associated in-memory data structures for all databases. After @@ -48,7 +43,7 @@ struct PreviousCatalogState { * * Must be called with the global lock acquired in exclusive mode. */ -PreviousCatalogState closeCatalog(OperationContext* opCtx); +MinVisibleTimestampMap closeCatalog(OperationContext* opCtx); /** * Restores the catalog and all in-memory state after a call to closeCatalog(). @@ -56,7 +51,7 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx); * Must be called with the global lock acquired in exclusive mode. */ void openCatalog(OperationContext* opCtx, - const PreviousCatalogState& catalogState, + const MinVisibleTimestampMap& catalogState, Timestamp stableTimestamp); /** diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp index 901c69450ee..4eca3df6a9a 100644 --- a/src/mongo/db/catalog/catalog_control_test.cpp +++ b/src/mongo/db/catalog/catalog_control_test.cpp @@ -75,8 +75,8 @@ TEST_F(CatalogControlTest, CloseAndOpenCatalog) { ServiceContext::UniqueOperationContext opCtx = cc().makeOperationContext(); Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - auto previousState = catalog::closeCatalog(opCtx.get()); - ASSERT_EQUALS(0U, previousState.minVisibleTimestampMap.size()); + auto map = catalog::closeCatalog(opCtx.get()); + ASSERT_EQUALS(0U, map.size()); catalog::openCatalog(opCtx.get(), {}, Timestamp()); } diff --git a/src/mongo/db/catalog/catalog_stats.cpp b/src/mongo/db/catalog/catalog_stats.cpp index d346e67f971..ea7b4ee2276 100644 --- a/src/mongo/db/catalog/catalog_stats.cpp +++ b/src/mongo/db/catalog/catalog_stats.cpp @@ -29,7 +29,7 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand -#include "mongo/db/catalog/catalog_stats.h" +#include "mongo/platform/basic.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/database_holder.h" @@ -37,10 +37,7 @@ #include "mongo/db/db_raii.h" #include "mongo/logv2/log.h" -namespace mongo::catalog_stats { - -// Number of time-series collections requiring extended range support -AtomicWord<int> requiresTimeseriesExtendedRangeSupport; +namespace mongo { namespace { class CatalogStatsSSS : public ServerStatusSection { @@ -61,7 +58,6 @@ public: int timeseries = 0; int internalCollections = 0; int internalViews = 0; - int timeseriesExtendedRange = 0; void toBson(BSONObjBuilder* builder) const { builder->append("collections", collections); @@ -71,9 +67,6 @@ public: builder->append("views", views); builder->append("internalCollections", internalCollections); builder->append("internalViews", internalViews); - if (timeseriesExtendedRange > 0) { - builder->append("timeseriesExtendedRange", timeseriesExtendedRange); - } } }; @@ -87,7 +80,6 @@ public: stats.capped = catalogStats.userCapped; stats.clustered = catalogStats.userClustered; stats.internalCollections = catalogStats.internal; - stats.timeseriesExtendedRange = requiresTimeseriesExtendedRangeSupport.load(); const auto viewCatalogDbNames = catalog->getViewCatalogDbNames(opCtx); for (const auto& tenantDbName : viewCatalogDbNames) { @@ -114,4 +106,4 @@ public: } catalogStatsSSS; } // namespace -} // namespace mongo::catalog_stats +} // namespace mongo diff --git a/src/mongo/db/catalog/catalog_stats.h b/src/mongo/db/catalog/catalog_stats.h deleted file mode 100644 index 54cc04f0bb6..00000000000 --- a/src/mongo/db/catalog/catalog_stats.h +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/platform/atomic_word.h" - -namespace mongo::catalog_stats { - -extern AtomicWord<int> requiresTimeseriesExtendedRangeSupport; - -} // namespace mongo::catalog_stats diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 2c9f03053d0..8af0ba8efc9 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -33,7 +33,6 @@ #include "mongo/db/catalog/coll_mod.h" -#include "mongo/db/stats/counters.h" #include <boost/optional.hpp> #include <memory> @@ -45,7 +44,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/coll_mod_gen.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop_failpoint_helpers.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" @@ -54,7 +53,6 @@ #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/database_sharding_state.h" -#include "mongo/db/s/shard_key_index_util.h" #include "mongo/db/server_options.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/recovery_unit.h" @@ -64,7 +62,6 @@ #include "mongo/db/views/view_catalog_helpers.h" #include "mongo/idl/command_generic_argument.h" #include "mongo/logv2/log.h" -#include "mongo/s/grid.h" #include "mongo/util/fail_point.h" #include "mongo/util/version/releases.h" #include "mongo/util/visit_helper.h" @@ -112,13 +109,12 @@ struct ParsedCollModRequest { boost::optional<Collection::Validator> collValidator; boost::optional<ValidationActionEnum> collValidationAction; boost::optional<ValidationLevelEnum> collValidationLevel; - boost::optional<bool> recordPreImages; + bool recordPreImages = false; boost::optional<ChangeStreamPreAndPostImagesOptions> changeStreamPreAndPostImagesOptions; int numModifications = 0; bool dryRun = false; boost::optional<long long> cappedSize; boost::optional<long long> cappedMax; - boost::optional<bool> timeseriesBucketsMayHaveMixedSchemaData; }; Status getNotSupportedOnViewError(StringData fieldName) { @@ -254,8 +250,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati "TTL indexes are not supported for capped collections."}; } if (auto status = index_key_validate::validateExpireAfterSeconds( - *cmdIndex.getExpireAfterSeconds(), - index_key_validate::ValidateExpireAfterSecondsMode::kSecondaryTTLIndex); + *cmdIndex.getExpireAfterSeconds()); !status.isOK()) { return {ErrorCodes::InvalidOptions, status.reason()}; } @@ -289,8 +284,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati } } else { std::vector<const IndexDescriptor*> indexes; - coll->getIndexCatalog()->findIndexesByKeyPattern( - opCtx, keyPattern, IndexCatalog::InclusionPolicy::kReady, &indexes); + coll->getIndexCatalog()->findIndexesByKeyPattern(opCtx, keyPattern, false, &indexes); if (indexes.size() > 1) { return {ErrorCodes::AmbiguousIndexKeyPattern, @@ -343,7 +337,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati if (cmrIndex->idx->unique()) { indexForOplog->setUnique(boost::none); } else { - // Disallow one-step unique conversion. The user has to set + // Disallow one-step unique convertion. The user has to set // 'prepareUnique' to true first. if (!cmrIndex->idx->prepareUnique()) { return Status(ErrorCodes::InvalidOptions, @@ -369,29 +363,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati return {ErrorCodes::BadValue, "can't hide _id index"}; } - // If the index is not hidden and we are trying to hide it, check if it is possible - // to drop the shard key index, so it could be possible to hide it. - if (!cmrIndex->idx->hidden() && *cmdIndex.getHidden()) { - if (auto catalogClient = Grid::get(opCtx)->catalogClient()) { - try { - auto shardedColl = catalogClient->getCollection(opCtx, nss); - - if (isLastNonHiddenRangedShardKeyIndex( - opCtx, - coll, - coll->getIndexCatalog(), - cmrIndex->idx->indexName(), - shardedColl.getKeyPattern().toBSON())) { - return {ErrorCodes::InvalidOptions, - "Can't hide the only compatible index for this collection's " - "shard key"}; - } - } catch (ExceptionFor<ErrorCodes::NamespaceNotFound>&) { - // The collection is unsharded or doesn't exist. - } - } - } - // Hiding a hidden index or unhiding a visible index should be treated as a no-op. if (cmrIndex->idx->hidden() == *cmdIndex.getHidden()) { indexForOplog->setHidden(boost::none); @@ -401,35 +372,12 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati } if (cmdIndex.getPrepareUnique()) { - // Check if prepareUnique is being set on a time-series collection. - if (isTimeseries) { - return {ErrorCodes::InvalidOptions, - "cannot set 'prepareUnique' for indexes of a time-series collection."}; - } parsed.numModifications++; // Attempting to modify with the same value should be treated as a no-op. if (cmrIndex->idx->prepareUnique() == *cmdIndex.getPrepareUnique() || cmrIndex->idx->unique()) { indexForOplog->setPrepareUnique(boost::none); } else { - // Checks if the index key pattern conflicts with the shard key pattern. - if (auto catalogClient = Grid::get(opCtx)->catalogClient()) { - try { - auto shardedColl = catalogClient->getCollection(opCtx, nss); - const ShardKeyPattern shardKeyPattern(shardedColl.getKeyPattern()); - if (!shardKeyPattern.isIndexUniquenessCompatible( - cmrIndex->idx->keyPattern())) { - return {ErrorCodes::InvalidOptions, - fmt::format( - "cannot set 'prepareUnique' for index {} with shard key " - "pattern {}", - cmrIndex->idx->keyPattern().toString(), - shardKeyPattern.toBSON().toString())}; - } - } catch (ExceptionFor<ErrorCodes::NamespaceNotFound>&) { - // The collection is unsharded or doesn't exist. - } - } cmrIndex->indexPrepareUnique = cmdIndex.getPrepareUnique(); } } @@ -482,11 +430,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati validatorObj.getOwned(), MatchExpressionParser::kDefaultSpecialFeatures, maxFeatureCompatibilityVersion); - - // Increment counters to track the usage of schema validators. - validatorCounters.incrementCounters( - cmd.kCommandName, parsed.collValidator->validatorDoc, parsed.collValidator->isOK()); - if (!parsed.collValidator->isOK()) { return parsed.collValidator->getStatus(); } @@ -590,9 +533,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati }, [&oplogEntryBuilder](std::int64_t value) { oplogEntryBuilder.append(CollMod::kExpireAfterSecondsFieldName, value); - return index_key_validate::validateExpireAfterSeconds( - value, - index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex); + return index_key_validate::validateExpireAfterSeconds(value); }, }, *expireAfterSeconds); @@ -611,17 +552,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati timeseries->serialize(&subObjBuilder); } - if (auto mixedSchema = cmr.getTimeseriesBucketsMayHaveMixedSchemaData()) { - if (!isTimeseries) { - return getOnlySupportedOnTimeseriesError( - CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName); - } - - parsed.timeseriesBucketsMayHaveMixedSchemaData = mixedSchema; - oplogEntryBuilder.append(CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName, - *mixedSchema); - } - if (auto& dryRun = cmr.getDryRun()) { parsed.dryRun = *dryRun; // The dry run option should never be included in a collMod oplog entry. @@ -666,8 +596,7 @@ void _setClusteredExpireAfterSeconds( if (!oldExpireAfterSeconds) { auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext()); opCtx->recoveryUnit()->onCommit([ttlCache, uuid = coll->uuid()](auto _) { - ttlCache->registerTTLInfo( - uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}}); + ttlCache->registerTTLInfo(uuid, TTLCollectionCache::ClusteredId()); }); } @@ -917,7 +846,7 @@ Status _collModInternal(OperationContext* opCtx, cmrNew.recordPreImages = false; } - if (cmrNew.recordPreImages && *cmrNew.recordPreImages) { + if (cmrNew.recordPreImages) { cmrNew.changeStreamPreAndPostImagesOptions = ChangeStreamPreAndPostImagesOptions(false); } @@ -947,11 +876,6 @@ Status _collModInternal(OperationContext* opCtx, *cmd.getExpireAfterSeconds()); } - if (auto mixedSchema = cmrNew.timeseriesBucketsMayHaveMixedSchemaData) { - coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( - opCtx, mixedSchema); - } - // Handle index modifications. processCollModIndexRequest( opCtx, &coll, cmrNew.indexRequest, &indexCollModInfo, result, mode); @@ -969,9 +893,8 @@ Status _collModInternal(OperationContext* opCtx, "Failed to set validationLevel"); } - if (cmrNew.recordPreImages.has_value() && - *cmrNew.recordPreImages != oldCollOptions.recordPreImages) { - coll.getWritableCollection(opCtx)->setRecordPreImages(opCtx, *cmrNew.recordPreImages); + if (cmrNew.recordPreImages != oldCollOptions.recordPreImages) { + coll.getWritableCollection(opCtx)->setRecordPreImages(opCtx, cmrNew.recordPreImages); } if (cmrNew.changeStreamPreAndPostImagesOptions.has_value() && @@ -991,9 +914,9 @@ Status _collModInternal(OperationContext* opCtx, } } - // Fix any invalid index options for indexes belonging to this collection. + // Remove any invalid index options for indexes belonging to this collection. std::vector<std::string> indexesWithInvalidOptions = - coll.getWritableCollection(opCtx)->repairInvalidIndexOptions(opCtx); + coll.getWritableCollection(opCtx)->removeInvalidIndexOptions(opCtx); for (const auto& indexWithInvalidOptions : indexesWithInvalidOptions) { const IndexDescriptor* desc = coll->getIndexCatalog()->findIndexByName(opCtx, indexWithInvalidOptions); @@ -1007,24 +930,24 @@ Status _collModInternal(OperationContext* opCtx, // (Generic FCV reference): TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated // upgrade/downgrade code. const auto currentVersion = serverGlobalParams.featureCompatibility.getVersion(); - if (coll->getTimeseriesOptions()) { - if (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest) { - // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest, - // collMod is called as part of the upgrade process to add the - // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series - // collections that are missing the flag. This indicates that the time-series - // collection existed in earlier server versions and may have mixed-schema data. - coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, - true); - } else if (currentVersion == - multiversion::GenericFCV::kDowngradingFromLatestToLastLTS) { - // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called - // as part of the downgrade process to remove the - // 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag for time-series - // collections that have the flag. - coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( - opCtx, boost::none); - } + if (coll->getTimeseriesOptions() && !coll->getTimeseriesBucketsMayHaveMixedSchemaData() && + (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest || + currentVersion == multiversion::GenericFCV::kLatest)) { + // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest, collMod is + // called as part of the upgrade process to add the + // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series + // collections that are missing the flag. This indicates that the time-series collection + // existed in earlier server versions and may have mixed-schema data. + coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, + true); + } else if (coll->getTimeseriesBucketsMayHaveMixedSchemaData() && + (currentVersion == multiversion::GenericFCV::kDowngradingFromLatestToLastLTS || + currentVersion == multiversion::GenericFCV::kLastLTS)) { + // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called as + // part of the downgrade process to remove the 'timeseriesBucketsMayHaveMixedSchemaData' + // catalog entry flag for time-series collections that have the flag. + coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData( + opCtx, boost::none); } // Only observe non-view collMods, as view operations are observed as operations on the @@ -1040,39 +963,6 @@ Status _collModInternal(OperationContext* opCtx, } // namespace -bool isCollModIndexUniqueConversion(const CollModRequest& request) { - auto index = request.getIndex(); - if (!index) { - return false; - } - if (auto indexUnique = index->getUnique(); !indexUnique) { - return false; - } - // Checks if the request is an actual unique conversion instead of a dry run. - if (auto dryRun = request.getDryRun(); dryRun && *dryRun) { - return false; - } - return true; -} - -CollModRequest makeCollModDryRunRequest(const CollModRequest& request) { - CollModRequest dryRunRequest; - CollModIndex dryRunIndex; - const auto& requestIndex = request.getIndex(); - dryRunIndex.setUnique(true); - if (auto keyPattern = requestIndex->getKeyPattern()) { - dryRunIndex.setKeyPattern(keyPattern); - } else if (auto name = requestIndex->getName()) { - dryRunIndex.setName(name); - } - if (auto uuid = request.getCollectionUUID()) { - dryRunRequest.setCollectionUUID(uuid); - } - dryRunRequest.setIndex(dryRunIndex); - dryRunRequest.setDryRun(true); - return dryRunRequest; -} - Status processCollModCommand(OperationContext* opCtx, const NamespaceStringOrUUID& nsOrUUID, const CollMod& cmd, diff --git a/src/mongo/db/catalog/coll_mod.h b/src/mongo/db/catalog/coll_mod.h index f2b0c1702d5..f08aca11444 100644 --- a/src/mongo/db/catalog/coll_mod.h +++ b/src/mongo/db/catalog/coll_mod.h @@ -48,18 +48,6 @@ class OperationContext; void addCollectionUUIDs(OperationContext* opCtx); /** - * Checks if the collMod request is converting an index to unique. - */ -bool isCollModIndexUniqueConversion(const CollModRequest& request); - -/** - * Constructs a valid collMod dry-run request from the original request. - * The 'dryRun' option can only be used with the index 'unique' option, so we assume 'request' must - * have the 'unique' option. The function will also remove other options from the original request. - */ -CollModRequest makeCollModDryRunRequest(const CollModRequest& request); - -/** * Performs the collection modification described in "cmd" on the collection "ns". */ Status processCollModCommand(OperationContext* opCtx, diff --git a/src/mongo/db/catalog/coll_mod_index.cpp b/src/mongo/db/catalog/coll_mod_index.cpp index 2ffecf7f1e1..d56acc639f4 100644 --- a/src/mongo/db/catalog/coll_mod_index.cpp +++ b/src/mongo/db/catalog/coll_mod_index.cpp @@ -67,10 +67,8 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx, // Do not refer to 'idx' within this commit handler as it may be be invalidated by // IndexCatalog::refreshEntry(). opCtx->recoveryUnit()->onCommit( - [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds]( - auto _) { - ttlCache->registerTTLInfo( - uuid, TTLCollectionCache::Info{indexName, /*isExpireAfterSecondsNaN=*/false}); + [ttlCache, uuid = coll->uuid(), indexName = idx->indexName()](auto _) { + ttlCache->registerTTLInfo(uuid, indexName); }); // Change the value of "expireAfterSeconds" on disk. @@ -79,29 +77,6 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx, return; } - // If the current `expireAfterSeconds` is NaN, it can never be equal to - // 'indexExpireAfterSeconds'. - if (oldExpireSecsElement.isNaN()) { - // Setting *oldExpireSecs is mostly for informational purposes. - // We could also use index_key_validate::kExpireAfterSecondsForInactiveTTLIndex but - // 0 is more consistent with the previous safeNumberLong() behavior and avoids potential - // showing the same value for the new and old values in the collMod response. - *oldExpireSecs = 0; - - // Change the value of "expireAfterSeconds" on disk. - autoColl->getWritableCollection(opCtx)->updateTTLSetting( - opCtx, idx->indexName(), indexExpireAfterSeconds); - - // Keep the TTL information maintained by the TTLCollectionCache in sync so that we don't - // try to fix up the TTL index during the next step-up. - auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext()); - const auto& coll = autoColl->getCollection(); - opCtx->recoveryUnit()->onCommit( - [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds]( - auto _) { ttlCache->unsetTTLIndexExpireAfterSecondsNaN(uuid, indexName); }); - return; - } - // This collection is already TTL. Compare the requested value against the existing setting // before updating the catalog. *oldExpireSecs = oldExpireSecsElement.safeNumberLong(); diff --git a/src/mongo/db/catalog/coll_mod_test.cpp b/src/mongo/db/catalog/coll_mod_test.cpp deleted file mode 100644 index c6335aa1c67..00000000000 --- a/src/mongo/db/catalog/coll_mod_test.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/catalog/coll_mod.h" - -#include <boost/optional.hpp> - -#include "mongo/db/coll_mod_gen.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { -TEST(CollModOptionTest, isConvertingIndexToUnique) { - IDLParserErrorContext ctx("collMod"); - auto requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}}"); - auto request = CollModRequest::parse(ctx, requestObj); - ASSERT_TRUE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true, hidden: true}}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_TRUE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson( - "{index: {keyPattern: {a: 1}, unique: true, hidden: true}, validationAction: 'warn'}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_TRUE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: true}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_FALSE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: false}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_TRUE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, prepareUnique: true}}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_FALSE(isCollModIndexUniqueConversion(request)); - - requestObj = fromjson("{validationAction: 'warn'}"); - request = CollModRequest::parse(ctx, requestObj); - ASSERT_FALSE(isCollModIndexUniqueConversion(request)); -} - -TEST(CollModOptionTest, makeDryRunRequest) { - IDLParserErrorContext ctx("collMod"); - auto requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}}"); - auto request = CollModRequest::parse(ctx, requestObj); - auto dryRunRequest = makeCollModDryRunRequest(request); - ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}"))); - ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique()); - ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun()); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true, hidden: true}}"); - request = CollModRequest::parse(ctx, requestObj); - dryRunRequest = makeCollModDryRunRequest(request); - ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}"))); - ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique()); - ASSERT_FALSE(dryRunRequest.getIndex()->getHidden()); - ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun()); - - requestObj = fromjson( - "{index: {keyPattern: {a: 1}, unique: true, hidden: true}, validationAction: 'warn'}"); - request = CollModRequest::parse(ctx, requestObj); - dryRunRequest = makeCollModDryRunRequest(request); - ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}"))); - ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique()); - ASSERT_FALSE(dryRunRequest.getIndex()->getHidden()); - ASSERT_FALSE(dryRunRequest.getValidationAction()); - ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun()); - - requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: false}"); - request = CollModRequest::parse(ctx, requestObj); - dryRunRequest = makeCollModDryRunRequest(request); - ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}"))); - ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique()); - ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun()); -} -} // namespace -} // namespace mongo diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 9be52a11a11..425b02dc9d5 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -97,9 +97,6 @@ struct CollectionUpdateArgs { bool preImageRecordingEnabledForCollection = false; bool changeStreamPreAndPostImagesEnabledForCollection = false; - // Set if the diff insert operation needs to check for the field's existence. - bool mustCheckExistenceForInsertOperations = false; - // Set if OpTimes were reserved for the update ahead of time. std::vector<OplogSlot> oplogSlots; }; @@ -560,28 +557,11 @@ public: boost::optional<bool> setting) = 0; /** - * Returns true if the passed in time-series bucket document contains mixed-schema data. Returns - * a non-OK status if the bucket's min/max is malformed. + * Returns true if the passed in time-series bucket document contains mixed-schema data. */ - virtual StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( + virtual bool doesTimeseriesBucketsDocContainMixedSchemaData( const BSONObj& bucketsDoc) const = 0; - /** - * Returns true if the time-series collection may have dates outside the standard range (roughly - * 1970-2038). The value may be updated in the background by another thread between calls, even - * if the caller holds a lock on the collection. The value may only transition from false to - * true. - */ - virtual bool getRequiresTimeseriesExtendedRangeSupport() const = 0; - - /** - * Sets the in-memory flag for this collection. This value can be retrieved by - * 'getRequiresTimeseriesExtendedRangeSupport'. - * - * Throws if this is not a time-series collection. - */ - virtual void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const = 0; - /* * Returns true if this collection is clustered. That is, its RecordIds store the value of the * cluster key. If the collection is clustered on _id, there is no separate _id index. @@ -640,10 +620,10 @@ public: bool prepareUnique) = 0; /** - * Repairs invalid index options on all indexes in this collection. Returns a list of - * index names that were repaired. + * Removes invalid index options on all indexes in this collection. Returns a list of index + * names that contained invalid index options. */ - virtual std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) = 0; + virtual std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) = 0; /** * Updates the 'temp' setting for this collection. diff --git a/src/mongo/db/catalog/collection_catalog.cpp b/src/mongo/db/catalog/collection_catalog.cpp index cdaef1c8e55..4e5df27b592 100644 --- a/src/mongo/db/catalog/collection_catalog.cpp +++ b/src/mongo/db/catalog/collection_catalog.cpp @@ -52,13 +52,10 @@ const ServiceContext::Decoration<LatestCollectionCatalog> getCatalog = ServiceContext::declareDecoration<LatestCollectionCatalog>(); std::shared_ptr<CollectionCatalog> batchedCatalogWriteInstance; -absl::flat_hash_set<Collection*> batchedCatalogClonedCollections; const OperationContext::Decoration<std::shared_ptr<const CollectionCatalog>> stashedCatalog = OperationContext::declareDecoration<std::shared_ptr<const CollectionCatalog>>(); -const auto maxUuid = UUID::parse("FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF").getValue(); -const auto minUuid = UUID::parse("00000000-0000-0000-0000-000000000000").getValue(); } // namespace class IgnoreExternalViewChangesForDatabase { @@ -94,12 +91,12 @@ public: static void setCollectionInCatalog(CollectionCatalog& catalog, std::shared_ptr<Collection> collection) { - catalog._collections = catalog._collections.set(collection->ns(), collection); - catalog._catalog = catalog._catalog.set(collection->uuid(), collection); + catalog._collections[collection->ns()] = collection; + catalog._catalog[collection->uuid()] = collection; // TODO SERVER-64608 Use tenantID from ns auto dbIdPair = std::make_pair(TenantDatabaseName(boost::none, collection->ns().db()), collection->uuid()); - catalog._orderedCollections = catalog._orderedCollections.set(dbIdPair, collection); + catalog._orderedCollections[dbIdPair] = collection; } PublishCatalogUpdates(OperationContext* opCtx, @@ -133,7 +130,7 @@ public: case UncommittedCatalogUpdates::Entry::Action::kRenamedCollection: { writeJobs.push_back( [& from = entry.nss, &to = entry.renameTo](CollectionCatalog& catalog) { - catalog._collections = catalog._collections.erase(from); + catalog._collections.erase(from); auto fromStr = from.ns(); auto toStr = to.ns(); @@ -154,9 +151,10 @@ public: break; } case UncommittedCatalogUpdates::Entry::Action::kRecreatedCollection: { - writeJobs.push_back([opCtx = _opCtx, collection = entry.collection]( - CollectionCatalog& catalog) { - catalog.registerCollection(opCtx, std::move(collection)); + writeJobs.push_back([opCtx = _opCtx, + collection = entry.collection, + uuid = *entry.externalUUID](CollectionCatalog& catalog) { + catalog.registerCollection(opCtx, uuid, std::move(collection)); }); // Fallthrough to the createCollection case to finish committing the collection. } @@ -223,65 +221,90 @@ private: UncommittedCatalogUpdates& _uncommittedCatalogUpdates; }; -CollectionCatalog::iterator::iterator(const TenantDatabaseName& tenantDbName, - OrderedCollectionMap::iterator it, - const OrderedCollectionMap& map) - : _map{map}, _mapIter{it}, _end(_map.upper_bound(std::make_pair(tenantDbName, maxUuid))) { - _skipUncommitted(); +CollectionCatalog::iterator::iterator(OperationContext* opCtx, + const TenantDatabaseName& tenantDbName, + const CollectionCatalog& catalog) + : _opCtx(opCtx), _tenantDbName(tenantDbName), _catalog(&catalog) { + auto minUuid = UUID::parse("00000000-0000-0000-0000-000000000000").getValue(); + + _mapIter = _catalog->_orderedCollections.lower_bound(std::make_pair(_tenantDbName, minUuid)); + + // Start with the first collection that is visible outside of its transaction. + while (!_exhausted() && !_mapIter->second->isCommitted()) { + _mapIter++; + } + + if (!_exhausted()) { + _uuid = _mapIter->first.second; + } } +CollectionCatalog::iterator::iterator(OperationContext* opCtx, + std::map<std::pair<TenantDatabaseName, UUID>, + std::shared_ptr<Collection>>::const_iterator mapIter, + const CollectionCatalog& catalog) + : _opCtx(opCtx), _mapIter(mapIter), _catalog(&catalog) {} + CollectionCatalog::iterator::value_type CollectionCatalog::iterator::operator*() { - if (_mapIter == _map.end()) { - return nullptr; + if (_exhausted()) { + return CollectionPtr(); } - return _mapIter->second.get(); + + return { + _opCtx, _mapIter->second.get(), LookupCollectionForYieldRestore(_mapIter->second->ns())}; +} + +Collection* CollectionCatalog::iterator::getWritableCollection(OperationContext* opCtx) { + return CollectionCatalog::get(opCtx)->lookupCollectionByUUIDForMetadataWrite( + opCtx, operator*()->uuid()); +} + +boost::optional<UUID> CollectionCatalog::iterator::uuid() { + return _uuid; } CollectionCatalog::iterator CollectionCatalog::iterator::operator++() { - invariant(_mapIter != _map.end()); - invariant(_mapIter != _end); _mapIter++; - _skipUncommitted(); - return *this; -} -bool CollectionCatalog::iterator::operator==(const iterator& other) const { - invariant(_map == other._map); + // Skip any collections that are not yet visible outside of their respective transactions. + while (!_exhausted() && !_mapIter->second->isCommitted()) { + _mapIter++; + } - if (other._mapIter == other._map.end()) { - return _mapIter == _map.end(); - } else if (_mapIter == _map.end()) { - return other._mapIter == other._map.end(); + if (_exhausted()) { + // If the iterator is at the end of the map or now points to an entry that does not + // correspond to the correct database. + _mapIter = _catalog->_orderedCollections.end(); + _uuid = boost::none; + return *this; } - return _mapIter->first.second == other._mapIter->first.second; + _uuid = _mapIter->first.second; + return *this; } -bool CollectionCatalog::iterator::operator!=(const iterator& other) const { - return !(*this == other); +CollectionCatalog::iterator CollectionCatalog::iterator::operator++(int) { + auto oldPosition = *this; + ++(*this); + return oldPosition; } -void CollectionCatalog::iterator::_skipUncommitted() { - // Advance to the next collection that is visible outside of its transaction. - while (_mapIter != _end && !_mapIter->second->isCommitted()) { - ++_mapIter; +bool CollectionCatalog::iterator::operator==(const iterator& other) const { + invariant(_catalog == other._catalog); + if (other._mapIter == _catalog->_orderedCollections.end()) { + return _uuid == boost::none; } -} -CollectionCatalog::Range::Range(const OrderedCollectionMap& map, - const TenantDatabaseName& tenantDbName) - : _map{map}, _tenantDbName{tenantDbName} {} - -CollectionCatalog::iterator CollectionCatalog::Range::begin() const { - return {_tenantDbName, _map.lower_bound(std::make_pair(_tenantDbName, minUuid)), _map}; + return _uuid == other._uuid; } -CollectionCatalog::iterator CollectionCatalog::Range::end() const { - return {_tenantDbName, _map.upper_bound(std::make_pair(_tenantDbName, maxUuid)), _map}; +bool CollectionCatalog::iterator::operator!=(const iterator& other) const { + return !(*this == other); } -bool CollectionCatalog::Range::empty() const { - return begin() == end(); +bool CollectionCatalog::iterator::_exhausted() { + return _mapIter == _catalog->_orderedCollections.end() || + _mapIter->first.first != _tenantDbName; } std::shared_ptr<const CollectionCatalog> CollectionCatalog::get(ServiceContext* svcCtx) { @@ -435,13 +458,6 @@ void CollectionCatalog::write(ServiceContext* svcCtx, CatalogWriteFn job) { void CollectionCatalog::write(OperationContext* opCtx, std::function<void(CollectionCatalog&)> job) { - // Calling the writer must be done with the GlobalLock held. Otherwise we risk having the - // BatchedCollectionCatalogWriter and this caller concurrently modifying the catalog. This is - // because normal operations calling this will all be serialized, but - // BatchedCollectionCatalogWriter skips this mechanism as it knows it is the sole user of the - // server by holding a Global MODE_X lock. - invariant(opCtx->lockState()->isNoop() || opCtx->lockState()->isLocked()); - // If global MODE_X lock are held we can re-use a cloned CollectionCatalog instance when // 'batchedCatalogWriteInstance' is set. Make sure we are the one holding the write lock. if (batchedCatalogWriteInstance) { @@ -453,33 +469,25 @@ void CollectionCatalog::write(OperationContext* opCtx, write(opCtx->getServiceContext(), std::move(job)); } -Status CollectionCatalog::createView(OperationContext* opCtx, - const NamespaceString& viewName, - const NamespaceString& viewOn, - const BSONArray& pipeline, - const BSONObj& collation, - const ViewsForDatabase::PipelineValidatorFn& pipelineValidator, - const ViewUpsertMode insertViewMode) const { - // A view document direct write can occur via the oplog application path, which may only hold a - // lock on the collection being updated (the database views collection). - invariant(insertViewMode == ViewUpsertMode::kAlreadyDurableView || - opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX)); +Status CollectionCatalog::createView( + OperationContext* opCtx, + const NamespaceString& viewName, + const NamespaceString& viewOn, + const BSONArray& pipeline, + const BSONObj& collation, + const ViewsForDatabase::PipelineValidatorFn& pipelineValidator) const { + invariant(opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX)); invariant(opCtx->lockState()->isCollectionLockedForMode( NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X)); - invariant(_viewsForDatabase.find(viewName.db())); + invariant(_viewsForDatabase.contains(viewName.db())); const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db()); - auto& uncommittedCatalogUpdates = UncommittedCatalogUpdates::get(opCtx); - if (uncommittedCatalogUpdates.shouldIgnoreExternalViewChanges(viewName.db())) { - return Status::OK(); - } - if (viewName.db() != viewOn.db()) return Status(ErrorCodes::BadValue, "View must be created on a view or collection in the same database"); - if (viewsForDb.lookup(viewName) || _collections.find(viewName)) + if (viewsForDb.lookup(viewName) || _collections.contains(viewName)) return Status(ErrorCodes::NamespaceExists, "Namespace already exists"); if (!NamespaceString::validCollectionName(viewOn.coll())) @@ -500,8 +508,7 @@ Status CollectionCatalog::createView(OperationContext* opCtx, pipeline, pipelineValidator, std::move(collator.getValue()), - ViewsForDatabase{viewsForDb}, - insertViewMode); + ViewsForDatabase{viewsForDb}); } return result; @@ -517,7 +524,7 @@ Status CollectionCatalog::modifyView( invariant(opCtx->lockState()->isCollectionLockedForMode( NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X)); - invariant(_viewsForDatabase.find(viewName.db())); + invariant(_viewsForDatabase.contains(viewName.db())); const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db()); if (viewName.db() != viewOn.db()) @@ -543,8 +550,7 @@ Status CollectionCatalog::modifyView( pipeline, pipelineValidator, CollatorInterface::cloneCollator(viewPtr->defaultCollator()), - ViewsForDatabase{viewsForDb}, - ViewUpsertMode::kUpdateView); + ViewsForDatabase{viewsForDb}); } return result; @@ -555,12 +561,12 @@ Status CollectionCatalog::dropView(OperationContext* opCtx, const NamespaceStrin invariant(opCtx->lockState()->isCollectionLockedForMode( NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X)); - invariant(_viewsForDatabase.find(viewName.db())); + invariant(_viewsForDatabase.contains(viewName.db())); const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db()); viewsForDb.requireValidCatalog(); // Make sure the view exists before proceeding. - if (!viewsForDb.lookup(viewName)) { + if (auto viewPtr = viewsForDb.lookup(viewName); !viewPtr) { return {ErrorCodes::NamespaceNotFound, str::stream() << "cannot drop missing view: " << viewName.ns()}; } @@ -604,10 +610,9 @@ Status CollectionCatalog::reloadViews(OperationContext* opCtx, StringData dbName // Create a copy of the ViewsForDatabase instance to modify it. Reset the views for this // database, but preserve the DurableViewCatalog pointer. - const ViewsForDatabase* viewsForDbPtr = _viewsForDatabase.find(dbName); - invariant(viewsForDbPtr); - ViewsForDatabase viewsForDb = *viewsForDbPtr; - + auto it = _viewsForDatabase.find(dbName); + invariant(it != _viewsForDatabase.end()); + ViewsForDatabase viewsForDb{it->second.durable}; viewsForDb.valid = false; viewsForDb.viewGraphNeedsRefresh = true; viewsForDb.viewMap.clear(); @@ -672,29 +677,28 @@ void CollectionCatalog::onOpenDatabase(OperationContext* opCtx, invariant(opCtx->lockState()->isDbLockedForMode(dbName, MODE_IS)); uassert(ErrorCodes::AlreadyInitialized, str::stream() << "Database " << dbName << " is already initialized", - !_viewsForDatabase.find(dbName)); + _viewsForDatabase.find(dbName) == _viewsForDatabase.end()); - _viewsForDatabase = _viewsForDatabase.set(dbName.toString(), std::move(viewsForDb)); + _viewsForDatabase[dbName] = std::move(viewsForDb); } void CollectionCatalog::onCloseDatabase(OperationContext* opCtx, TenantDatabaseName tenantDbName) { invariant(opCtx->lockState()->isDbLockedForMode(tenantDbName.dbName(), MODE_X)); auto rid = ResourceId(RESOURCE_DATABASE, tenantDbName.dbName()); removeResource(rid, tenantDbName.dbName()); - _viewsForDatabase = _viewsForDatabase.erase(tenantDbName.dbName()); + _viewsForDatabase.erase(tenantDbName.dbName()); } -void CollectionCatalog::onCloseCatalog() { - if (_shadowCatalog) { - return; - } - +void CollectionCatalog::onCloseCatalog(OperationContext* opCtx) { + invariant(opCtx->lockState()->isW()); + invariant(!_shadowCatalog); _shadowCatalog.emplace(); for (auto& entry : _catalog) - _shadowCatalog = _shadowCatalog->insert({entry.first, entry.second->ns()}); + _shadowCatalog->insert({entry.first, entry.second->ns()}); } -void CollectionCatalog::onOpenCatalog() { +void CollectionCatalog::onOpenCatalog(OperationContext* opCtx) { + invariant(opCtx->lockState()->isW()); invariant(_shadowCatalog); _shadowCatalog.reset(); ++_epoch; @@ -704,10 +708,6 @@ uint64_t CollectionCatalog::getEpoch() const { return _epoch; } -CollectionCatalog::Range CollectionCatalog::range(const TenantDatabaseName& tenantDbName) const { - return {_orderedCollections, tenantDbName}; -} - std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByUUIDForRead( OperationContext* opCtx, const UUID& uuid) const { auto [found, uncommittedColl, newColl] = @@ -761,7 +761,6 @@ Collection* CollectionCatalog::lookupCollectionByUUIDForMetadataWrite(OperationC // on the thread doing the batch write and it would trigger the regular path where we do a // copy-on-write on the catalog when committing. if (_isCatalogBatchWriter()) { - batchedCatalogClonedCollections.emplace(cloned.get()); PublishCatalogUpdates::setCollectionInCatalog(*batchedCatalogWriteInstance, std::move(cloned)); return ptr; @@ -795,8 +794,8 @@ bool CollectionCatalog::isCollectionAwaitingVisibility(UUID uuid) const { } std::shared_ptr<Collection> CollectionCatalog::_lookupCollectionByUUID(UUID uuid) const { - const std::shared_ptr<Collection>* coll = _catalog.find(uuid); - return coll ? *coll : nullptr; + auto foundIt = _catalog.find(uuid); + return foundIt == _catalog.end() ? nullptr : foundIt->second; } std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByNamespaceForRead( @@ -813,8 +812,8 @@ std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByNamespace return nullptr; } - const std::shared_ptr<Collection>* collPtr = _collections.find(nss); - auto coll = collPtr ? *collPtr : nullptr; + auto it = _collections.find(nss); + auto coll = (it == _collections.end() ? nullptr : it->second); return (coll && coll->isCommitted()) ? coll : nullptr; } @@ -844,8 +843,8 @@ Collection* CollectionCatalog::lookupCollectionByNamespaceForMetadataWrite( return nullptr; } - const std::shared_ptr<Collection>* collPtr = _collections.find(nss); - auto coll = collPtr ? *collPtr : nullptr; + auto it = _collections.find(nss); + auto coll = (it == _collections.end() ? nullptr : it->second); if (!coll || !coll->isCommitted()) return nullptr; @@ -865,7 +864,6 @@ Collection* CollectionCatalog::lookupCollectionByNamespaceForMetadataWrite( // on the thread doing the batch write and it would trigger the regular path where we do a // copy-on-write on the catalog when committing. if (_isCatalogBatchWriter()) { - batchedCatalogClonedCollections.emplace(cloned.get()); PublishCatalogUpdates::setCollectionInCatalog(*batchedCatalogWriteInstance, std::move(cloned)); return ptr; @@ -892,8 +890,8 @@ CollectionPtr CollectionCatalog::lookupCollectionByNamespace(OperationContext* o return nullptr; } - const std::shared_ptr<Collection>* collPtr = _collections.find(nss); - auto coll = collPtr ? *collPtr : nullptr; + auto it = _collections.find(nss); + auto coll = (it == _collections.end() ? nullptr : it->second); return (coll && coll->isCommitted()) ? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns())) : nullptr; @@ -911,21 +909,20 @@ boost::optional<NamespaceString> CollectionCatalog::lookupNSSByUUID(OperationCon return boost::none; } - const std::shared_ptr<Collection>* collPtr = _catalog.find(uuid); - if (collPtr) { - auto coll = *collPtr; - boost::optional<NamespaceString> ns = coll->ns(); - invariant(!ns.value().isEmpty()); - return coll->isCommitted() ? ns : boost::none; + auto foundIt = _catalog.find(uuid); + if (foundIt != _catalog.end()) { + boost::optional<NamespaceString> ns = foundIt->second->ns(); + invariant(!ns.get().isEmpty()); + return _collections.find(ns.get())->second->isCommitted() ? ns : boost::none; } // Only in the case that the catalog is closed and a UUID is currently unknown, resolve it // using the pre-close state. This ensures that any tasks reloading the catalog can see their // own updates. if (_shadowCatalog) { - auto* shadowIt = _shadowCatalog->find(uuid); - if (shadowIt) - return *shadowIt; + auto shadowIt = _shadowCatalog->find(uuid); + if (shadowIt != _shadowCatalog->end()) + return shadowIt->second; } return boost::none; } @@ -941,11 +938,10 @@ boost::optional<UUID> CollectionCatalog::lookupUUIDByNSS(OperationContext* opCtx return boost::none; } - const std::shared_ptr<Collection>* collPtr = _collections.find(nss); - if (collPtr) { - auto coll = *collPtr; - const boost::optional<UUID>& uuid = coll->uuid(); - return coll->isCommitted() ? uuid : boost::none; + auto it = _collections.find(nss); + if (it != _collections.end()) { + const boost::optional<UUID>& uuid = it->second->uuid(); + return it->second->isCommitted() ? uuid : boost::none; } return boost::none; } @@ -1090,34 +1086,23 @@ std::vector<TenantDatabaseName> CollectionCatalog::getAllDbNames() const { return ret; } -void CollectionCatalog::setAllDatabaseProfileFilters(std::shared_ptr<ProfileFilter> filter) { - auto dbProfileSettingsWriter = _databaseProfileSettings.transient(); - for (const auto& [dbName, settings] : _databaseProfileSettings) { - ProfileSettings clone = settings; - clone.filter = filter; - dbProfileSettingsWriter.set(dbName, std::move(clone)); - } - _databaseProfileSettings = dbProfileSettingsWriter.persistent(); -} - void CollectionCatalog::setDatabaseProfileSettings( StringData dbName, CollectionCatalog::ProfileSettings newProfileSettings) { - _databaseProfileSettings = - _databaseProfileSettings.set(dbName.toString(), std::move(newProfileSettings)); + _databaseProfileSettings[dbName] = newProfileSettings; } CollectionCatalog::ProfileSettings CollectionCatalog::getDatabaseProfileSettings( StringData dbName) const { - const ProfileSettings* settings = _databaseProfileSettings.find(dbName); - if (settings) { - return *settings; + auto it = _databaseProfileSettings.find(dbName); + if (it != _databaseProfileSettings.end()) { + return it->second; } return {serverGlobalParams.defaultProfile, ProfileFilter::getDefault()}; } void CollectionCatalog::clearDatabaseProfileSettings(StringData dbName) { - _databaseProfileSettings = _databaseProfileSettings.erase(dbName.toString()); + _databaseProfileSettings.erase(dbName); } CollectionCatalog::Stats CollectionCatalog::getStats() const { @@ -1145,9 +1130,9 @@ CollectionCatalog::ViewCatalogSet CollectionCatalog::getViewCatalogDbNames( } void CollectionCatalog::registerCollection(OperationContext* opCtx, + const UUID& uuid, std::shared_ptr<Collection> coll) { auto nss = coll->ns(); - auto uuid = coll->uuid(); // TODO SERVER-64608 Use tenantId from nss auto tenantDbName = TenantDatabaseName(boost::none, nss.db()); _ensureNamespaceDoesNotExist(opCtx, nss, NamespaceType::kAll); @@ -1162,12 +1147,12 @@ void CollectionCatalog::registerCollection(OperationContext* opCtx, auto dbIdPair = std::make_pair(tenantDbName, uuid); // Make sure no entry related to this uuid. - invariant(!_catalog.find(uuid)); + invariant(_catalog.find(uuid) == _catalog.end()); invariant(_orderedCollections.find(dbIdPair) == _orderedCollections.end()); - _catalog = _catalog.set(uuid, coll); - _collections = _collections.set(nss, coll); - _orderedCollections = _orderedCollections.set(dbIdPair, coll); + _catalog[uuid] = coll; + _collections[nss] = coll; + _orderedCollections[dbIdPair] = coll; if (!nss.isOnInternalDb() && !nss.isSystem()) { _stats.userCollections += 1; @@ -1193,7 +1178,7 @@ void CollectionCatalog::registerCollection(OperationContext* opCtx, std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(OperationContext* opCtx, const UUID& uuid) { - invariant(_catalog.find(uuid)); + invariant(_catalog.find(uuid) != _catalog.end()); auto coll = std::move(_catalog[uuid]); auto ns = coll->ns(); @@ -1204,12 +1189,12 @@ std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(OperationCon LOGV2_DEBUG(20281, 1, "Deregistering collection", logAttrs(ns), "uuid"_attr = uuid); // Make sure collection object exists. - invariant(_collections.find(ns)); + invariant(_collections.find(ns) != _collections.end()); invariant(_orderedCollections.find(dbIdPair) != _orderedCollections.end()); - _orderedCollections = _orderedCollections.erase(dbIdPair); - _collections = _collections.erase(ns); - _catalog = _catalog.erase(uuid); + _orderedCollections.erase(dbIdPair); + _collections.erase(ns); + _catalog.erase(uuid); if (!ns.isOnInternalDb() && !ns.isSystem()) { _stats.userCollections -= 1; @@ -1242,18 +1227,18 @@ void CollectionCatalog::registerUncommittedView(OperationContext* opCtx, // namespaces here. _ensureNamespaceDoesNotExist(opCtx, nss, NamespaceType::kCollection); - _uncommittedViews = _uncommittedViews.insert(nss); + _uncommittedViews.emplace(nss); } void CollectionCatalog::deregisterUncommittedView(const NamespaceString& nss) { - _uncommittedViews = _uncommittedViews.erase(nss); + _uncommittedViews.erase(nss); } void CollectionCatalog::_ensureNamespaceDoesNotExist(OperationContext* opCtx, const NamespaceString& nss, NamespaceType type) const { auto existingCollection = _collections.find(nss); - if (existingCollection) { + if (existingCollection != _collections.end()) { LOGV2(5725001, "Conflicted registering namespace, already have a collection with the same namespace", "nss"_attr = nss); @@ -1261,7 +1246,7 @@ void CollectionCatalog::_ensureNamespaceDoesNotExist(OperationContext* opCtx, } if (type == NamespaceType::kAll) { - if (_uncommittedViews.find(nss)) { + if (_uncommittedViews.contains(nss)) { LOGV2(5725002, "Conflicted registering namespace, already have a view with the same namespace", "nss"_attr = nss); @@ -1289,25 +1274,26 @@ void CollectionCatalog::deregisterAllCollectionsAndViews() { auto ns = entry.second->ns(); LOGV2_DEBUG(20283, 1, "Deregistering collection", logAttrs(ns), "uuid"_attr = uuid); + + entry.second.reset(); } - _collections = {}; - _orderedCollections = {}; - _catalog = {}; - _viewsForDatabase = {}; + _collections.clear(); + _orderedCollections.clear(); + _catalog.clear(); + _viewsForDatabase.clear(); _stats = {}; - _resourceInformation = {}; + _resourceInformation.clear(); } void CollectionCatalog::clearViews(OperationContext* opCtx, StringData dbName) const { invariant(opCtx->lockState()->isCollectionLockedForMode( NamespaceString(dbName, NamespaceString::kSystemDotViewsCollectionName), MODE_X)); - const ViewsForDatabase* viewsForDbPtr = _viewsForDatabase.find(dbName); - invariant(viewsForDbPtr); - - ViewsForDatabase viewsForDb = *viewsForDbPtr; + auto it = _viewsForDatabase.find(dbName); + invariant(it != _viewsForDatabase.end()); + ViewsForDatabase viewsForDb = it->second; viewsForDb.viewMap.clear(); viewsForDb.viewGraph.clear(); @@ -1318,6 +1304,16 @@ void CollectionCatalog::clearViews(OperationContext* opCtx, StringData dbName) c catalog._replaceViewsForDatabase(dbName, std::move(viewsForDb)); }); } + +CollectionCatalog::iterator CollectionCatalog::begin(OperationContext* opCtx, + const TenantDatabaseName& tenantDbName) const { + return iterator(opCtx, tenantDbName, *this); +} + +CollectionCatalog::iterator CollectionCatalog::end(OperationContext* opCtx) const { + return iterator(opCtx, _orderedCollections.end(), *this); +} + boost::optional<std::string> CollectionCatalog::lookupResourceName(const ResourceId& rid) const { invariant(rid.getType() == RESOURCE_DATABASE || rid.getType() == RESOURCE_COLLECTION); @@ -1325,6 +1321,7 @@ boost::optional<std::string> CollectionCatalog::lookupResourceName(const Resourc if (search == _resourceInformation.end()) { return boost::none; } + const std::set<std::string>& namespaces = search->second; // When there are multiple namespaces mapped to the same ResourceId, return boost::none as the @@ -1344,14 +1341,12 @@ void CollectionCatalog::removeResource(const ResourceId& rid, const std::string& return; } - std::set<std::string> namespaces = search->second; + std::set<std::string>& namespaces = search->second; namespaces.erase(entry); // Remove the map entry if this is the last namespace in the set for the ResourceId. if (namespaces.size() == 0) { - _resourceInformation = _resourceInformation.erase(search, rid); - } else { - _resourceInformation = _resourceInformation.set(rid, std::move(namespaces)); + _resourceInformation.erase(search); } } @@ -1361,17 +1356,16 @@ void CollectionCatalog::addResource(const ResourceId& rid, const std::string& en auto search = _resourceInformation.find(rid); if (search == _resourceInformation.end()) { std::set<std::string> newSet = {entry}; - _resourceInformation = _resourceInformation.set(rid, std::move(newSet)); + _resourceInformation.insert(std::make_pair(rid, newSet)); return; } - if (const auto& namespaces = search->second; namespaces.count(entry) > 0) { + std::set<std::string>& namespaces = search->second; + if (namespaces.count(entry) > 0) { return; } - std::set<std::string> namespaces = search->second; namespaces.insert(entry); - _resourceInformation = _resourceInformation.set(rid, std::move(namespaces)); } void CollectionCatalog::invariantHasExclusiveAccessToCollection(OperationContext* opCtx, @@ -1391,15 +1385,15 @@ boost::optional<const ViewsForDatabase&> CollectionCatalog::_getViewsForDatabase return uncommittedViews; } - const ViewsForDatabase* viewsForDb = _viewsForDatabase.find(dbName); - if (!viewsForDb) { + auto it = _viewsForDatabase.find(dbName); + if (it == _viewsForDatabase.end()) { return boost::none; } - return *viewsForDb; + return it->second; } void CollectionCatalog::_replaceViewsForDatabase(StringData dbName, ViewsForDatabase&& views) { - _viewsForDatabase = _viewsForDatabase.set(dbName.toString(), std::move(views)); + _viewsForDatabase[dbName] = std::move(views); } Status CollectionCatalog::_createOrUpdateView( @@ -1409,19 +1403,15 @@ Status CollectionCatalog::_createOrUpdateView( const BSONArray& pipeline, const ViewsForDatabase::PipelineValidatorFn& pipelineValidator, std::unique_ptr<CollatorInterface> collator, - ViewsForDatabase&& viewsForDb, - ViewUpsertMode insertViewMode) const { - // A view document direct write can occur via the oplog application path, which may only hold a - // lock on the collection being updated (the database views collection). - invariant(insertViewMode == ViewUpsertMode::kAlreadyDurableView || - opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX)); + ViewsForDatabase&& viewsForDb) const { + invariant(opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX)); invariant(opCtx->lockState()->isCollectionLockedForMode( NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X)); viewsForDb.requireValidCatalog(); - // Build the BSON definition for this view to be saved in the durable view catalog and/or to - // insert in the viewMap. If the collation is empty, omit it from the definition altogether. + // Build the BSON definition for this view to be saved in the durable view catalog. If the + // collation is empty, omit it from the definition altogether. BSONObjBuilder viewDefBuilder; viewDefBuilder.append("_id", viewName.ns()); viewDefBuilder.append("viewOn", viewOn.coll()); @@ -1430,42 +1420,25 @@ Status CollectionCatalog::_createOrUpdateView( viewDefBuilder.append("collation", collator->getSpec().toBSON()); } - BSONObj viewDef = viewDefBuilder.obj(); BSONObj ownedPipeline = pipeline.getOwned(); - ViewDefinition view( + auto view = std::make_shared<ViewDefinition>( viewName.db(), viewName.coll(), viewOn.coll(), ownedPipeline, std::move(collator)); - // If the view is already in the durable view catalog, we don't need to validate the graph. If - // we need to update the durable view catalog, we need to check that the resulting dependency - // graph is acyclic and within the maximum depth. - const bool viewGraphNeedsValidation = insertViewMode != ViewUpsertMode::kAlreadyDurableView; - Status graphStatus = - viewsForDb.upsertIntoGraph(opCtx, view, pipelineValidator, viewGraphNeedsValidation); + // Check that the resulting dependency graph is acyclic and within the maximum depth. + Status graphStatus = viewsForDb.upsertIntoGraph(opCtx, *(view.get()), pipelineValidator); if (!graphStatus.isOK()) { return graphStatus; } - if (insertViewMode != ViewUpsertMode::kAlreadyDurableView) { - viewsForDb.durable->upsert(opCtx, viewName, viewDef); - } + viewsForDb.durable->upsert(opCtx, viewName, viewDefBuilder.obj()); + viewsForDb.viewMap.clear(); viewsForDb.valid = false; - auto res = [&] { - switch (insertViewMode) { - case ViewUpsertMode::kCreateView: - case ViewUpsertMode::kAlreadyDurableView: - return viewsForDb.insert(opCtx, viewDef); - case ViewUpsertMode::kUpdateView: - viewsForDb.viewMap.clear(); - viewsForDb.viewGraphNeedsRefresh = true; - viewsForDb.stats = {}; - - // Reload the view catalog with the changes applied. - return viewsForDb.reload(opCtx); - } - MONGO_UNREACHABLE; - }(); + viewsForDb.viewGraphNeedsRefresh = true; + viewsForDb.stats = {}; + // Reload the view catalog with the changes applied. + auto res = viewsForDb.reload(opCtx); if (res.isOK()) { auto& uncommittedCatalogUpdates = UncommittedCatalogUpdates::get(opCtx); uncommittedCatalogUpdates.addView(opCtx, viewName); @@ -1484,9 +1457,16 @@ bool CollectionCatalog::_isCatalogBatchWriter() const { bool CollectionCatalog::_alreadyClonedForBatchedWriter( const std::shared_ptr<Collection>& collection) const { - // We may skip cloning the Collection instance if and only if have already cloned it for write - // use in this batch writer. - return _isCatalogBatchWriter() && batchedCatalogClonedCollections.contains(collection.get()); + // We may skip cloning the Collection instance if and only if we are currently in a batched + // catalog write and all references to this Collection is owned by the cloned CollectionCatalog + // instance owned by the batch writer. i.e. the Collection is uniquely owned by the batch + // writer. When the batch writer initially clones the catalog, all collections will have a + // 'use_count' of at least kNumCollectionReferencesStored*2 (because there are at least 2 + // catalog instances). To check for uniquely owned we need to check that the reference count is + // exactly kNumCollectionReferencesStored (owned by a single catalog) while also account for the + // instance that is extracted from the catalog and provided as a parameter to this function, we + // therefore need to add 1. + return _isCatalogBatchWriter() && collection.use_count() == kNumCollectionReferencesStored + 1; } CollectionCatalogStasher::CollectionCatalogStasher(OperationContext* opCtx) @@ -1547,7 +1527,10 @@ const Collection* LookupCollectionForYieldRestore::operator()(OperationContext* // state. After a query yields its locks, the replication state may have changed, invalidating // our current choice of ReadSource. Using the same preconditions, change our ReadSource if // necessary. - SnapshotHelper::changeReadSourceIfNeeded(opCtx, collection->ns()); + auto [newReadSource, _] = SnapshotHelper::shouldChangeReadSource(opCtx, collection->ns()); + if (newReadSource) { + opCtx->recoveryUnit()->setTimestampReadSource(*newReadSource); + } return collection.get(); } @@ -1556,7 +1539,6 @@ BatchedCollectionCatalogWriter::BatchedCollectionCatalogWriter(OperationContext* : _opCtx(opCtx) { invariant(_opCtx->lockState()->isW()); invariant(!batchedCatalogWriteInstance); - invariant(batchedCatalogClonedCollections.empty()); auto& storage = getCatalog(_opCtx->getServiceContext()); // hold onto base so if we need to delete it we can do it outside of the lock @@ -1579,7 +1561,6 @@ BatchedCollectionCatalogWriter::~BatchedCollectionCatalogWriter() { // Clear out batched pointer so no more attempts of batching are made _batchedInstance = nullptr; batchedCatalogWriteInstance = nullptr; - batchedCatalogClonedCollections.clear(); } } // namespace mongo diff --git a/src/mongo/db/catalog/collection_catalog.h b/src/mongo/db/catalog/collection_catalog.h index 01727543a4e..f47f94397f9 100644 --- a/src/mongo/db/catalog/collection_catalog.h +++ b/src/mongo/db/catalog/collection_catalog.h @@ -40,10 +40,6 @@ #include "mongo/db/tenant_database_name.h" #include "mongo/db/views/view.h" #include "mongo/stdx/unordered_map.h" -#include "mongo/util/functional.h" -#include "mongo/util/immutable/map.h" -#include "mongo/util/immutable/unordered_map.h" -#include "mongo/util/immutable/unordered_set.h" #include "mongo/util/uuid.h" namespace mongo { @@ -53,23 +49,31 @@ class Database; class CollectionCatalog { friend class iterator; - using OrderedCollectionMap = - immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>; public: using CollectionInfoFn = std::function<bool(const CollectionPtr& collection)>; using ViewIteratorCallback = std::function<bool(const ViewDefinition& view)>; + // Number of how many Collection references for a single Collection that is stored in the + // catalog. Used to determine whether there are external references (uniquely owned). Needs to + // be kept in sync with the data structures below. + static constexpr size_t kNumCollectionReferencesStored = 3; class iterator { public: using value_type = CollectionPtr; - iterator(const TenantDatabaseName& tenantDbName, - OrderedCollectionMap::iterator it, - const OrderedCollectionMap& catalog); + iterator(OperationContext* opCtx, + const TenantDatabaseName& tenantDbName, + const CollectionCatalog& catalog); + iterator(OperationContext* opCtx, + std::map<std::pair<TenantDatabaseName, UUID>, + std::shared_ptr<Collection>>::const_iterator mapIter, + const CollectionCatalog& catalog); value_type operator*(); iterator operator++(); + iterator operator++(int); + boost::optional<UUID> uuid(); Collection* getWritableCollection(OperationContext* opCtx); @@ -81,30 +85,19 @@ public: bool operator!=(const iterator& other) const; private: - void _skipUncommitted(); - - const OrderedCollectionMap& _map; - immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::iterator - _mapIter; - immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::iterator - _end; - }; - - class Range { - public: - Range(const OrderedCollectionMap&, const TenantDatabaseName& tenantDbName); - iterator begin() const; - iterator end() const; - bool empty() const; + bool _exhausted(); - private: - OrderedCollectionMap _map; + OperationContext* _opCtx; TenantDatabaseName _tenantDbName; + boost::optional<UUID> _uuid; + std::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::const_iterator + _mapIter; + const CollectionCatalog* _catalog; }; struct ProfileSettings { int level; - std::shared_ptr<const ProfileFilter> filter; // nullable + std::shared_ptr<ProfileFilter> filter; // nullable ProfileSettings(int level, std::shared_ptr<ProfileFilter> filter) : level(level), filter(filter) { @@ -122,19 +115,6 @@ public: } }; - enum class ViewUpsertMode { - // Insert all data for that view into the view map, view graph, and durable view catalog. - kCreateView, - - // Insert into the view map and view graph without reinserting the view into the durable - // view catalog. Skip view graph validation. - kAlreadyDurableView, - - // Reload the view map, insert into the view graph (flagging it as needing refresh), and - // update the durable view catalog. - kUpdateView, - }; - static std::shared_ptr<const CollectionCatalog> get(ServiceContext* svcCtx); static std::shared_ptr<const CollectionCatalog> get(OperationContext* opCtx); @@ -167,16 +147,14 @@ public: * * Must be in WriteUnitOfWork. View creation rolls back if the unit of work aborts. * - * Caller must ensure corresponding database exists. Expects db.system.views MODE_X lock and - * view namespace MODE_IX lock (unless 'insertViewMode' is set to kAlreadyDurableView). + * Caller must ensure corresponding database exists. */ Status createView(OperationContext* opCtx, const NamespaceString& viewName, const NamespaceString& viewOn, const BSONArray& pipeline, const BSONObj& collation, - const ViewsForDatabase::PipelineValidatorFn& pipelineValidator, - ViewUpsertMode insertViewMode = ViewUpsertMode::kCreateView) const; + const ViewsForDatabase::PipelineValidatorFn& pipelineValidator) const; /** * Drop the view named 'viewName'. @@ -257,7 +235,9 @@ public: /** * Register the collection with `uuid`. */ - void registerCollection(OperationContext* opCtx, std::shared_ptr<Collection> collection); + void registerCollection(OperationContext* opCtx, + const UUID& uuid, + std::shared_ptr<Collection> collection); /** * Deregister the collection. @@ -426,11 +406,6 @@ public: std::vector<TenantDatabaseName> getAllDbNames() const; /** - * Updates the profile filter on all databases with non-default settings. - */ - void setAllDatabaseProfileFilters(std::shared_ptr<ProfileFilter> filter); - - /** * Sets 'newProfileSettings' as the profiling settings for the database 'dbName'. */ void setDatabaseProfileSettings(StringData dbName, ProfileSettings newProfileSettings); @@ -499,14 +474,14 @@ public: * * Must be called with the global lock acquired in exclusive mode. */ - void onCloseCatalog(); + void onCloseCatalog(OperationContext* opCtx); /** * Puts the catalog back in open state, removing the pre-close state. See onCloseCatalog. * * Must be called with the global lock acquired in exclusive mode. */ - void onOpenCatalog(); + void onOpenCatalog(OperationContext* opCtx); /** * The epoch is incremented whenever the catalog is closed and re-opened. @@ -519,13 +494,8 @@ public: */ uint64_t getEpoch() const; - /** - * Provides an iterable range for the collections belonging to the specified database. - * - * Will not observe any updates made to the catalog after the creation of the 'Range'. The - * 'Range' object just remain in scope for the duration of the iteration. - */ - Range range(const TenantDatabaseName& tenantDbName) const; + iterator begin(OperationContext* opCtx, const TenantDatabaseName& tenantDbName) const; + iterator end(OperationContext* opCtx) const; /** * Lookup the name of a resource by its ResourceId. If there are multiple namespaces mapped to @@ -578,8 +548,7 @@ private: const BSONArray& pipeline, const ViewsForDatabase::PipelineValidatorFn& pipelineValidator, std::unique_ptr<CollatorInterface> collator, - ViewsForDatabase&& viewsForDb, - ViewUpsertMode insertViewMode) const; + ViewsForDatabase&& viewsForDb) const; /** * Returns true if this CollectionCatalog instance is part of an ongoing batched catalog write. @@ -611,17 +580,15 @@ private: * When present, indicates that the catalog is in closed state, and contains a map from UUID * to pre-close NSS. See also onCloseCatalog. */ - boost::optional<immutable::unordered_map<UUID, NamespaceString, UUID::Hash>> _shadowCatalog; + boost::optional<mongo::stdx::unordered_map<UUID, NamespaceString, UUID::Hash>> _shadowCatalog; - using CollectionCatalogMap = - immutable::unordered_map<UUID, std::shared_ptr<Collection>, UUID::Hash>; + using CollectionCatalogMap = stdx::unordered_map<UUID, std::shared_ptr<Collection>, UUID::Hash>; + using OrderedCollectionMap = + std::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>; using NamespaceCollectionMap = - immutable::unordered_map<NamespaceString, std::shared_ptr<Collection>>; - using UncommittedViewsSet = immutable::unordered_set<NamespaceString>; - using DatabaseProfileSettingsMap = - immutable::unordered_map<std::string, ProfileSettings, StringMapHasher, StringMapEq>; - using ViewsForDatabaseMap = - immutable::unordered_map<std::string, ViewsForDatabase, StringMapHasher, StringMapEq>; + stdx::unordered_map<NamespaceString, std::shared_ptr<Collection>>; + using UncommittedViewsSet = stdx::unordered_set<NamespaceString>; + using DatabaseProfileSettingsMap = StringMap<ProfileSettings>; CollectionCatalogMap _catalog; OrderedCollectionMap _orderedCollections; // Ordered by <tenantDbName, collUUID> pair @@ -629,7 +596,7 @@ private: UncommittedViewsSet _uncommittedViews; // Map of database names to their corresponding views and other associated state. - ViewsForDatabaseMap _viewsForDatabase; + StringMap<ViewsForDatabase> _viewsForDatabase; // Incremented whenever the CollectionCatalog gets closed and reopened (onCloseCatalog and // onOpenCatalog). @@ -644,7 +611,7 @@ private: uint64_t _epoch = 0; // Mapping from ResourceId to a set of strings that contains collection and database namespaces. - immutable::map<ResourceId, std::set<std::string>> _resourceInformation; + std::map<ResourceId, std::set<std::string>> _resourceInformation; /** * Contains non-default database profile settings. New collections, current collections and diff --git a/src/mongo/db/catalog/collection_catalog_bm.cpp b/src/mongo/db/catalog/collection_catalog_bm.cpp index 47b24320009..3ff0e3375bd 100644 --- a/src/mongo/db/catalog/collection_catalog_bm.cpp +++ b/src/mongo/db/catalog/collection_catalog_bm.cpp @@ -27,7 +27,6 @@ * it in the license file. */ -#include "mongo/db/tenant_database_name.h" #include <benchmark/benchmark.h> #include "mongo/db/catalog/collection_catalog.h" @@ -79,7 +78,7 @@ void createCollections(OperationContext* opCtx, int numCollections) { for (auto i = 0; i < numCollections; i++) { const NamespaceString nss("collection_catalog_bm", std::to_string(i)); CollectionCatalog::write(opCtx, [&](CollectionCatalog& catalog) { - catalog.registerCollection(opCtx, std::make_shared<CollectionMock>(nss)); + catalog.registerCollection(opCtx, UUID::gen(), std::make_shared<CollectionMock>(nss)); }); } } @@ -93,8 +92,6 @@ void BM_CollectionCatalogWrite(benchmark::State& state) { createCollections(opCtx.get(), state.range(0)); - Lock::GlobalLock lk{opCtx.get(), MODE_IX}; - for (auto _ : state) { benchmark::ClobberMemory(); CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) {}); @@ -117,124 +114,7 @@ void BM_CollectionCatalogWriteBatchedWithGlobalExclusiveLock(benchmark::State& s } } -void BM_CollectionCatalogCreateDropCollection(benchmark::State& state) { - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - - createCollections(opCtx.get(), state.range(0)); - - for (auto _ : state) { - benchmark::ClobberMemory(); - CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) { - const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0))); - const UUID uuid = UUID::gen(); - catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(uuid, nss)); - catalog.deregisterCollection(opCtx.get(), uuid); - }); - } -} - -void BM_CollectionCatalogCreateNCollectionsBatched(benchmark::State& state) { - for (auto _ : state) { - benchmark::ClobberMemory(); - - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - - Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - BatchedCollectionCatalogWriter batched(opCtx.get()); - - auto numCollections = state.range(0); - for (auto i = 0; i < numCollections; i++) { - const NamespaceString nss("collection_catalog_bm", std::to_string(i)); - CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) { - catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(nss)); - }); - } - } -} - -void BM_CollectionCatalogCreateNCollections(benchmark::State& state) { - for (auto _ : state) { - benchmark::ClobberMemory(); - - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - - auto numCollections = state.range(0); - for (auto i = 0; i < numCollections; i++) { - const NamespaceString nss("collection_catalog_bm", std::to_string(i)); - CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) { - catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(nss)); - }); - } - } -} - -void BM_CollectionCatalogLookupCollectionByNamespace(benchmark::State& state) { - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - - createCollections(opCtx.get(), state.range(0)); - const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0) / 2)); - - for (auto _ : state) { - benchmark::ClobberMemory(); - auto coll = - CollectionCatalog::get(opCtx.get())->lookupCollectionByNamespace(opCtx.get(), nss); - invariant(coll); - } -} - -void BM_CollectionCatalogLookupCollectionByUUID(benchmark::State& state) { - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - - createCollections(opCtx.get(), state.range(0)); - const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0) / 2)); - auto coll = CollectionCatalog::get(opCtx.get())->lookupCollectionByNamespace(opCtx.get(), nss); - invariant(coll->ns() == nss); - const UUID uuid = coll->uuid(); - - for (auto _ : state) { - benchmark::ClobberMemory(); - auto res = CollectionCatalog::get(opCtx.get())->lookupCollectionByUUID(opCtx.get(), uuid); - invariant(res == coll); - } -} - -void BM_CollectionCatalogIterateCollections(benchmark::State& state) { - auto serviceContext = setupServiceContext(); - ThreadClient threadClient(serviceContext); - ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext(); - - createCollections(opCtx.get(), state.range(0)); - - for (auto _ : state) { - benchmark::ClobberMemory(); - auto catalog = CollectionCatalog::get(opCtx.get()); - auto count = 0; - for ([[maybe_unused]] auto&& coll : - catalog->range(TenantDatabaseName(boost::none, "collection_catalog_bm"))) { - benchmark::DoNotOptimize(count++); - } - } -} - BENCHMARK(BM_CollectionCatalogWrite)->Ranges({{{1}, {100'000}}}); BENCHMARK(BM_CollectionCatalogWriteBatchedWithGlobalExclusiveLock)->Ranges({{{1}, {100'000}}}); -BENCHMARK(BM_CollectionCatalogCreateDropCollection)->Ranges({{{1}, {100'000}}}); -BENCHMARK(BM_CollectionCatalogCreateNCollectionsBatched)->Ranges({{{1}, {100'000}}}); -BENCHMARK(BM_CollectionCatalogCreateNCollections)->Ranges({{{1}, {32'768}}}); -BENCHMARK(BM_CollectionCatalogLookupCollectionByNamespace)->Ranges({{{1}, {100'000}}}); -BENCHMARK(BM_CollectionCatalogLookupCollectionByUUID)->Ranges({{{1}, {100'000}}}); -BENCHMARK(BM_CollectionCatalogIterateCollections)->Ranges({{{1}, {100'000}}}); } // namespace mongo diff --git a/src/mongo/db/catalog/collection_catalog_helper.cpp b/src/mongo/db/catalog/collection_catalog_helper.cpp index ed816f2e1cf..f6db8f69764 100644 --- a/src/mongo/db/catalog/collection_catalog_helper.cpp +++ b/src/mongo/db/catalog/collection_catalog_helper.cpp @@ -66,10 +66,11 @@ void forEachCollectionFromDb(OperationContext* opCtx, CollectionCatalog::CollectionInfoFn predicate) { auto catalogForIteration = CollectionCatalog::get(opCtx); - size_t collectionCount = 0; - for (auto&& coll : catalogForIteration->range(tenantDbName)) { - auto uuid = coll->uuid(); + for (auto collectionIt = catalogForIteration->begin(opCtx, tenantDbName); + collectionIt != catalogForIteration->end(opCtx);) { + auto uuid = collectionIt.uuid().get(); if (predicate && !catalogForIteration->checkIfCollectionSatisfiable(uuid, predicate)) { + ++collectionIt; continue; } @@ -93,6 +94,10 @@ void forEachCollectionFromDb(OperationContext* opCtx, clk.reset(); } + // Increment iterator before calling callback. This allows for collection deletion inside + // this callback even if we are in batched inplace mode. + ++collectionIt; + // The NamespaceString couldn't be resolved from the uuid, so the collection was dropped. if (!collection) continue; @@ -100,14 +105,7 @@ void forEachCollectionFromDb(OperationContext* opCtx, if (!callback(collection)) break; - // This was a rough heuristic that was found that 400 collections would take 100 - // milliseconds with calling checkForInterrupt() (with freeStorage: 1). - // We made the checkForInterrupt() occur after 200 collections to be conservative. - if (!(collectionCount % 200)) { - opCtx->checkForInterrupt(); - } hangBeforeGettingNextCollection.pauseWhileSet(); - collectionCount += 1; } } diff --git a/src/mongo/db/catalog/collection_catalog_test.cpp b/src/mongo/db/catalog/collection_catalog_test.cpp index d05913ac7bf..e28bcd4ad28 100644 --- a/src/mongo/db/catalog/collection_catalog_test.cpp +++ b/src/mongo/db/catalog/collection_catalog_test.cpp @@ -72,7 +72,12 @@ public: std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(colUUID, nss); col = CollectionPtr(collection.get(), CollectionPtr::NoYieldTag{}); // Register dummy collection in catalog. - catalog.registerCollection(opCtx.get(), collection); + catalog.registerCollection(opCtx.get(), colUUID, collection); + + // Validate that kNumCollectionReferencesStored is correct, add one reference for the one we + // hold in this function. + ASSERT_EQUALS(collection.use_count(), + CollectionCatalog::kNumCollectionReferencesStored + 1); } protected: @@ -93,14 +98,17 @@ public: NamespaceString fooNss("foo", "coll" + std::to_string(counter)); NamespaceString barNss("bar", "coll" + std::to_string(counter)); + auto fooUuid = UUID::gen(); std::shared_ptr<Collection> fooColl = std::make_shared<CollectionMock>(fooNss); + + auto barUuid = UUID::gen(); std::shared_ptr<Collection> barColl = std::make_shared<CollectionMock>(barNss); - dbMap["foo"].insert(std::make_pair(fooColl->uuid(), fooColl.get())); - dbMap["bar"].insert(std::make_pair(barColl->uuid(), barColl.get())); + dbMap["foo"].insert(std::make_pair(fooUuid, fooColl.get())); + dbMap["bar"].insert(std::make_pair(barUuid, barColl.get())); - catalog.registerCollection(&opCtx, fooColl); - catalog.registerCollection(&opCtx, barColl); + catalog.registerCollection(&opCtx, fooUuid, fooColl); + catalog.registerCollection(&opCtx, barUuid, barColl); } } @@ -126,10 +134,10 @@ public: void checkCollections(const TenantDatabaseName& tenantDbName) { unsigned long counter = 0; - auto orderedIt = collsIterator(tenantDbName.dbName()); - auto catalogRange = catalog.range(tenantDbName); - auto catalogIt = catalogRange.begin(); - for (; catalogIt != catalogRange.end() && + + for (auto [orderedIt, catalogIt] = std::tuple{collsIterator(tenantDbName.dbName()), + catalog.begin(&opCtx, tenantDbName)}; + catalogIt != catalog.end(&opCtx) && orderedIt != collsIteratorEnd(tenantDbName.dbName()); ++catalogIt, ++orderedIt) { @@ -277,13 +285,17 @@ public: for (int i = 0; i < 5; i++) { NamespaceString nss("resourceDb", "coll" + std::to_string(i)); std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(nss); + auto uuid = collection->uuid(); - catalog.registerCollection(&opCtx, std::move(collection)); + catalog.registerCollection(&opCtx, uuid, std::move(collection)); } int numEntries = 0; - for (auto&& coll : catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) { - auto collName = coll->ns().ns(); + for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb")); + it != catalog.end(&opCtx); + it++) { + auto coll = *it; + std::string collName = coll->ns().ns(); ResourceId rid(RESOURCE_COLLECTION, collName); ASSERT_NE(catalog.lookupResourceName(rid), boost::none); @@ -294,7 +306,10 @@ public: void tearDown() { std::vector<UUID> collectionsToDeregister; - for (auto&& coll : catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) { + for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb")); + it != catalog.end(&opCtx); + ++it) { + auto coll = *it; auto uuid = coll->uuid(); if (!coll) { break; @@ -308,8 +323,9 @@ public: } int numEntries = 0; - for ([[maybe_unused]] auto&& coll : - catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) { + for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb")); + it != catalog.end(&opCtx); + it++) { numEntries++; } ASSERT_EQ(0, numEntries); @@ -386,14 +402,13 @@ TEST_F(CollectionCatalogIterationTest, EndAtEndOfSection) { } TEST_F(CollectionCatalogIterationTest, GetUUIDWontRepositionEvenIfEntryIsDropped) { - auto range = catalog.range(TenantDatabaseName(boost::none, "bar")); - auto it = range.begin(); + auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "bar")); auto collsIt = collsIterator("bar"); auto uuid = collsIt->first; catalog.deregisterCollection(&opCtx, uuid); dropColl("bar", uuid); - ASSERT_EQUALS(uuid, (*it)->uuid()); + ASSERT_EQUALS(uuid, it.uuid()); } TEST_F(CollectionCatalogTest, OnCreateCollection) { @@ -418,13 +433,13 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUID) { TEST_F(CollectionCatalogTest, InsertAfterLookup) { auto newUUID = UUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newUUID, newNss); + std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss); auto newCol = newCollShared.get(); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), newUUID) == nullptr); ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), newUUID), boost::none); - catalog.registerCollection(opCtx.get(), std::move(newCollShared)); + catalog.registerCollection(opCtx.get(), newUUID, std::move(newCollShared)); ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol); ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss); } @@ -467,7 +482,7 @@ TEST_F(CollectionCatalogTest, RenameCollection) { NamespaceString oldNss(nss.db(), "oldcol"); std::shared_ptr<Collection> collShared = std::make_shared<CollectionMock>(uuid, oldNss); auto collection = collShared.get(); - catalog.registerCollection(opCtx.get(), std::move(collShared)); + catalog.registerCollection(opCtx.get(), uuid, std::move(collShared)); auto yieldableColl = catalog.lookupCollectionByUUID(opCtx.get(), uuid); ASSERT(yieldableColl); ASSERT_EQUALS(yieldableColl, collection); @@ -504,7 +519,7 @@ TEST_F(CollectionCatalogTest, RenameCollection) { TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDropped) { { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onCloseCatalog(); + catalog.onCloseCatalog(opCtx.get()); } catalog.deregisterCollection(opCtx.get(), colUUID); @@ -513,7 +528,7 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDrop { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onOpenCatalog(); + catalog.onOpenCatalog(opCtx.get()); } ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), colUUID), boost::none); @@ -522,25 +537,25 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDrop TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreatedNSS) { auto newUUID = UUID::gen(); NamespaceString newNss(nss.db(), "newcol"); - std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newUUID, newNss); + std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss); auto newCol = newCollShared.get(); // Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs. { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onCloseCatalog(); + catalog.onCloseCatalog(opCtx.get()); } ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), newUUID) == nullptr); ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), newUUID), boost::none); - catalog.registerCollection(opCtx.get(), std::move(newCollShared)); + catalog.registerCollection(opCtx.get(), newUUID, std::move(newCollShared)); ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol); ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss); // Ensure that collection still exists after opening the catalog again. { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onOpenCatalog(); + catalog.onOpenCatalog(opCtx.get()); } ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol); @@ -549,29 +564,25 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreated TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsFreshestNSS) { NamespaceString newNss(nss.db(), "newcol"); - std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(colUUID, newNss); + std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss); auto newCol = newCollShared.get(); { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onCloseCatalog(); + catalog.onCloseCatalog(opCtx.get()); } catalog.deregisterCollection(opCtx.get(), colUUID); ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), colUUID) == nullptr); ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss); - { - Lock::GlobalWrite lk(opCtx.get()); - catalog.registerCollection(opCtx.get(), std::move(newCollShared)); - } - + catalog.registerCollection(opCtx.get(), colUUID, std::move(newCollShared)); ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), colUUID), newCol); ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), newNss); // Ensure that collection still exists after opening the catalog again. { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onOpenCatalog(); + catalog.onOpenCatalog(opCtx.get()); } ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), colUUID), newCol); @@ -584,8 +595,8 @@ TEST_F(CollectionCatalogTest, CollectionCatalogEpoch) { { Lock::GlobalLock globalLk(opCtx.get(), MODE_X); - catalog.onCloseCatalog(); - catalog.onOpenCatalog(); + catalog.onCloseCatalog(opCtx.get()); + catalog.onOpenCatalog(opCtx.get()); } auto incrementedEpoch = catalog.getEpoch(); @@ -609,7 +620,8 @@ TEST_F(CollectionCatalogTest, GetAllCollectionNamesAndGetAllDbNames) { std::vector<NamespaceString> nsss = {aColl, b1Coll, b2Coll, cColl, d1Coll, d2Coll, d3Coll}; for (auto& nss : nsss) { std::shared_ptr<Collection> newColl = std::make_shared<CollectionMock>(nss); - catalog.registerCollection(opCtx.get(), std::move(newColl)); + auto uuid = UUID::gen(); + catalog.registerCollection(opCtx.get(), uuid, std::move(newColl)); } std::vector<NamespaceString> dCollList = {d1Coll, d2Coll, d3Coll}; @@ -664,7 +676,8 @@ TEST_F(CollectionCatalogTest, GetAllCollectionNamesAndGetAllDbNamesWithUncommitt std::vector<NamespaceString> nsss = {aColl, b1Coll, b2Coll, cColl, d1Coll, d2Coll, d3Coll}; for (auto& nss : nsss) { std::shared_ptr<Collection> newColl = std::make_shared<CollectionMock>(nss); - catalog.registerCollection(opCtx.get(), std::move(newColl)); + auto uuid = UUID::gen(); + catalog.registerCollection(opCtx.get(), uuid, std::move(newColl)); } // One dbName with only an invisible collection does not appear in dbNames. Use const_cast to diff --git a/src/mongo/db/catalog/collection_compact.cpp b/src/mongo/db/catalog/collection_compact.cpp index b6cc7cb444d..4fed779eaae 100644 --- a/src/mongo/db/catalog/collection_compact.cpp +++ b/src/mongo/db/catalog/collection_compact.cpp @@ -74,10 +74,10 @@ StatusWith<int64_t> compactCollection(OperationContext* opCtx, Database* database = autoDb.getDb(); uassert(ErrorCodes::NamespaceNotFound, "database does not exist", database); - // The collection lock will be upgraded to an exclusive lock if the record store does not - // support online compaction. + // The collection lock will be downgraded to an intent lock if the record store supports + // online compaction. boost::optional<Lock::CollectionLock> collLk; - collLk.emplace(opCtx, collectionNss, MODE_IX); + collLk.emplace(opCtx, collectionNss, MODE_X); CollectionPtr collection = getCollectionForCompact(opCtx, collectionNss); DisableDocumentValidation validationDisabler(opCtx); @@ -91,9 +91,10 @@ StatusWith<int64_t> compactCollection(OperationContext* opCtx, str::stream() << "cannot compact collection with record store: " << recordStore->name()); - if (!recordStore->supportsOnlineCompaction()) { - // Storage engines that disallow online compaction should compact under an exclusive lock. - collLk.emplace(opCtx, collectionNss, MODE_X); + if (recordStore->supportsOnlineCompaction()) { + // Storage engines that allow online compaction should do so using an intent lock on the + // collection. + collLk.emplace(opCtx, collectionNss, MODE_IX); // Ensure the collection was not dropped during the re-lock. collection = getCollectionForCompact(opCtx, collectionNss); diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index cdf3730fa1e..b79c8c78914 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -40,8 +40,6 @@ #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/crypto/fle_crypto.h" #include "mongo/db/auth/security_token.h" -#include "mongo/db/catalog/backwards_compatible_collection_options_util.h" -#include "mongo/db/catalog/catalog_stats.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/catalog/document_validation.h" @@ -49,7 +47,6 @@ #include "mongo/db/catalog/index_consistency.h" #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/catalog/local_oplog_info.h" -#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" #include "mongo/db/catalog/uncommitted_multikey.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands/server_status_metric.h" @@ -83,7 +80,6 @@ #include "mongo/db/storage/record_store.h" #include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/timeseries/timeseries_constants.h" -#include "mongo/db/timeseries/timeseries_extended_range.h" #include "mongo/db/timeseries/timeseries_index_schema_conversion_functions.h" #include "mongo/db/transaction_participant.h" #include "mongo/db/ttl_collection_cache.h" @@ -354,42 +350,35 @@ bool indexTypeSupportsPathLevelMultikeyTracking(StringData accessMethod) { return accessMethod == IndexNames::BTREE || accessMethod == IndexNames::GEO_2DSPHERE; } -StatusWith<bool> doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { +bool doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) { auto minIt = min.begin(); auto minEnd = min.end(); auto maxIt = max.begin(); auto maxEnd = max.end(); while (minIt != minEnd && maxIt != maxEnd) { - // The 'control.min' and 'control.max' fields have the same ordering. - if (minIt->fieldNameStringData() != maxIt->fieldNameStringData()) { - return Status{ - ErrorCodes::BadValue, - "Encountered inconsistent field name ordering in time-series bucket min/max"}; - } - - if (minIt->canonicalType() != maxIt->canonicalType()) { + bool typeMatch = minIt->canonicalType() == maxIt->canonicalType(); + if (!typeMatch) { return true; } else if (minIt->type() == Object) { - auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); - if (!result.isOK() || result.getValue()) { - return result; + // The 'control.min' and 'control.max' fields have the same ordering. + invariant(minIt->fieldNameStringData() == maxIt->fieldNameStringData()); + if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { + return true; } } else if (minIt->type() == Array) { - auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj()); - if (!result.isOK() || result.getValue()) { - return result; + if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) { + return true; } } + invariant(typeMatch); minIt++; maxIt++; } - if (minIt != minEnd || maxIt != maxEnd) { - return Status{ErrorCodes::BadValue, - "Encountered extra field(s) in time-series bucket min/max"}; - } + // The 'control.min' and 'control.max' fields have the same cardinality. + invariant(minIt == minEnd && maxIt == maxEnd); return false; } @@ -550,11 +539,11 @@ void CollectionImpl::init(OperationContext* opCtx) { if (opCtx->lockState()->inAWriteUnitOfWork()) { opCtx->recoveryUnit()->onCommit([svcCtx, uuid](auto ts) { TTLCollectionCache::get(svcCtx).registerTTLInfo( - uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}}); + uuid, TTLCollectionCache::ClusteredId{}); }); } else { - TTLCollectionCache::get(svcCtx).registerTTLInfo( - uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}}); + TTLCollectionCache::get(svcCtx).registerTTLInfo(uuid, + TTLCollectionCache::ClusteredId{}); } } } @@ -736,8 +725,6 @@ Collection::Validator CollectionImpl::parseValidator( auto expCtx = make_intrusive<ExpressionContext>( opCtx, CollatorInterface::cloneCollator(_shared->_collator.get()), ns()); - expCtx->variables.setDefaultRuntimeConstants(opCtx); - // The MatchExpression and contained ExpressionContext created as part of the validator are // owned by the Collection and will outlive the OperationContext they were created under. expCtx->opCtx = nullptr; @@ -824,9 +811,9 @@ Status CollectionImpl::insertDocumentsForOplog(OperationContext* opCtx, _cappedDeleteAsNeeded(opCtx, records->begin()->id); - // We do not need to notify capped waiters, as we have not yet updated oplog visibility, so - // these inserts will not be visible. When visibility updates, it will notify capped - // waiters. + opCtx->recoveryUnit()->onCommit( + [this](boost::optional<Timestamp>) { _shared->notifyCappedWaitersIfNeeded(); }); + return status; } @@ -1598,19 +1585,6 @@ bool CollectionImpl::isTemporary() const { } boost::optional<bool> CollectionImpl::getTimeseriesBucketsMayHaveMixedSchemaData() const { - if (!getTimeseriesOptions()) { - return boost::none; - } - - // If present, reuse storageEngine options to work around the issue described in SERVER-91194 - boost::optional<bool> optBackwardsCompatibleFlag = getFlagFromStorageEngineBson( - _metadata->options.storageEngine, - backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData); - if (optBackwardsCompatibleFlag) { - return *optBackwardsCompatibleFlag; - } - - // Else, fallback to legacy parameter return _metadata->timeseriesBucketsMayHaveMixedSchemaData; } @@ -1626,21 +1600,11 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext "setting"_attr = setting); _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { - // Reuse storageEngine options to work around the issue described in SERVER-91194 - if (setting.has_value()) { - md.options.storageEngine = setFlagToStorageEngineBson( - md.options.storageEngine, - backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData, - *setting); - } - - // Also update legacy parameter for compatibility when downgrading to older sub-versions - // only relying on this option (best-effort because it may be lost due to SERVER-91194) md.timeseriesBucketsMayHaveMixedSchemaData = setting; }); } -StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( +bool CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( const BSONObj& bucketsDoc) const { if (!getTimeseriesOptions()) { return false; @@ -1653,29 +1617,6 @@ StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData( return doesMinMaxHaveMixedSchemaData(minObj, maxObj); } -bool CollectionImpl::getRequiresTimeseriesExtendedRangeSupport() const { - return _shared->_requiresTimeseriesExtendedRangeSupport.load(); -} - -void CollectionImpl::setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const { - uassert(6679401, "This is not a time-series collection", _metadata->options.timeseries); - - bool expected = false; - bool set = _shared->_requiresTimeseriesExtendedRangeSupport.compareAndSwap(&expected, true); - if (set) { - catalog_stats::requiresTimeseriesExtendedRangeSupport.fetchAndAdd(1); - if (!timeseries::collectionHasTimeIndex(opCtx, *this)) { - LOGV2_WARNING( - 6679402, - "Time-series collection contains dates outside the standard range. Some query " - "optimizations may be disabled. Please consider building an index on timeField to " - "re-enable them.", - "nss"_attr = ns().getTimeseriesViewNamespace(), - "timeField"_attr = _metadata->options.timeseries->getTimeField()); - } - } -} - bool CollectionImpl::isClustered() const { return getClusteredInfo().is_initialized(); } @@ -1814,8 +1755,7 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, int scale) const { const IndexCatalog* idxCatalog = getIndexCatalog(); - auto ii = idxCatalog->getIndexIterator( - opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + std::unique_ptr<IndexCatalog::IndexIterator> ii = idxCatalog->getIndexIterator(opCtx, true); uint64_t totalSize = 0; @@ -1836,18 +1776,9 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx, } uint64_t CollectionImpl::getIndexFreeStorageBytes(OperationContext* const opCtx) const { - // Unfinished index builds are excluded to avoid a potential deadlock when trying to collect - // statistics from the index table while the index build is in the bulk load phase. See - // SERVER-77018. This should not be too impactful as: - // - During the collection scan phase, the index table is unused. - // - During the bulk load phase, getFreeStorageBytes will probably return EBUSY, as the ident is - // in use by the index builder. (And worst case results in the deadlock). - // - It might be possible to return meaningful data post bulk-load, but reusable bytes should be - // low anyways as the collection has been bulk loaded. Additionally, this would be a inaccurate - // anyways as the build is in progress. - // - Once the index build is finished, this will be eventually accounted for. const auto idxCatalog = getIndexCatalog(); - auto indexIt = idxCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + const bool includeUnfinished = true; + auto indexIt = idxCatalog->getIndexIterator(opCtx, includeUnfinished); uint64_t totalSize = 0; while (indexIt->more()) { @@ -1871,7 +1802,8 @@ Status CollectionImpl::truncate(OperationContext* opCtx) { // 1) store index specs std::vector<BSONObj> indexSpecs; { - auto ii = _indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + std::unique_ptr<IndexCatalog::IndexIterator> ii = + _indexCatalog->getIndexIterator(opCtx, false); while (ii->more()) { const IndexDescriptor* idx = ii->next()->descriptor(); indexSpecs.push_back(idx->infoObj().getOwned()); @@ -2206,22 +2138,20 @@ void CollectionImpl::updatePrepareUniqueSetting(OperationContext* opCtx, }); } -std::vector<std::string> CollectionImpl::repairInvalidIndexOptions(OperationContext* opCtx) { +std::vector<std::string> CollectionImpl::removeInvalidIndexOptions(OperationContext* opCtx) { std::vector<std::string> indexesWithInvalidOptions; _writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) { for (auto& index : md.indexes) { - if (index.isPresent()) { - BSONObj oldSpec = index.spec; - - Status status = index_key_validate::validateIndexSpec(opCtx, oldSpec).getStatus(); - if (status.isOK()) { - continue; - } + BSONObj oldSpec = index.spec; - indexesWithInvalidOptions.push_back(std::string(index.nameStringData())); - index.spec = index_key_validate::repairIndexSpec(NamespaceString(md.ns), oldSpec); + Status status = index_key_validate::validateIndexSpecFieldNames(oldSpec); + if (status.isOK()) { + continue; } + + indexesWithInvalidOptions.push_back(std::string(index.nameStringData())); + index.spec = index_key_validate::removeUnknownFields(NamespaceString(md.ns), oldSpec); } }); @@ -2298,65 +2228,44 @@ bool CollectionImpl::isIndexMultikey(OperationContext* opCtx, StringData indexName, MultikeyPaths* multikeyPaths, int indexOffset) const { - int offset = indexOffset; - if (offset < 0) { - offset = _metadata->findIndexOffset(indexName); - invariant(offset >= 0, - str::stream() << "cannot get multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - } else { - invariant(offset < int(_metadata->indexes.size()), - str::stream() << "out of bounds index offset for multikey info " << indexName - << " @ " << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - invariant(indexName == _metadata->indexes[offset].nameStringData(), - str::stream() << "invalid index offset for multikey info " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - } - - // If we have uncommitted multikey writes we need to check here to read our own writes + auto isMultikey = [this, multikeyPaths, indexName, indexOffset]( + const BSONCollectionCatalogEntry::MetaData& metadata) { + int offset = indexOffset; + if (offset < 0) { + offset = metadata.findIndexOffset(indexName); + invariant(offset >= 0, + str::stream() << "cannot get multikey for index " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON()); + } else { + invariant(offset < int(metadata.indexes.size()), + str::stream() + << "out of bounds index offset for multikey info " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + invariant(indexName == metadata.indexes[offset].nameStringData(), + str::stream() + << "invalid index offset for multikey info " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + } + + const auto& index = metadata.indexes[offset]; + stdx::lock_guard lock(index.multikeyMutex); + if (multikeyPaths && !index.multikeyPaths.empty()) { + *multikeyPaths = index.multikeyPaths; + } + + return index.multikey; + }; + const auto& uncommittedMultikeys = UncommittedMultikey::get(opCtx).resources(); if (uncommittedMultikeys) { if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) { - const auto& index = it->second.indexes[offset]; - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; - } - } - - // Otherwise read from the metadata cache if there are no concurrent multikey writers - { - const auto& index = _metadata->indexes[offset]; - // Check for concurrent writers, this can race with writers where it can be set immediately - // after checking. This is fine we know that the reader in that case opened its snapshot - // before the writer and we do not need to observe its result. - if (index.concurrentWriters.load() == 0) { - stdx::lock_guard lock(index.multikeyMutex); - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; + return isMultikey(it->second); } } - // We need to read from the durable catalog if there are concurrent multikey writers to avoid - // reading between the multikey write committing in the storage engine but before its onCommit - // handler made the write visible for readers. - auto snapshotMetadata = DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId()); - int snapshotOffset = snapshotMetadata->findIndexOffset(indexName); - invariant(snapshotOffset >= 0, - str::stream() << "cannot get multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - const auto& index = snapshotMetadata->indexes[snapshotOffset]; - if (multikeyPaths && !index.multikeyPaths.empty()) { - *multikeyPaths = index.multikeyPaths; - } - return index.multikey; + return isMultikey(*_metadata); } bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, @@ -2364,31 +2273,31 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, const MultikeyPaths& multikeyPaths, int indexOffset) const { - int offset = indexOffset; - if (offset < 0) { - offset = _metadata->findIndexOffset(indexName); - invariant(offset >= 0, - str::stream() << "cannot set multikey for index " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON()); - } else { - invariant(offset < int(_metadata->indexes.size()), - str::stream() << "out of bounds index offset for multikey update" << indexName - << " @ " << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - invariant(indexName == _metadata->indexes[offset].nameStringData(), - str::stream() << "invalid index offset for multikey update " << indexName << " @ " - << getCatalogId() << " : " << _metadata->toBSON() - << "; offset : " << offset - << " ; actual : " << _metadata->findIndexOffset(indexName)); - } - - auto setMultikey = [offset, - multikeyPaths](const BSONCollectionCatalogEntry::MetaData& metadata) { + auto setMultikey = [this, indexName, multikeyPaths, indexOffset]( + const BSONCollectionCatalogEntry::MetaData& metadata) { + int offset = indexOffset; + if (offset < 0) { + offset = metadata.findIndexOffset(indexName); + invariant(offset >= 0, + str::stream() << "cannot set multikey for index " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON()); + } else { + invariant(offset < int(metadata.indexes.size()), + str::stream() + << "out of bounds index offset for multikey update" << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + invariant(indexName == metadata.indexes[offset].nameStringData(), + str::stream() + << "invalid index offset for multikey update " << indexName << " @ " + << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset + << " ; actual : " << metadata.findIndexOffset(indexName)); + } + auto* index = &metadata.indexes[offset]; stdx::lock_guard lock(index->multikeyMutex); - auto tracksPathLevelMultikeyInfo = !index->multikeyPaths.empty(); + auto tracksPathLevelMultikeyInfo = !metadata.indexes[offset].multikeyPaths.empty(); if (!tracksPathLevelMultikeyInfo) { invariant(multikeyPaths.empty()); @@ -2404,7 +2313,7 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, // We are tracking path-level multikey information for this index. invariant(!multikeyPaths.empty()); - invariant(multikeyPaths.size() == index->multikeyPaths.size()); + invariant(multikeyPaths.size() == metadata.indexes[offset].multikeyPaths.size()); index->multikey = true; @@ -2443,31 +2352,11 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, } BSONCollectionCatalogEntry::MetaData* metadata = nullptr; bool hasSetMultikey = false; - if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) { metadata = &it->second; hasSetMultikey = setMultikey(*metadata); } else { - // First time this OperationContext needs to change multikey information for this - // collection. We cannot use the cached metadata in this collection as we may have just - // committed a multikey change concurrently to the storage engine without being able to - // observe it if its onCommit handlers haven't run yet. - auto metadataLocal = *DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId()); - // When reading from the durable catalog the index offsets are different because when - // removing indexes in-memory just zeros out the slot instead of actually removing it. We - // must adjust the entries so they match how they are stored in _metadata so we can rely on - // the index offsets being stable. The order of valid indexes are the same, so we can - // iterate from the end and move them into the right positions. - int localIdx = metadataLocal.indexes.size() - 1; - metadataLocal.indexes.resize(_metadata->indexes.size()); - for (int i = _metadata->indexes.size() - 1; i >= 0 && localIdx != i; --i) { - if (_metadata->indexes[i].isPresent()) { - metadataLocal.indexes[i] = std::move(metadataLocal.indexes[localIdx]); - metadataLocal.indexes[localIdx] = {}; - --localIdx; - } - } - + BSONCollectionCatalogEntry::MetaData metadataLocal(*_metadata); hasSetMultikey = setMultikey(metadataLocal); if (hasSetMultikey) { metadata = &uncommittedMultikeys->emplace(this, std::move(metadataLocal)).first->second; @@ -2482,44 +2371,8 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx, DurableCatalog::get(opCtx)->putMetaData(opCtx, getCatalogId(), *metadata); - // RAII Helper object to ensure we decrement the concurrent counter if and only if we - // incremented it in a preCommit handler. - class ConcurrentMultikeyWriteTracker { - public: - ConcurrentMultikeyWriteTracker( - std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> meta, int indexOffset) - : metadata(std::move(meta)), offset(indexOffset) {} - - ~ConcurrentMultikeyWriteTracker() { - if (hasIncremented) { - metadata->indexes[offset].concurrentWriters.fetchAndSubtract(1); - } - } - - void preCommit() { - metadata->indexes[offset].concurrentWriters.fetchAndAdd(1); - hasIncremented = true; - } - - private: - std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> metadata; - int offset; - bool hasIncremented = false; - }; - - auto concurrentWriteTracker = - std::make_shared<ConcurrentMultikeyWriteTracker>(_metadata, offset); - - // Mark this index that there is an ongoing multikey write. This forces readers to read from the - // durable catalog to determine if the index is multikey or not. - opCtx->recoveryUnit()->registerPreCommitHook( - [concurrentWriteTracker](OperationContext*) { concurrentWriteTracker->preCommit(); }); - - // Capture a reference to 'concurrentWriteTracker' to extend the lifetime of this object until - // commiting/rolling back the transaction is fully complete. opCtx->recoveryUnit()->onCommit( - [this, uncommittedMultikeys, setMultikey = std::move(setMultikey), concurrentWriteTracker]( - auto ts) { + [this, uncommittedMultikeys, setMultikey = std::move(setMultikey)](auto ts) { // Merge in changes to this index, other indexes may have been updated since we made our // copy. Don't check for result as another thread could be setting multikey at the same // time diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index 58662679df1..d3379568d22 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -329,11 +329,7 @@ public: void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx, boost::optional<bool> setting) final; - StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( - const BSONObj& bucketsDoc) const final; - - bool getRequiresTimeseriesExtendedRangeSupport() const final; - void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const final; + bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const final; /** * isClustered() relies on the object returned from getClusteredInfo(). If @@ -450,7 +446,7 @@ public: StringData idxName, bool prepareUnique) final; - std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) final; + std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) final; void setIsTemp(OperationContext* opCtx, bool isTemp) final; @@ -583,18 +579,6 @@ private: AtomicWord<bool> _committed{true}; - // Time-series collections are allowed to contain measurements with arbitrary dates; - // however, many of our query optimizations only work properly with dates that can be stored - // as an offset in seconds from the Unix epoch within 31 bits (roughly 1970-2038). When this - // flag is set to true, these optimizations will be disabled. It must be set to true if the - // collection contains any measurements with dates outside this normal range. - // - // This is set from the write path where we only hold an IX lock, so we want to be able to - // set it from a const method on the Collection. In order to do this, we need to make it - // mutable. Given that the value may only transition from false to true, but never back - // again, and that we store and retrieve it atomically, this should be safe. - mutable AtomicWord<bool> _requiresTimeseriesExtendedRangeSupport{false}; - // Capped information. const bool _isCapped; diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index 820ed47cdf4..00d21ec21dd 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -253,19 +253,10 @@ public: std::abort(); } - StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( - const BSONObj& bucketsDoc) const { + bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const { std::abort(); } - bool getRequiresTimeseriesExtendedRangeSupport() const { - MONGO_UNREACHABLE; - } - - void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const { - MONGO_UNREACHABLE; - } - bool isClustered() const { return false; } @@ -431,7 +422,7 @@ public: std::abort(); } - std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) { + std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) { std::abort(); } diff --git a/src/mongo/db/catalog/collection_operation_source.cpp b/src/mongo/db/catalog/collection_operation_source.cpp index 198e929c7f4..84b291f19ce 100644 --- a/src/mongo/db/catalog/collection_operation_source.cpp +++ b/src/mongo/db/catalog/collection_operation_source.cpp @@ -38,8 +38,6 @@ StringData toString(OperationSource source) { static constexpr StringData kTimeseriesInsertString = "time-series insert"_sd; static constexpr StringData kTimeseriesUpdateString = "time-series update"_sd; static constexpr StringData kTimeseriesDeleteString = "time-series delete"_sd; - static constexpr StringData kTimeseriesBucketCompressionString = - "time-series bucket compression"_sd; switch (source) { case OperationSource::kStandard: @@ -52,8 +50,6 @@ StringData toString(OperationSource source) { return kTimeseriesUpdateString; case OperationSource::kTimeseriesDelete: return kTimeseriesDeleteString; - case OperationSource::kTimeseriesBucketCompression: - return kTimeseriesBucketCompressionString; } MONGO_UNREACHABLE; diff --git a/src/mongo/db/catalog/collection_operation_source.h b/src/mongo/db/catalog/collection_operation_source.h index 7546d681850..6cfff61882f 100644 --- a/src/mongo/db/catalog/collection_operation_source.h +++ b/src/mongo/db/catalog/collection_operation_source.h @@ -43,7 +43,6 @@ enum class OperationSource { kTimeseriesInsert, kTimeseriesUpdate, kTimeseriesDelete, - kTimeseriesBucketCompression }; StringData toString(OperationSource source); diff --git a/src/mongo/db/catalog/collection_test.cpp b/src/mongo/db/catalog/collection_test.cpp index 00c131756ac..fd73c68bad8 100644 --- a/src/mongo/db/catalog/collection_test.cpp +++ b/src/mongo/db/catalog/collection_test.cpp @@ -403,9 +403,7 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) { "max" : { "x" : [ 2, 3 ] } } })")}; for (const auto& controlDoc : mixedSchemaControlDocs) { - auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc); - ASSERT_OK(mixedSchema) << controlDoc; - ASSERT_TRUE(mixedSchema.getValue()) << controlDoc; + ASSERT_TRUE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)); } std::vector<BSONObj> nonMixedSchemaControlDocs = { @@ -462,27 +460,7 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) { for (const auto& controlDoc : nonMixedSchemaControlDocs) { - auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc); - ASSERT_OK(mixedSchema) << controlDoc; - ASSERT_FALSE(mixedSchema.getValue()) << controlDoc; - } - - std::vector<BSONObj> malformedControlDocs = { - // Inconsistent field name ordering - ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 }, - "max" : { "y" : 2, "x" : 2 } } })"), - - // Extra field in min - ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 }, - "max" : { "x" : 2 } } })"), - - // Extra field in max - ::mongo::fromjson(R"({ "control" : { "min" : { "y" : 1 }, - "max" : { "y" : 2, "x" : 2 } } })")}; - - for (const auto& controlDoc : malformedControlDocs) { - ASSERT_NOT_OK(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)) - << controlDoc; + ASSERT_FALSE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc)); } } diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp index 9d0c487b41a..42c5ac18dc7 100644 --- a/src/mongo/db/catalog/collection_validation.cpp +++ b/src/mongo/db/catalog/collection_validation.cpp @@ -78,7 +78,8 @@ void _validateIndexesInternalStructure(OperationContext* opCtx, // Need to use the IndexCatalog here because the 'validateState->indexes' object hasn't been // constructed yet. It must be initialized to ensure we're validating all indexes. const IndexCatalog* indexCatalog = validateState->getCollection()->getIndexCatalog(); - const auto it = indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + const std::unique_ptr<IndexCatalog::IndexIterator> it = + indexCatalog->getIndexIterator(opCtx, false); // Validate Indexes Internal Structure, checking if index files have been compromised or // corrupted. @@ -97,11 +98,14 @@ void _validateIndexesInternalStructure(OperationContext* opCtx, auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()]; - iam->validate(opCtx, nullptr, &curIndexResults); + int64_t numValidated; + iam->validate(opCtx, &numValidated, &curIndexResults); if (!curIndexResults.valid) { results->valid = false; } + + curIndexResults.keysTraversedFromFullValidate = numValidated; } } @@ -133,6 +137,33 @@ void _validateIndexes(OperationContext* opCtx, auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()]; curIndexResults.keysTraversed = numTraversedKeys; + // If we are performing a full index validation, we have information on the number of index + // keys validated in _validateIndexesInternalStructure (when we validated the internal + // structure of the index). Check if this is consistent with 'numTraversedKeys' from + // traverseIndex above. + if (validateState->isFullIndexValidation()) { + invariant(opCtx->lockState()->isCollectionLockedForMode(validateState->nss(), MODE_X)); + + // The number of keys counted in _validateIndexesInternalStructure, when checking the + // internal structure of the index. + const int64_t numIndexKeys = curIndexResults.keysTraversedFromFullValidate; + + // Check if currIndexResults is valid to ensure that this index is not corrupted or + // comprised (which was set in _validateIndexesInternalStructure). If the index is + // corrupted, there is no use in checking if the traversal yielded the same key count. + if (curIndexResults.valid) { + if (numIndexKeys != numTraversedKeys) { + curIndexResults.valid = false; + string msg = str::stream() + << "number of traversed index entries (" << numTraversedKeys + << ") does not match the number of expected index entries (" << numIndexKeys + << ")"; + results->errors.push_back(msg); + results->valid = false; + } + } + } + if (!curIndexResults.valid) { results->valid = false; } @@ -162,8 +193,7 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, ValidateResults tempValidateResults; BSONObjBuilder tempBuilder; - indexValidator->traverseRecordStore( - opCtx, &tempValidateResults, &tempBuilder, validateState->validationVersion()); + indexValidator->traverseRecordStore(opCtx, &tempValidateResults, &tempBuilder); } LOGV2_OPTIONS( @@ -200,7 +230,7 @@ void _gatherIndexEntryErrors(OperationContext* opCtx, LOGV2_OPTIONS(20301, {LogComponent::kIndex}, "Finished traversing through all the indexes"); - indexConsistency->addIndexEntryErrors(opCtx, result); + indexConsistency->addIndexEntryErrors(result); } void _validateIndexKeyCount(OperationContext* opCtx, @@ -217,94 +247,6 @@ void _validateIndexKeyCount(OperationContext* opCtx, } } -void _printIndexSpec(const ValidateState* validateState, StringData indexName) { - auto& indexes = validateState->getIndexes(); - auto indexEntry = - std::find_if(indexes.begin(), - indexes.end(), - [&](const std::shared_ptr<const IndexCatalogEntry> indexEntry) -> bool { - return indexEntry->descriptor()->indexName() == indexName; - }); - if (indexEntry != indexes.end()) { - auto indexSpec = (*indexEntry)->descriptor()->infoObj(); - LOGV2_ERROR(7463100, "Index failed validation", "spec"_attr = indexSpec); - } -} - -/** - * Logs oplog entries related to corrupted records/indexes in validation results. - */ -void _logOplogEntriesForInvalidResults(OperationContext* opCtx, ValidateResults* results) { - if (results->recordTimestamps.empty()) { - return; - } - - LOGV2( - 7464200, - "Validation failed: oplog timestamps referenced by corrupted collection and index entries", - "numTimestamps"_attr = results->recordTimestamps.size()); - - // Set up read on oplog collection. - try { - AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead); - const auto& oplogCollection = oplogRead.getCollection(); - - if (!oplogCollection) { - for (auto it = results->recordTimestamps.rbegin(); - it != results->recordTimestamps.rend(); - it++) { - const auto& timestamp = *it; - LOGV2(8080900, - " Validation failed: Oplog entry timestamp for corrupted collection and " - "index entry", - "timestamp"_attr = timestamp); - } - return; - } - - // Log oplog entries in reverse from most recent timestamp to oldest. - // Due to oplog truncation, if we fail to find any oplog entry for a particular timestamp, - // we can stop searching for oplog entries with earlier timestamps. - auto recordStore = oplogCollection->getRecordStore(); - uassert(ErrorCodes::InternalError, - "Validation failed: Unable to get oplog record store for corrupted collection and " - "index entries", - recordStore); - - auto cursor = recordStore->getCursor(opCtx, /*forward=*/false); - uassert(ErrorCodes::CursorNotFound, - "Validation failed: Unable to get cursor to oplog collection.", - cursor); - - for (auto it = results->recordTimestamps.rbegin(); it != results->recordTimestamps.rend(); - it++) { - const auto& timestamp = *it; - - // A record id in the oplog collection is equivalent to the document's timestamp field. - RecordId recordId(timestamp.asULL()); - auto record = cursor->seekExact(recordId); - if (!record) { - LOGV2(7464201, - " Validation failed: Stopping oplog entry search for corrupted collection " - "and index entries.", - "timestamp"_attr = timestamp); - break; - } - - LOGV2( - 7464202, - " Validation failed: Oplog entry found for corrupted collection and index entry", - "timestamp"_attr = timestamp, - "oplogEntryDoc"_attr = redact(record->data.toBson())); - } - } catch (DBException& ex) { - LOGV2_ERROR(7464203, - "Validation failed: Unable to fetch entries from oplog collection for " - "corrupted collection and index entries", - "ex"_attr = ex); - } -} - void _reportValidationResults(OperationContext* opCtx, ValidateState* validateState, ValidateResults* results, @@ -321,19 +263,17 @@ void _reportValidationResults(OperationContext* opCtx, // Report detailed index validation results gathered when using {full: true} for validated // indexes. - int nIndexes = results->indexResultsMap.size(); - for (const auto& [indexName, vr] : results->indexResultsMap) { - if (!vr.valid) { - results->valid = false; - _printIndexSpec(validateState, indexName); + for (const auto& index : validateState->getIndexes()) { + const std::string indexName = index->descriptor()->indexName(); + auto& indexResultsMap = results->indexResultsMap; + if (indexResultsMap.find(indexName) == indexResultsMap.end()) { + continue; } - if (validateState->getSkippedIndexes().contains(indexName)) { - // Index internal state was checked and cleared, so it was reported in indexResultsMap, - // but we did not verify the index contents against the collection, so we should exclude - // it from this report. - --nIndexes; - continue; + auto& vr = indexResultsMap.at(indexName); + + if (!vr.valid) { + results->valid = false; } BSONObjBuilder bob(indexDetails.subobjStart(indexName)); @@ -354,7 +294,7 @@ void _reportValidationResults(OperationContext* opCtx, results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end()); } - output->append("nIndexes", nIndexes); + output->append("nIndexes", static_cast<int>(validateState->getIndexes().size())); output->append("keysPerIndex", keysPerIndex.done()); output->append("indexDetails", indexDetails.done()); } @@ -364,7 +304,6 @@ void _reportInvalidResults(OperationContext* opCtx, ValidateResults* results, BSONObjBuilder* output) { _reportValidationResults(opCtx, validateState, results, output); - _logOplogEntriesForInvalidResults(opCtx, results); LOGV2_OPTIONS(20302, {LogComponent::kIndex}, "Validation complete -- Corruption found", @@ -402,6 +341,35 @@ void addErrorIfUnequal(boost::optional<ValidationActionEnum> stored, results); } +std::string multikeyPathsToString(MultikeyPaths paths) { + str::stream builder; + builder << "["; + auto pathIt = paths.begin(); + while (true) { + builder << "{"; + + auto pathSet = *pathIt; + auto setIt = pathSet.begin(); + while (true) { + builder << *setIt++; + if (setIt == pathSet.end()) { + break; + } else { + builder << ","; + } + } + builder << "}"; + + if (++pathIt == paths.end()) { + break; + } else { + builder << ","; + } + } + builder << "]"; + return builder; +} + void _validateCatalogEntry(OperationContext* opCtx, ValidateState* validateState, ValidateResults* results) { @@ -442,23 +410,20 @@ void _validateCatalogEntry(OperationContext* opCtx, } const auto& indexCatalog = collection->getIndexCatalog(); - auto indexIt = indexCatalog->getIndexIterator(opCtx, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + auto indexIt = indexCatalog->getIndexIterator(opCtx, /*includeUnfinishedIndexes=*/true); while (indexIt->more()) { const IndexCatalogEntry* indexEntry = indexIt->next(); const std::string indexName = indexEntry->descriptor()->indexName(); Status status = - index_key_validate::validateIndexSpec(opCtx, indexEntry->descriptor()->infoObj()) - .getStatus(); + index_key_validate::validateIndexSpecFieldNames(indexEntry->descriptor()->infoObj()); if (!status.isOK()) { - results->warnings.push_back( - fmt::format("The index specification for index '{}' contains invalid fields. {}. " - "Run the 'collMod' command on the collection without any arguments " - "to fix the invalid index options", + results->valid = false; + results->errors.push_back( + fmt::format("The index specification for index '{}' contains invalid field names. " + "{}. Run the 'collMod' command on the collection without any arguments " + "to remove the invalid index options", indexName, status.reason())); } @@ -594,15 +559,14 @@ Status validate(OperationContext* opCtx, const NamespaceString& nss, ValidateMode mode, RepairMode repairMode, - const AdditionalOptions& additionalOptions, ValidateResults* results, BSONObjBuilder* output, - bool logDiagnostics) { + bool turnOnExtraLoggingForTest) { invariant(!opCtx->lockState()->isLocked() || storageGlobalParams.repair); // This is deliberately outside of the try-catch block, so that any errors thrown in the // constructor fail the cmd, as opposed to returning OK with valid:false. - ValidateState validateState(opCtx, nss, mode, repairMode, additionalOptions, logDiagnostics); + ValidateState validateState(opCtx, nss, mode, repairMode, turnOnExtraLoggingForTest); const auto replCoord = repl::ReplicationCoordinator::get(opCtx); // Check whether we are allowed to read from this node after acquiring our locks. If we are @@ -621,14 +585,6 @@ Status validate(OperationContext* opCtx, opCtx->recoveryUnit()->abandonSnapshot(); opCtx->recoveryUnit()->setPrepareConflictBehavior(oldPrepareConflictBehavior); }); - - // Relax corruption detection so that we log and continue scanning instead of failing early. - auto oldDataCorruptionMode = opCtx->recoveryUnit()->getDataCorruptionDetectionMode(); - opCtx->recoveryUnit()->setDataCorruptionDetectionMode( - DataCorruptionDetectionMode::kLogAndContinue); - ON_BLOCK_EXIT( - [&] { opCtx->recoveryUnit()->setDataCorruptionDetectionMode(oldDataCorruptionMode); }); - if (validateState.fixErrors()) { // Note: cannot set PrepareConflictBehavior here, since the validate command with repair // needs kIngnoreConflictsAllowWrites, but validate repair at startup cannot set that here @@ -705,8 +661,7 @@ Status validate(OperationContext* opCtx, // the collection. For clustered collections, the validator also verifies that the // record key (RecordId) matches the cluster key field in the record value (document's // cluster key). - indexValidator.traverseRecordStore( - opCtx, results, output, additionalOptions.validationVersion); + indexValidator.traverseRecordStore(opCtx, results, output); // Pause collection validation while a lock is held and between collection and index data // validation. @@ -780,7 +735,8 @@ Status validate(OperationContext* opCtx, return e.toStatus(); } string err = str::stream() << "exception during collection validation: " << e.toString(); - results->warnings.push_back(err); + results->errors.push_back(err); + results->valid = false; LOGV2_OPTIONS(5160302, {LogComponent::kIndex}, "Validation failed due to exception", diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h index f68258fee14..378244f39b7 100644 --- a/src/mongo/db/catalog/collection_validation.h +++ b/src/mongo/db/catalog/collection_validation.h @@ -29,9 +29,6 @@ #pragma once -#include "mongo/base/status.h" -#include "mongo/bson/bson_validate.h" -#include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/catalog/validate_results.h" #include "mongo/db/namespace_string.h" @@ -90,13 +87,6 @@ enum class RepairMode { }; /** - * Additional validation options that can run in any mode. - */ -struct AdditionalOptions { - ValidationVersion validationVersion = currentValidationVersion; -}; - -/** * Expects the caller to hold no locks. * * Background validation does not support any type of full validation above. @@ -111,10 +101,9 @@ Status validate(OperationContext* opCtx, const NamespaceString& nss, ValidateMode mode, RepairMode repairMode, - const AdditionalOptions& additionalOptions, ValidateResults* results, BSONObjBuilder* output, - bool logDiagnostics); + bool turnOnExtraLoggingForTest = false); /** * Checks whether a failpoint has been hit in the above validate() code.. diff --git a/src/mongo/db/catalog/collection_validation_test.cpp b/src/mongo/db/catalog/collection_validation_test.cpp index 870c0408565..4ffff4f653b 100644 --- a/src/mongo/db/catalog/collection_validation_test.cpp +++ b/src/mongo/db/catalog/collection_validation_test.cpp @@ -47,9 +47,6 @@ namespace { const NamespaceString kNss = NamespaceString("test.t"); class CollectionValidationTest : public CatalogTestFixture { -protected: - CollectionValidationTest(Options options = {}) : CatalogTestFixture(std::move(options)) {} - private: void setUp() override { CatalogTestFixture::setUp(); @@ -61,11 +58,6 @@ private: }; }; -class CollectionValidationDiskTest : public CollectionValidationTest { -protected: - CollectionValidationDiskTest() : CollectionValidationTest(Options{}.ephemeral(false)) {} -}; - /** * Calls validate on collection kNss with both kValidateFull and kValidateNormal validation levels * and verifies the results. @@ -86,14 +78,8 @@ std::vector<std::pair<BSONObj, ValidateResults>> foregroundValidate( for (auto mode : modes) { ValidateResults validateResults; BSONObjBuilder output; - ASSERT_OK(CollectionValidation::validate(opCtx, - kNss, - mode, - repairMode, - /*additionalOptions=*/{}, - &validateResults, - &output, - /*logDiagnostics=*/false)); + ASSERT_OK(CollectionValidation::validate( + opCtx, kNss, mode, repairMode, &validateResults, &output)); BSONObj obj = output.obj(); BSONObjBuilder validateResultsBuilder; validateResults.appendToResultObj(&validateResultsBuilder, true /* debugging */); @@ -159,10 +145,8 @@ void backgroundValidate(OperationContext* opCtx, kNss, CollectionValidation::ValidateMode::kBackground, CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, &validateResults, - &output, - /*logDiagnostics=*/false)); + &output)); BSONObj obj = output.obj(); ASSERT_EQ(validateResults.valid, valid); @@ -286,19 +270,6 @@ TEST_F(CollectionValidationTest, ValidateEnforceFastCount) { {CollectionValidation::ValidateMode::kForegroundFullEnforceFastCount}); } -TEST_F(CollectionValidationDiskTest, ValidateIndexDetailResultsSurfaceVerifyErrors) { - FailPointEnableBlock fp{"WTValidateIndexStructuralDamage"}; - auto opCtx = operationContext(); - insertDataRange(opCtx, 0, 5); // initialize collection - foregroundValidate( - opCtx, - /*valid*/ false, - /*numRecords*/ std::numeric_limits<int32_t>::min(), // uninitialized - /*numInvalidDocuments*/ std::numeric_limits<int32_t>::min(), // uninitialized - /*numErrors*/ 1, - {CollectionValidation::ValidateMode::kForegroundFull}); -} - /** * Waits for a parallel running collection validation operation to start and then hang at a * failpoint. diff --git a/src/mongo/db/catalog/collection_writer_test.cpp b/src/mongo/db/catalog/collection_writer_test.cpp index 7b8719a190f..1f828f38800 100644 --- a/src/mongo/db/catalog/collection_writer_test.cpp +++ b/src/mongo/db/catalog/collection_writer_test.cpp @@ -58,7 +58,7 @@ protected: std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(kNss); CollectionCatalog::write(getServiceContext(), [&](CollectionCatalog& catalog) { - catalog.registerCollection(operationContext(), std::move(collection)); + catalog.registerCollection(operationContext(), UUID::gen(), std::move(collection)); }); } @@ -253,6 +253,7 @@ public: CollectionCatalog::write(getServiceContext(), [&](CollectionCatalog& catalog) { for (size_t i = 0; i < NumCollections; ++i) { catalog.registerCollection(operationContext(), + UUID::gen(), std::make_shared<CollectionMock>( NamespaceString("many", fmt::format("coll{}", i)))); } diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp index c64decf02a3..74be2bd73e4 100644 --- a/src/mongo/db/catalog/create_collection.cpp +++ b/src/mongo/db/catalog/create_collection.cpp @@ -44,7 +44,7 @@ #include "mongo/db/catalog/index_key_validate.h" #include "mongo/db/commands.h" #include "mongo/db/commands/create_gen.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" @@ -117,9 +117,7 @@ Status validateClusteredIndexSpec(OperationContext* opCtx, if (expireAfterSeconds) { // Not included in the indexSpec itself. - auto status = index_key_validate::validateExpireAfterSeconds( - *expireAfterSeconds, - index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex); + auto status = index_key_validate::validateExpireAfterSeconds(*expireAfterSeconds); if (!status.isOK()) { return status; } @@ -208,117 +206,6 @@ Status _createView(OperationContext* opCtx, }); } -BSONObj _generateTimeseriesValidator(StringData timeField) { - // '$jsonSchema' : { - // bsonType: 'object', - // required: ['_id', 'control', 'data'], - // properties: { - // _id: {bsonType: 'objectId'}, - // control: { - // bsonType: 'object', - // required: ['version', 'min', 'max'], - // properties: { - // version: {bsonType: 'number'}, - // min: { - // bsonType: 'object', - // required: ['%s'], - // properties: {'%s': {bsonType: 'date'}} - // }, - // max: { - // bsonType: 'object', - // required: ['%s'], - // properties: {'%s': {bsonType: 'date'}} - // }, - // closed: {bsonType: 'bool'}, - // count: {bsonType: 'number', minimum: 1}, - // }, - // additionalProperties: false, - // }, - // data: {bsonType: 'object'}, - // meta: {} - // }, - // additionalProperties: false - // } - BSONObjBuilder validator; - BSONObjBuilder schema(validator.subobjStart("$jsonSchema")); - schema.append("bsonType", "object"); - schema.append("required", - BSON_ARRAY("_id" - << "control" - << "data")); - { - BSONObjBuilder properties(schema.subobjStart("properties")); - { - BSONObjBuilder _id(properties.subobjStart("_id")); - _id.append("bsonType", "objectId"); - _id.done(); - } - { - BSONObjBuilder control(properties.subobjStart("control")); - control.append("bsonType", "object"); - control.append("required", - BSON_ARRAY("version" - << "min" - << "max")); - { - BSONObjBuilder innerProperties(control.subobjStart("properties")); - { - BSONObjBuilder version(innerProperties.subobjStart("version")); - version.append("bsonType", "number"); - version.done(); - } - { - BSONObjBuilder min(innerProperties.subobjStart("min")); - min.append("bsonType", "object"); - min.append("required", BSON_ARRAY(timeField)); - BSONObjBuilder minProperties(min.subobjStart("properties")); - BSONObjBuilder timeFieldObj(minProperties.subobjStart(timeField)); - timeFieldObj.append("bsonType", "date"); - timeFieldObj.done(); - minProperties.done(); - min.done(); - } - - { - BSONObjBuilder max(innerProperties.subobjStart("max")); - max.append("bsonType", "object"); - max.append("required", BSON_ARRAY(timeField)); - BSONObjBuilder maxProperties(max.subobjStart("properties")); - BSONObjBuilder timeFieldObj(maxProperties.subobjStart(timeField)); - timeFieldObj.append("bsonType", "date"); - timeFieldObj.done(); - maxProperties.done(); - max.done(); - } - { - BSONObjBuilder closed(innerProperties.subobjStart("closed")); - closed.append("bsonType", "bool"); - closed.done(); - } - { - BSONObjBuilder count(innerProperties.subobjStart("count")); - count.append("bsonType", "number"); - count.append("minimum", 1); - count.done(); - } - innerProperties.done(); - } - control.append("additionalProperties", false); - control.done(); - } - { - BSONObjBuilder data(properties.subobjStart("data")); - data.append("bsonType", "object"); - data.done(); - } - properties.append("meta", BSONObj{}); - properties.done(); - } - schema.append("additionalProperties", false); - schema.done(); - return validator.obj(); -} - Status _createTimeseries(OperationContext* opCtx, const NamespaceString& ns, const CollectionOptions& optionsArg) { @@ -343,13 +230,46 @@ Status _createTimeseries(OperationContext* opCtx, maxSpanSeconds == options.timeseries->getBucketMaxSpanSeconds()); options.timeseries->setBucketMaxSpanSeconds(maxSpanSeconds); - // Set the validator option to a JSON schema enforcing constraints on bucket documents. // This validation is only structural to prevent accidental corruption by users and // cannot cover all constraints. Leave the validationLevel and validationAction to their // strict/error defaults. auto timeField = options.timeseries->getTimeField(); - auto validatorObj = _generateTimeseriesValidator(timeField); + auto validatorObj = fromjson(fmt::sprintf(R"( +{ +'$jsonSchema' : { + bsonType: 'object', + required: ['_id', 'control', 'data'], + properties: { + _id: {bsonType: 'objectId'}, + control: { + bsonType: 'object', + required: ['version', 'min', 'max'], + properties: { + version: {bsonType: 'number'}, + min: { + bsonType: 'object', + required: ['%s'], + properties: {'%s': {bsonType: 'date'}} + }, + max: { + bsonType: 'object', + required: ['%s'], + properties: {'%s': {bsonType: 'date'}} + }, + closed: {bsonType: 'bool'} + } + }, + data: {bsonType: 'object'}, + meta: {} + }, + additionalProperties: false +} +})", + timeField, + timeField, + timeField, + timeField)); bool existingBucketCollectionIsCompatible = false; @@ -401,9 +321,8 @@ Status _createTimeseries(OperationContext* opCtx, // Cluster time-series buckets collections by _id. auto expireAfterSeconds = options.expireAfterSeconds; if (expireAfterSeconds) { - uassertStatusOK(index_key_validate::validateExpireAfterSeconds( - *expireAfterSeconds, - index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex)); + uassertStatusOK( + index_key_validate::validateExpireAfterSeconds(*expireAfterSeconds)); bucketsOptions.expireAfterSeconds = expireAfterSeconds; } diff --git a/src/mongo/db/catalog/create_collection_test.cpp b/src/mongo/db/catalog/create_collection_test.cpp index b5cce6e6686..3141d18905c 100644 --- a/src/mongo/db/catalog/create_collection_test.cpp +++ b/src/mongo/db/catalog/create_collection_test.cpp @@ -34,7 +34,7 @@ #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/create_collection.h" #include "mongo/db/catalog/database_holder.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" #include "mongo/db/repl/replication_coordinator.h" diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h index f3a5a1393be..2640abefbcb 100644 --- a/src/mongo/db/catalog/database.h +++ b/src/mongo/db/catalog/database.h @@ -110,18 +110,13 @@ public: * If we are applying a 'drop' oplog entry on a secondary, 'dropOpTime' will contain the optime * of the oplog entry. * - * When fromMigrate is set, the related oplog entry will be marked with a 'fromMigrate' field to - * reduce its visibility (e.g. in change streams). - * * The caller should hold a DB X lock and ensure there are no index builds in progress on the * collection. * N.B. Namespace argument is passed by value as it may otherwise disappear or change. */ virtual Status dropCollection(OperationContext* opCtx, NamespaceString nss, - repl::OpTime dropOpTime = {}, - bool markFromMigrate = false) const = 0; - + repl::OpTime dropOpTime = {}) const = 0; virtual Status dropCollectionEvenIfSystem(OperationContext* opCtx, NamespaceString nss, repl::OpTime dropOpTime = {}, diff --git a/src/mongo/db/catalog/database_holder_impl.cpp b/src/mongo/db/catalog/database_holder_impl.cpp index f28a44bc0dd..28e44082c2a 100644 --- a/src/mongo/db/catalog/database_holder_impl.cpp +++ b/src/mongo/db/catalog/database_holder_impl.cpp @@ -37,7 +37,7 @@ #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_impl.h" #include "mongo/db/catalog/database_impl.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/op_observer.h" #include "mongo/db/operation_context.h" @@ -220,7 +220,8 @@ void DatabaseHolderImpl::dropDb(OperationContext* opCtx, Database* db) { invariant(opCtx->lockState()->isDbLockedForMode(name.dbName(), MODE_X)); auto catalog = CollectionCatalog::get(opCtx); - for (auto&& coll : catalog->range(name)) { + for (auto collIt = catalog->begin(opCtx, name); collIt != catalog->end(opCtx); ++collIt) { + auto coll = *collIt; if (!coll) { break; } @@ -235,7 +236,8 @@ void DatabaseHolderImpl::dropDb(OperationContext* opCtx, Database* db) { auto const serviceContext = opCtx->getServiceContext(); - for (auto&& coll : catalog->range(name)) { + for (auto collIt = catalog->begin(opCtx, name); collIt != catalog->end(opCtx); ++collIt) { + auto coll = *collIt; if (!coll) { break; } diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index 62f789ad003..077fc8d2313 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -299,7 +299,7 @@ void DatabaseImpl::clearTmpCollections(OperationContext* opCtx) const { CollectionCatalog::CollectionInfoFn callback = [&](const CollectionPtr& collection) { try { WriteUnitOfWork wuow(opCtx); - Status status = dropCollection(opCtx, collection->ns(), {}, false); + Status status = dropCollection(opCtx, collection->ns(), {}); if (!status.isOK()) { LOGV2_WARNING(20327, "could not drop temp collection '{namespace}': {error}", @@ -438,8 +438,7 @@ Status DatabaseImpl::dropView(OperationContext* opCtx, NamespaceString viewName) Status DatabaseImpl::dropCollection(OperationContext* opCtx, NamespaceString nss, - repl::OpTime dropOpTime, - bool markFromMigrate) const { + repl::OpTime dropOpTime) const { // Cannot drop uncommitted collections. invariant(!UncommittedCatalogUpdates::isCreatedCollection(opCtx, nss)); @@ -475,7 +474,7 @@ Status DatabaseImpl::dropCollection(OperationContext* opCtx, } } - return dropCollectionEvenIfSystem(opCtx, nss, dropOpTime, markFromMigrate); + return dropCollectionEvenIfSystem(opCtx, nss, dropOpTime); } Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx, @@ -745,7 +744,7 @@ void DatabaseImpl::_checkCanCreateCollection(OperationContext* opCtx, const CollectionOptions& options) const { if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, nss)) { if (options.isView()) { - uasserted(ErrorCodes::NamespaceExists, + uasserted(17399, str::stream() << "Cannot create collection " << nss << " - collection already exists."); } else { @@ -1090,10 +1089,6 @@ Status DatabaseImpl::userCreateNS(OperationContext* opCtx, ExtensionsCallbackNoop(), allowedFeatures); - // Increment counters to track the usage of schema validators. - validatorCounters.incrementCounters( - "create", collectionOptions.validator, statusWithMatcher.isOK()); - // We check the status of the parse to see if there are any banned features, but we don't // actually need the result for now. if (!statusWithMatcher.isOK()) { diff --git a/src/mongo/db/catalog/database_impl.h b/src/mongo/db/catalog/database_impl.h index c87f811c7d8..fa09177bc1b 100644 --- a/src/mongo/db/catalog/database_impl.h +++ b/src/mongo/db/catalog/database_impl.h @@ -62,20 +62,16 @@ public: * If we are applying a 'drop' oplog entry on a secondary, 'dropOpTime' will contain the optime * of the oplog entry. * - * When fromMigrate is set, the related oplog entry will be marked with a 'fromMigrate' field to - * reduce its visibility (e.g. in change streams). - * * The caller should hold a DB X lock and ensure there are no index builds in progress on the * collection. */ Status dropCollection(OperationContext* opCtx, NamespaceString nss, - repl::OpTime dropOpTime, - bool markFromMigrate) const final; + repl::OpTime dropOpTime) const final; Status dropCollectionEvenIfSystem(OperationContext* opCtx, NamespaceString nss, repl::OpTime dropOpTime, - bool markFromMigrate) const final; + bool markFromMigrate = false) const final; Status dropView(OperationContext* opCtx, NamespaceString viewName) const final; diff --git a/src/mongo/db/catalog/database_test.cpp b/src/mongo/db/catalog/database_test.cpp index c4669cd85b2..dd72c38588c 100644 --- a/src/mongo/db/catalog/database_test.cpp +++ b/src/mongo/db/catalog/database_test.cpp @@ -39,7 +39,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/jsobj.h" @@ -261,7 +261,7 @@ void _testDropCollectionThrowsExceptionIfThereAreIndexesInProgress(OperationCont collection->ns(), indexInfoObj, IndexBuildMethod::kHybrid, UUID::gen()); { WriteUnitOfWork wuow(opCtx); - ASSERT_OK(indexBuildBlock->init(opCtx, collection, /*forRecovery=*/false)); + ASSERT_OK(indexBuildBlock->init(opCtx, collection)); wuow.commit(); } ON_BLOCK_EXIT([&indexBuildBlock, opCtx, collection] { diff --git a/src/mongo/db/catalog/drop_collection.cpp b/src/mongo/db/catalog/drop_collection.cpp index 12e78022b2b..f04a8fa2b79 100644 --- a/src/mongo/db/catalog/drop_collection.cpp +++ b/src/mongo/db/catalog/drop_collection.cpp @@ -36,10 +36,9 @@ #include "mongo/db/audit.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_uuid_mismatch.h" -#include "mongo/db/catalog/collection_uuid_mismatch_info.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index_builds_coordinator.h" @@ -57,16 +56,7 @@ namespace { MONGO_FAIL_POINT_DEFINE(hangDropCollectionBeforeLockAcquisition); MONGO_FAIL_POINT_DEFINE(hangDuringDropCollection); -Status _checkNssAndReplState(OperationContext* opCtx, - const CollectionPtr& coll, - const NamespaceString& nss, - const boost::optional<UUID>& expectedUUID = boost::none) { - try { - checkCollectionUUIDMismatch(opCtx, nss, coll, expectedUUID); - } catch (const DBException& ex) { - return ex.toStatus(); - } - +Status _checkNssAndReplState(OperationContext* opCtx, const CollectionPtr& coll) { if (!coll) { return Status(ErrorCodes::NamespaceNotFound, "ns not found"); } @@ -205,13 +195,19 @@ Status _abortIndexBuildsAndDrop(OperationContext* opCtx, CollectionPtr coll = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, startingNss); - Status status = _checkNssAndReplState(opCtx, coll, startingNss, expectedUUID); + Status status = _checkNssAndReplState(opCtx, coll); if (!status.isOK()) { return status; } warnEncryptedCollectionsIfNeeded(opCtx, coll); + try { + checkCollectionUUIDMismatch(opCtx, startingNss, coll, expectedUUID); + } catch (const DBException& ex) { + return ex.toStatus(); + } + if (MONGO_unlikely(hangDuringDropCollection.shouldFail())) { LOGV2(518090, "hangDuringDropCollection fail point enabled. Blocking until fail point is " @@ -260,7 +256,7 @@ Status _abortIndexBuildsAndDrop(OperationContext* opCtx, opCtx->recoveryUnit()->abandonSnapshot(); coll = CollectionCatalog::get(opCtx)->lookupCollectionByUUID(opCtx, collectionUUID); - status = _checkNssAndReplState(opCtx, coll, startingNss, expectedUUID); + status = _checkNssAndReplState(opCtx, coll); if (!status.isOK()) { return status; } @@ -308,7 +304,7 @@ Status _dropCollectionForApplyOps(OperationContext* opCtx, Lock::CollectionLock collLock(opCtx, collectionName, MODE_X); const CollectionPtr& coll = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, collectionName); - Status status = _checkNssAndReplState(opCtx, coll, collectionName); + Status status = _checkNssAndReplState(opCtx, coll); if (!status.isOK()) { return status; } @@ -352,7 +348,6 @@ Status _dropCollection(OperationContext* opCtx, const boost::optional<UUID>& expectedUUID, DropReply* reply, DropCollectionSystemCollectionMode systemCollectionMode, - bool fromMigrate, boost::optional<UUID> dropIfUUIDNotMatching = boost::none) { try { @@ -360,13 +355,7 @@ Status _dropCollection(OperationContext* opCtx, AutoGetDb autoDb(opCtx, collectionName.db(), MODE_IX); auto db = autoDb.getDb(); if (!db) { - return expectedUUID - ? Status{CollectionUUIDMismatchInfo(collectionName.db().toString(), - *expectedUUID, - collectionName.coll().toString(), - boost::none), - "Database does not exist"} - : Status(ErrorCodes::NamespaceNotFound, "ns not found"); + return Status(ErrorCodes::NamespaceNotFound, "ns not found"); } if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, collectionName)) { @@ -375,14 +364,13 @@ Status _dropCollection(OperationContext* opCtx, std::move(autoDb), collectionName, expectedUUID, - [opCtx, systemCollectionMode, fromMigrate](Database* db, - const NamespaceString& resolvedNs) { + [opCtx, systemCollectionMode](Database* db, const NamespaceString& resolvedNs) { WriteUnitOfWork wuow(opCtx); auto status = systemCollectionMode == DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops - ? db->dropCollection(opCtx, resolvedNs, {}, fromMigrate) - : db->dropCollectionEvenIfSystem(opCtx, resolvedNs, {}, fromMigrate); + ? db->dropCollection(opCtx, resolvedNs) + : db->dropCollectionEvenIfSystem(opCtx, resolvedNs); if (!status.isOK()) { return status; } @@ -395,18 +383,14 @@ Status _dropCollection(OperationContext* opCtx, dropIfUUIDNotMatching); } - auto dropTimeseries = [opCtx, - &expectedUUID, - &autoDb, - &collectionName, - &reply, - fromMigrate](const NamespaceString& bucketNs, bool dropView) { + auto dropTimeseries = [opCtx, &expectedUUID, &autoDb, &collectionName, &reply]( + const NamespaceString& bucketNs, bool dropView) { return _abortIndexBuildsAndDrop( opCtx, std::move(autoDb), bucketNs, expectedUUID, - [opCtx, dropView, &expectedUUID, &collectionName, &reply, fromMigrate]( + [opCtx, dropView, &expectedUUID, &collectionName, &reply]( Database* db, const NamespaceString& bucketsNs) { // Disallow checking the expectedUUID when dropping time-series collections. uassert(ErrorCodes::InvalidOptions, @@ -430,13 +414,11 @@ Status _dropCollection(OperationContext* opCtx, // Drop the buckets collection in its own writeConflictRetry so that if // it throws a WCE, only the buckets collection drop is retried. - writeConflictRetry( - opCtx, "drop", bucketsNs.ns(), [opCtx, db, &bucketsNs, fromMigrate] { - WriteUnitOfWork wuow(opCtx); - db->dropCollectionEvenIfSystem(opCtx, bucketsNs, {}, fromMigrate) - .ignore(); - wuow.commit(); - }); + writeConflictRetry(opCtx, "drop", bucketsNs.ns(), [opCtx, db, &bucketsNs] { + WriteUnitOfWork wuow(opCtx); + db->dropCollectionEvenIfSystem(opCtx, bucketsNs).ignore(); + wuow.commit(); + }); return Status::OK(); }, @@ -486,8 +468,7 @@ Status dropCollection(OperationContext* opCtx, const NamespaceString& nss, const boost::optional<UUID>& expectedUUID, DropReply* reply, - DropCollectionSystemCollectionMode systemCollectionMode, - bool fromMigrate) { + DropCollectionSystemCollectionMode systemCollectionMode) { if (!serverGlobalParams.quiet.load()) { LOGV2(518070, "CMD: drop", logAttrs(nss)); } @@ -502,16 +483,14 @@ Status dropCollection(OperationContext* opCtx, const auto collectionName = nss.isTimeseriesBucketsCollection() ? nss.getTimeseriesViewNamespace() : nss; - return _dropCollection( - opCtx, collectionName, expectedUUID, reply, systemCollectionMode, fromMigrate); + return _dropCollection(opCtx, collectionName, expectedUUID, reply, systemCollectionMode); } Status dropCollection(OperationContext* opCtx, const NamespaceString& nss, DropReply* reply, - DropCollectionSystemCollectionMode systemCollectionMode, - bool fromMigrate) { - return dropCollection(opCtx, nss, boost::none, reply, systemCollectionMode, fromMigrate); + DropCollectionSystemCollectionMode systemCollectionMode) { + return dropCollection(opCtx, nss, boost::none, reply, systemCollectionMode); } Status dropCollectionIfUUIDNotMatching(OperationContext* opCtx, @@ -533,7 +512,6 @@ Status dropCollectionIfUUIDNotMatching(OperationContext* opCtx, boost::none, &repl, DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops, - false /*fromMigrate*/, expectedUUID); } diff --git a/src/mongo/db/catalog/drop_collection.h b/src/mongo/db/catalog/drop_collection.h index 016bc5e3d1a..7f857d88547 100644 --- a/src/mongo/db/catalog/drop_collection.h +++ b/src/mongo/db/catalog/drop_collection.h @@ -50,21 +50,17 @@ enum class DropCollectionSystemCollectionMode { * Drops the collection "collectionName" and populates "reply" with statistics about what * was removed. Aborts in-progress index builds on the collection if two phase index builds are * supported. Throws if the expectedUUID does not match the UUID of the collection being dropped. - * When fromMigrate is set, the related oplog entry will be marked accordingly using the - * 'fromMigrate' field to reduce its visibility (e.g. in change streams). */ Status dropCollection(OperationContext* opCtx, const NamespaceString& collectionName, const boost::optional<UUID>& expectedUUID, DropReply* reply, - DropCollectionSystemCollectionMode systemCollectionMode, - bool fromMigrate = false); + DropCollectionSystemCollectionMode systemCollectionMode); Status dropCollection(OperationContext* opCtx, const NamespaceString& collectionName, DropReply* reply, - DropCollectionSystemCollectionMode systemCollectionMode, - bool fromMigrate = false); + DropCollectionSystemCollectionMode systemCollectionMode); /** * Drops the collection "collectionName" only if its uuid is not matching "expectedUUID". diff --git a/src/mongo/db/catalog/drop_database.cpp b/src/mongo/db/catalog/drop_database.cpp index 5e11a71dbfc..85e2087a321 100644 --- a/src/mongo/db/catalog/drop_database.cpp +++ b/src/mongo/db/catalog/drop_database.cpp @@ -38,7 +38,7 @@ #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/op_observer.h" @@ -99,19 +99,17 @@ void _finishDropDatabase(OperationContext* opCtx, IndexBuildsCoordinator::get(opCtx)->assertNoBgOpInProgForDb(dbName); } - // Testing depends on this failpoint stopping execution before the dropDatabase oplog entry is - // written, as well as before the in-memory state is cleared. - if (MONGO_unlikely(dropDatabaseHangBeforeInMemoryDrop.shouldFail())) { - LOGV2(20334, "dropDatabase - fail point dropDatabaseHangBeforeInMemoryDrop enabled"); - dropDatabaseHangBeforeInMemoryDrop.pauseWhileSet(); - } - writeConflictRetry(opCtx, "dropDatabase_database", dbName, [&] { WriteUnitOfWork wunit(opCtx); opCtx->getServiceContext()->getOpObserver()->onDropDatabase(opCtx, dbName); wunit.commit(); }); + if (MONGO_unlikely(dropDatabaseHangBeforeInMemoryDrop.shouldFail())) { + LOGV2(20334, "dropDatabase - fail point dropDatabaseHangBeforeInMemoryDrop enabled"); + dropDatabaseHangBeforeInMemoryDrop.pauseWhileSet(); + } + auto databaseHolder = DatabaseHolder::get(opCtx); databaseHolder->dropDb(opCtx, db); dropPendingGuard.dismiss(); @@ -229,7 +227,9 @@ Status _dropDatabase(OperationContext* opCtx, const std::string& dbName, bool ab std::vector<NamespaceString> collectionsToDrop; auto catalog = CollectionCatalog::get(opCtx); - for (auto&& collection : catalog->range(db->name())) { + for (auto collIt = catalog->begin(opCtx, db->name()); collIt != catalog->end(opCtx); + ++collIt) { + auto collection = *collIt; if (!collection) { break; } diff --git a/src/mongo/db/catalog/drop_database_test.cpp b/src/mongo/db/catalog/drop_database_test.cpp index e14cf74b809..0e46980f33c 100644 --- a/src/mongo/db/catalog/drop_database_test.cpp +++ b/src/mongo/db/catalog/drop_database_test.cpp @@ -36,7 +36,7 @@ #include "mongo/db/catalog/database_holder.h" #include "mongo/db/catalog/drop_database.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/jsobj.h" #include "mongo/db/namespace_string.h" diff --git a/src/mongo/db/catalog/drop_indexes.cpp b/src/mongo/db/catalog/drop_indexes.cpp index 4720a7ee83c..2c10eb005c5 100644 --- a/src/mongo/db/catalog/drop_indexes.cpp +++ b/src/mongo/db/catalog/drop_indexes.cpp @@ -39,7 +39,7 @@ #include "mongo/db/catalog/collection_uuid_mismatch.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" @@ -50,9 +50,7 @@ #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/database_sharding_state.h" #include "mongo/db/s/shard_key_index_util.h" -#include "mongo/db/server_feature_flags_gen.h" #include "mongo/db/service_context.h" -#include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/logv2/log.h" #include "mongo/util/visit_helper.h" @@ -109,13 +107,9 @@ Status checkReplState(OperationContext* opCtx, StatusWith<const IndexDescriptor*> getDescriptorByKeyPattern(OperationContext* opCtx, const IndexCatalog* indexCatalog, const BSONObj& keyPattern) { + const bool includeUnfinished = true; std::vector<const IndexDescriptor*> indexes; - indexCatalog->findIndexesByKeyPattern(opCtx, - keyPattern, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen, - &indexes); + indexCatalog->findIndexesByKeyPattern(opCtx, keyPattern, includeUnfinished, &indexes); if (indexes.empty()) { return Status(ErrorCodes::IndexNotFound, str::stream() << "can't find index with key: " << keyPattern); @@ -309,24 +303,12 @@ void dropReadyIndexes(OperationContext* opCtx, if (desc->isIdIndex()) { return false; } - // For any index that is compatible with the shard key, if - // gFeatureFlagShardKeyIndexOptionalHashedSharding is enabled and - // the shard key is hashed, allow users to drop the hashed index. Note - // skipDroppingHashedShardKeyIndex is used in some tests to prevent dropIndexes - // from dropping the hashed shard key index so we can continue to test chunk - // migration with hashed sharding. Otherwise, dropIndexes with '*' would drop - // the index and prevent chunk migration from running. - const auto& shardKey = collDescription.getShardKeyPattern(); - const bool skipDropIndex = skipDroppingHashedShardKeyIndex || - !(gFeatureFlagShardKeyIndexOptionalHashedSharding.isEnabled( - serverGlobalParams.featureCompatibility) && - shardKey.isHashedPattern()); + if (isCompatibleWithShardKey(opCtx, CollectionPtr(collection), desc->getEntry(), - shardKey.toBSON(), - false /* requiresSingleKey */) && - skipDropIndex) { + collDescription.getKeyPattern(), + false /* requiresSingleKey */)) { return false; } @@ -356,20 +338,17 @@ void dropReadyIndexes(OperationContext* opCtx, return; } + bool includeUnfinished = true; for (const auto& indexName : indexNames) { if (collDescription.isSharded()) { uassert( ErrorCodes::CannotDropShardKeyIndex, "Cannot drop the only compatible index for this collection's shard key", - !isLastNonHiddenRangedShardKeyIndex( + !isLastShardKeyIndex( opCtx, collection, indexCatalog, indexName, collDescription.getKeyPattern())); } - auto desc = indexCatalog->findIndexByName(opCtx, - indexName, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + auto desc = indexCatalog->findIndexByName(opCtx, indexName, includeUnfinished); if (!desc) { uasserted(ErrorCodes::IndexNotFound, str::stream() << "index not found with name [" << indexName << "]"); @@ -528,6 +507,7 @@ DropIndexesReply dropIndexes(OperationContext* opCtx, // the index catalog. This would indicate that while we yielded our locks during the // abort phase, a new identical index was created. auto indexCatalog = collection->getWritableCollection(opCtx)->getIndexCatalog(); + const bool includeUnfinished = false; for (const auto& indexName : indexNames) { auto collDescription = CollectionShardingState::get(opCtx, nss)->getCollectionDescription(opCtx); @@ -535,19 +515,14 @@ DropIndexesReply dropIndexes(OperationContext* opCtx, if (collDescription.isSharded()) { uassert(ErrorCodes::CannotDropShardKeyIndex, "Cannot drop the only compatible index for this collection's shard key", - !isLastNonHiddenRangedShardKeyIndex(opCtx, - collection->getCollection(), - indexCatalog, - indexName, - collDescription.getKeyPattern())); + !isLastShardKeyIndex(opCtx, + collection->getCollection(), + indexCatalog, + indexName, + collDescription.getKeyPattern())); } - auto desc = - indexCatalog->findIndexByName(opCtx, - indexName, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + auto desc = indexCatalog->findIndexByName(opCtx, indexName, includeUnfinished); if (!desc) { // A similar index wasn't created while we yielded the locks during abort. continue; diff --git a/src/mongo/db/catalog/health_log.cpp b/src/mongo/db/catalog/health_log.cpp index 9626d456111..aa49302f169 100644 --- a/src/mongo/db/catalog/health_log.cpp +++ b/src/mongo/db/catalog/health_log.cpp @@ -27,14 +27,19 @@ * it in the license file. */ +#include "mongo/platform/basic.h" + #include "mongo/db/catalog/health_log.h" #include "mongo/db/catalog/health_log_gen.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" -#include "mongo/db/namespace_string.h" namespace mongo { namespace { +const ServiceContext::Decoration<HealthLog> getHealthLog = + ServiceContext::declareDecoration<HealthLog>(); + const int64_t kDefaultHealthlogSize = 100'000'000; CollectionOptions getOptions(void) { @@ -46,17 +51,24 @@ CollectionOptions getOptions(void) { } } // namespace -HealthLog::HealthLog() - : _writer(NamespaceString::kLocalHealthLogNamespace, getOptions(), kMaxBufferSize) {} +HealthLog::HealthLog() : _writer(nss, getOptions(), kMaxBufferSize) {} -void HealthLog::startup() { +void HealthLog::startup(void) { _writer.startup(std::string("healthlog writer")); } -void HealthLog::shutdown() { +void HealthLog::shutdown(void) { _writer.shutdown(); } +HealthLog& HealthLog::get(ServiceContext* svcCtx) { + return getHealthLog(svcCtx); +} + +HealthLog& HealthLog::get(OperationContext* opCtx) { + return getHealthLog(opCtx->getServiceContext()); +} + bool HealthLog::log(const HealthLogEntry& entry) { BSONObjBuilder builder; OID oid; @@ -65,4 +77,6 @@ bool HealthLog::log(const HealthLogEntry& entry) { entry.serialize(&builder); return _writer.insertDocument(builder.obj()); } + +const NamespaceString HealthLog::nss("local", "system.healthlog"); } // namespace mongo diff --git a/src/mongo/db/catalog/health_log.h b/src/mongo/db/catalog/health_log.h index f9fe3c5b4c3..ba2bcbf440a 100644 --- a/src/mongo/db/catalog/health_log.h +++ b/src/mongo/db/catalog/health_log.h @@ -29,14 +29,21 @@ #pragma once -#include "mongo/db/catalog/health_log_interface.h" #include "mongo/db/concurrency/deferred_writer.h" +#include "mongo/db/service_context.h" namespace mongo { class HealthLogEntry; -class HealthLog : public HealthLogInterface { +/** + * The interface to the local healthlog. + * + * This class contains facilities for creating and asynchronously writing to the local healthlog + * collection. There should only be one instance of this class, initialized on startup and cleaned + * up on shutdown. + */ +class HealthLog { HealthLog(const HealthLog&) = delete; HealthLog& operator=(const HealthLog&) = delete; @@ -48,11 +55,38 @@ public: */ HealthLog(); - void startup() override; + /** + * The maximum size of the in-memory buffer of health-log entries, in bytes. + */ + static const int64_t kMaxBufferSize = 25'000'000; + + /** + * Start the worker thread writing the buffer to the collection. + */ + void startup(void); - void shutdown() override; + /** + * Stop the worker thread. + */ + void shutdown(void); - bool log(const HealthLogEntry& entry) override; + /** + * The name of the collection. + */ + static const NamespaceString nss; + + /** + * Get the current context's HealthLog. + */ + static HealthLog& get(ServiceContext* ctx); + static HealthLog& get(OperationContext* ctx); + + /** + * Asynchronously insert the given entry. + * + * Return `false` iff there is no more space in the buffer. + */ + bool log(const HealthLogEntry& entry); private: DeferredWriter _writer; diff --git a/src/mongo/db/catalog/health_log_interface.cpp b/src/mongo/db/catalog/health_log_interface.cpp deleted file mode 100644 index 69e8f03020f..00000000000 --- a/src/mongo/db/catalog/health_log_interface.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright (C) 2023-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/catalog/health_log_interface.h" -#include "mongo/db/operation_context.h" - -namespace mongo { - -namespace { -const auto getHealthLog = ServiceContext::declareDecoration<std::unique_ptr<HealthLogInterface>>(); -} // namespace - -void HealthLogInterface::set(ServiceContext* serviceContext, - std::unique_ptr<HealthLogInterface> newHealthLog) { - auto& healthLog = getHealthLog(serviceContext); - invariant(!healthLog); - - healthLog = std::move(newHealthLog); -} - -HealthLogInterface* HealthLogInterface::get(ServiceContext* svcCtx) { - return getHealthLog(svcCtx).get(); -} - -HealthLogInterface* HealthLogInterface::get(OperationContext* opCtx) { - return getHealthLog(opCtx->getServiceContext()).get(); -} -} // namespace mongo diff --git a/src/mongo/db/catalog/health_log_interface.h b/src/mongo/db/catalog/health_log_interface.h deleted file mode 100644 index 885015fb6be..00000000000 --- a/src/mongo/db/catalog/health_log_interface.h +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Copyright (C) 2023-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/db/catalog/health_log_gen.h" -#include "mongo/db/service_context.h" - -namespace mongo { - -/** - * The interface to the local healthlog. - * - * This class contains facilities for creating and asynchronously writing to the local healthlog - * collection. There should only be one instance of this class, initialized on startup and cleaned - * up on shutdown. - */ -class HealthLogInterface { - HealthLogInterface(const HealthLogInterface&) = delete; - HealthLogInterface& operator=(const HealthLogInterface&) = delete; - -public: - /** - * The maximum size of the in-memory buffer of health-log entries, in bytes. - */ - static const int64_t kMaxBufferSize = 25'000'000; - - /** - * Stores a health log on the specified service context. May only be called once for the - * lifetime of the service context. - */ - static void set(ServiceContext* serviceContext, - std::unique_ptr<HealthLogInterface> newHealthLog); - - /** - * Get the current context's HealthLog. set() above must be called before any get() calls. - */ - static HealthLogInterface* get(ServiceContext* ctx); - static HealthLogInterface* get(OperationContext* ctx); - - /** - * Required to use HealthLogInterface as a ServiceContext decorator. - * - * Should not be used anywhere else. - */ - HealthLogInterface() = default; - virtual ~HealthLogInterface() = default; - - /** - * Start the worker thread writing the buffer to the collection. - */ - virtual void startup() = 0; - - /** - * Stop the worker thread. - */ - virtual void shutdown() = 0; - - /** - * Asynchronously insert the given entry. - * - * Return `false` iff there is no more space in the buffer. - */ - virtual bool log(const HealthLogEntry& entry) = 0; -}; -} // namespace mongo diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp index 34dbf0c49ad..6174e891b64 100644 --- a/src/mongo/db/catalog/index_build_block.cpp +++ b/src/mongo/db/catalog/index_build_block.cpp @@ -44,6 +44,7 @@ #include "mongo/db/query/collection_index_usage_tracker_decoration.h" #include "mongo/db/query/collection_query_info.h" #include "mongo/db/storage/durable_catalog.h" +#include "mongo/db/storage/storage_parameters_gen.h" #include "mongo/db/ttl_collection_cache.h" #include "mongo/db/vector_clock.h" #include "mongo/logv2/log.h" @@ -57,7 +58,14 @@ IndexBuildBlock::IndexBuildBlock(const NamespaceString& nss, const BSONObj& spec, IndexBuildMethod method, boost::optional<UUID> indexBuildUUID) - : _nss(nss), _spec(spec.getOwned()), _method(method), _buildUUID(indexBuildUUID) {} + : _nss(nss), + _spec(spec.getOwned()), + _method(method), + _buildUUID(indexBuildUUID), + _pooledBuilder( + gOperationMemoryPoolBlockInitialSizeKB.loadRelaxed() * static_cast<size_t>(1024), + SharedBufferFragmentBuilder::DoubleGrowStrategy( + gOperationMemoryPoolBlockMaxSizeKB.loadRelaxed() * static_cast<size_t>(1024))) {} void IndexBuildBlock::keepTemporaryTables() { if (_indexBuildInterceptor) { @@ -75,11 +83,6 @@ void IndexBuildBlock::_completeInit(OperationContext* opCtx, Collection* collect .registerIndex(desc->indexName(), desc->keyPattern(), IndexFeatures::make(desc, collection->ns().isOnInternalDb())); - opCtx->recoveryUnit()->onRollback( - [collectionDecorations = collection->getSharedDecorations(), indexName = _indexName] { - CollectionIndexUsageTrackerDecoration::get(collectionDecorations) - .unregisterIndex(indexName); - }); } Status IndexBuildBlock::initForResume(OperationContext* opCtx, @@ -89,9 +92,7 @@ Status IndexBuildBlock::initForResume(OperationContext* opCtx, _indexName = _spec.getStringField("name").toString(); auto descriptor = collection->getIndexCatalog()->findIndexByName( - opCtx, - _indexName, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + opCtx, _indexName, true /* includeUnfinishedIndexes */); auto indexCatalogEntry = descriptor->getEntry(); @@ -127,7 +128,7 @@ Status IndexBuildBlock::initForResume(OperationContext* opCtx, return Status::OK(); } -Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection, bool forRecovery) { +Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) { // Being in a WUOW means all timestamping responsibility can be pushed up to the caller. invariant(opCtx->lockState()->inAWriteUnitOfWork()); @@ -155,25 +156,14 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection, bo !replCoord->getMemberState().primary() && isBackgroundIndex; } - if (!forRecovery) { - // Setup on-disk structures. We skip this during startup recovery for unfinished indexes as - // everything is already in-place. - Status status = collection->prepareForIndexBuild( - opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild); - if (!status.isOK()) - return status; - } + // Setup on-disk structures. + Status status = collection->prepareForIndexBuild( + opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild); + if (!status.isOK()) + return status; - auto indexCatalog = collection->getIndexCatalog(); - IndexCatalogEntry* indexCatalogEntry = nullptr; - if (forRecovery) { - auto desc = indexCatalog->findIndexByName( - opCtx, _indexName, IndexCatalog::InclusionPolicy::kUnfinished); - indexCatalogEntry = desc->getEntry(); - } else { - indexCatalogEntry = indexCatalog->createIndexEntry( - opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone); - } + auto indexCatalogEntry = collection->getIndexCatalog()->createIndexEntry( + opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone); if (_method == IndexBuildMethod::kHybrid) { _indexBuildInterceptor = std::make_unique<IndexBuildInterceptor>(opCtx, indexCatalogEntry); @@ -284,10 +274,7 @@ void IndexBuildBlock::success(OperationContext* opCtx, Collection* collection) { // Note that TTL deletion is supported on capped clustered collections via bounded // collection scan, which does not use an index. if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) && !coll->isCapped()) { - TTLCollectionCache::get(svcCtx).registerTTLInfo( - coll->uuid(), - TTLCollectionCache::Info{ - indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()}); + TTLCollectionCache::get(svcCtx).registerTTLInfo(coll->uuid(), indexName); } }); } @@ -295,18 +282,14 @@ void IndexBuildBlock::success(OperationContext* opCtx, Collection* collection) { const IndexCatalogEntry* IndexBuildBlock::getEntry(OperationContext* opCtx, const CollectionPtr& collection) const { auto descriptor = collection->getIndexCatalog()->findIndexByName( - opCtx, - _indexName, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + opCtx, _indexName, true /* includeUnfinishedIndexes */); return descriptor->getEntry(); } IndexCatalogEntry* IndexBuildBlock::getEntry(OperationContext* opCtx, Collection* collection) { auto descriptor = collection->getIndexCatalog()->findIndexByName( - opCtx, - _indexName, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished); + opCtx, _indexName, true /* includeUnfinishedIndexes */); return descriptor->getEntry(); } diff --git a/src/mongo/db/catalog/index_build_block.h b/src/mongo/db/catalog/index_build_block.h index 2b94849b46c..48d0f6e49af 100644 --- a/src/mongo/db/catalog/index_build_block.h +++ b/src/mongo/db/catalog/index_build_block.h @@ -62,7 +62,7 @@ public: * * Must be called from within a `WriteUnitOfWork` */ - Status init(OperationContext* opCtx, Collection* collection, bool forRecovery); + Status init(OperationContext* opCtx, Collection* collection); /** * Makes sure that an entry for the index was created at startup in the IndexCatalog. Returns @@ -111,6 +111,13 @@ public: return _spec; } + /** + * Returns a memory pool for creating temporary objects for this index build. + */ + SharedBufferFragmentBuilder& getPooledBuilder() { + return _pooledBuilder; + } + private: void _completeInit(OperationContext* opCtx, Collection* collection); @@ -124,5 +131,7 @@ private: std::string _indexNamespace; std::unique_ptr<IndexBuildInterceptor> _indexBuildInterceptor; + + SharedBufferFragmentBuilder _pooledBuilder; }; } // namespace mongo diff --git a/src/mongo/db/catalog/index_build_entry_test.cpp b/src/mongo/db/catalog/index_build_entry_test.cpp index d53c7e0046c..ffa6a870874 100644 --- a/src/mongo/db/catalog/index_build_entry_test.cpp +++ b/src/mongo/db/catalog/index_build_entry_test.cpp @@ -32,7 +32,6 @@ #include <string> #include <vector> -#include "mongo/bson/bson_validate.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/bsontypes.h" @@ -125,7 +124,7 @@ TEST(IndexBuildEntryTest, SerializeAndDeserialize) { entry.setCommitReadyMembers(generateCommitReadyMembers(3)); BSONObj obj = entry.toBSON(); - ASSERT_TRUE(validateBSON(obj).isOK()); + ASSERT_TRUE(obj.valid()); IDLParserErrorContext ctx("IndexBuildsEntry Parser"); IndexBuildEntry rebuiltEntry = IndexBuildEntry::parse(ctx, obj); diff --git a/src/mongo/db/catalog/index_builds_manager.cpp b/src/mongo/db/catalog/index_builds_manager.cpp index 0c44c44de48..82483f9cf21 100644 --- a/src/mongo/db/catalog/index_builds_manager.cpp +++ b/src/mongo/db/catalog/index_builds_manager.cpp @@ -38,7 +38,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_repair.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/storage/storage_repair_observer.h" @@ -113,8 +113,7 @@ Status IndexBuildsManager::setUpIndexBuild(OperationContext* opCtx, std::vector<BSONObj> indexes; try { indexes = writeConflictRetry(opCtx, "IndexBuildsManager::setUpIndexBuild", nss.ns(), [&]() { - return uassertStatusOK( - builder->init(opCtx, collection, specs, onInit, options.forRecovery, resumeInfo)); + return uassertStatusOK(builder->init(opCtx, collection, specs, onInit, resumeInfo)); }); } catch (const DBException& ex) { return ex.toStatus(); diff --git a/src/mongo/db/catalog/index_builds_manager.h b/src/mongo/db/catalog/index_builds_manager.h index 37614751eb5..cddb46c3d16 100644 --- a/src/mongo/db/catalog/index_builds_manager.h +++ b/src/mongo/db/catalog/index_builds_manager.h @@ -73,7 +73,6 @@ public: IndexConstraints indexConstraints = IndexConstraints::kEnforce; IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase; IndexBuildMethod method = IndexBuildMethod::kHybrid; - bool forRecovery = false; }; IndexBuildsManager() = default; diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h index 6f4fa0c8ae3..2754ac3e22f 100644 --- a/src/mongo/db/catalog/index_catalog.h +++ b/src/mongo/db/catalog/index_catalog.h @@ -197,12 +197,6 @@ public: std::unique_ptr<std::vector<IndexCatalogEntry*>> _ownedContainer; }; - enum class InclusionPolicy { - kReady = 1 << 0, - kUnfinished = 1 << 1, - kFrozen = 1 << 2, - }; - IndexCatalog() = default; virtual ~IndexCatalog() = default; @@ -243,10 +237,9 @@ public: * * @return null if cannot find */ - virtual const IndexDescriptor* findIndexByName( - OperationContext* opCtx, - StringData name, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0; + virtual const IndexDescriptor* findIndexByName(OperationContext* opCtx, + StringData name, + bool includeUnfinishedIndexes = false) const = 0; /** * Find index by matching key pattern and options. The key pattern, collation spec, and partial @@ -258,7 +251,7 @@ public: OperationContext* opCtx, const BSONObj& key, const BSONObj& indexSpec, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0; + bool includeUnfinishedIndexes = false) const = 0; /** * Find indexes with a matching key pattern, putting them into the vector 'matches'. The key @@ -268,13 +261,12 @@ public: */ virtual void findIndexesByKeyPattern(OperationContext* opCtx, const BSONObj& key, - InclusionPolicy inclusionPolicy, + bool includeUnfinishedIndexes, std::vector<const IndexDescriptor*>* matches) const = 0; - virtual void findIndexByType( - OperationContext* opCtx, - const std::string& type, - std::vector<const IndexDescriptor*>& matches, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0; + virtual void findIndexByType(OperationContext* opCtx, + const std::string& type, + std::vector<const IndexDescriptor*>& matches, + bool includeUnfinishedIndexes = false) const = 0; /** * Reload the index definition for 'oldDesc' from the CollectionCatalogEntry. 'oldDesc' @@ -316,7 +308,7 @@ public: * Returns an iterator for the index descriptors in this IndexCatalog. */ virtual std::unique_ptr<IndexIterator> getIndexIterator( - OperationContext* opCtx, InclusionPolicy inclusionPolicy) const = 0; + OperationContext* opCtx, bool includeUnfinishedIndexes) const = 0; // ---- index set modifiers ------ @@ -419,16 +411,6 @@ public: const IndexDescriptor* desc) = 0; /** - * Resets the index given its descriptor. - * - * This can only be called during startup recovery as it involves recreating the index table to - * allow bulk cursors to be used again. - */ - virtual Status resetUnfinishedIndexForRecovery(OperationContext* opCtx, - Collection* collection, - const IndexDescriptor* desc) = 0; - - /** * Drops an unfinished index given its descriptor. * * The caller must hold the collection X lock. @@ -539,16 +521,4 @@ public: Collection* coll, IndexCatalogEntry* index) = 0; }; - -inline IndexCatalog::InclusionPolicy operator|(IndexCatalog::InclusionPolicy lhs, - IndexCatalog::InclusionPolicy rhs) { - return static_cast<IndexCatalog::InclusionPolicy>( - static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(lhs) | - static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(rhs)); -} - -inline bool operator&(IndexCatalog::InclusionPolicy lhs, IndexCatalog::InclusionPolicy rhs) { - return static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(lhs) & - static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(rhs); -} } // namespace mongo diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h index 2cf80bb8d3f..fa77370bb96 100644 --- a/src/mongo/db/catalog/index_catalog_entry.h +++ b/src/mongo/db/catalog/index_catalog_entry.h @@ -64,6 +64,8 @@ public: inline IndexCatalogEntry(IndexCatalogEntry&&) = delete; inline IndexCatalogEntry& operator=(IndexCatalogEntry&&) = delete; + virtual void init(std::unique_ptr<IndexAccessMethod> accessMethod) = 0; + virtual const std::string& getIdent() const = 0; virtual std::shared_ptr<Ident> getSharedIdent() const = 0; @@ -73,8 +75,6 @@ public: virtual IndexAccessMethod* accessMethod() const = 0; - virtual void setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) = 0; - virtual bool isHybridBuilding() const = 0; virtual IndexBuildInterceptor* indexBuildInterceptor() const = 0; @@ -95,7 +95,6 @@ public: /// --------------------- virtual void setIsReady(bool newIsReady) = 0; - virtual void setIsFrozen(bool newIsFrozen) = 0; virtual void setDropped() = 0; virtual bool isDropped() const = 0; diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.cpp b/src/mongo/db/catalog/index_catalog_entry_impl.cpp index e104623ef2d..daf74714c84 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_entry_impl.cpp @@ -39,7 +39,7 @@ #include "mongo/base/init.h" #include "mongo/db/catalog/collection.h" #include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/matcher/expression.h" @@ -122,7 +122,7 @@ IndexCatalogEntryImpl::IndexCatalogEntryImpl(OperationContext* const opCtx, } } -void IndexCatalogEntryImpl::setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) { +void IndexCatalogEntryImpl::init(std::unique_ptr<IndexAccessMethod> accessMethod) { invariant(!_accessMethod); _accessMethod = std::move(accessMethod); } @@ -179,10 +179,6 @@ void IndexCatalogEntryImpl::setIsReady(bool newIsReady) { _isReady = newIsReady; } -void IndexCatalogEntryImpl::setIsFrozen(bool newIsFrozen) { - _isFrozen = newIsFrozen; -} - void IndexCatalogEntryImpl::setMultikey(OperationContext* opCtx, const CollectionPtr& collection, const KeyStringSet& multikeyMetadataKeys, @@ -374,8 +370,7 @@ Status IndexCatalogEntryImpl::_setMultikeyInMultiDocumentTransaction( } std::shared_ptr<Ident> IndexCatalogEntryImpl::getSharedIdent() const { - return _accessMethod ? std::shared_ptr<Ident>{shared_from_this(), _accessMethod->getIdentPtr()} - : nullptr; + return {shared_from_this(), _accessMethod->getIdentPtr()}; // aliasing constructor } // ---- diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.h b/src/mongo/db/catalog/index_catalog_entry_impl.h index 0760989b99a..821176164e1 100644 --- a/src/mongo/db/catalog/index_catalog_entry_impl.h +++ b/src/mongo/db/catalog/index_catalog_entry_impl.h @@ -61,6 +61,8 @@ public: std::unique_ptr<IndexDescriptor> descriptor, // ownership passes to me bool isFrozen); + void init(std::unique_ptr<IndexAccessMethod> accessMethod) final; + const std::string& getIdent() const final { return _ident; } @@ -78,8 +80,6 @@ public: return _accessMethod.get(); } - void setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) final; - bool isHybridBuilding() const final { return _indexBuildInterceptor != nullptr; } @@ -110,8 +110,6 @@ public: void setIsReady(bool newIsReady) final; - void setIsFrozen(bool newIsFrozen) final; - void setDropped() final { _isDropped.store(true); } diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp index 2c3086a54cf..6733dac0ba1 100644 --- a/src/mongo/db/catalog/index_catalog_impl.cpp +++ b/src/mongo/db/catalog/index_catalog_impl.cpp @@ -47,6 +47,7 @@ #include "mongo/db/catalog/uncommitted_catalog_updates.h" #include "mongo/db/client.h" #include "mongo/db/clientcursor.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/field_ref.h" #include "mongo/db/fts/fts_spec.h" @@ -204,30 +205,13 @@ Status IndexCatalogImpl::init(OperationContext* opCtx, Collection* collection) { auto descriptor = std::make_unique<IndexDescriptor>(_getAccessMethodName(keyPattern), spec); - // TTL indexes with NaN 'expireAfterSeconds' cause problems in multiversion settings. - if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName)) { - if (spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()) { - LOGV2_OPTIONS(6852200, - {logv2::LogTag::kStartupWarnings}, - "Found an existing TTL index with NaN 'expireAfterSeconds' in the " - "catalog.", - "ns"_attr = collection->ns(), - "uuid"_attr = collection->uuid(), - "index"_attr = indexName, - "spec"_attr = spec); - } - } - // TTL indexes are not compatible with capped collections. // Note that TTL deletion is supported on capped clustered collections via bounded // collection scan, which does not use an index. if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) && !collection->isCapped()) { TTLCollectionCache::get(opCtx->getServiceContext()) - .registerTTLInfo( - collection->uuid(), - TTLCollectionCache::Info{ - indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()}); + .registerTTLInfo(collection->uuid(), indexName); } bool ready = collection->isIndexReady(indexName); @@ -273,8 +257,8 @@ Status IndexCatalogImpl::init(OperationContext* opCtx, Collection* collection) { } std::unique_ptr<IndexCatalog::IndexIterator> IndexCatalogImpl::getIndexIterator( - OperationContext* const opCtx, InclusionPolicy inclusionPolicy) const { - if (inclusionPolicy == InclusionPolicy::kReady) { + OperationContext* const opCtx, const bool includeUnfinishedIndexes) const { + if (!includeUnfinishedIndexes) { // If the caller only wants the ready indexes, we return an iterator over the catalog's // ready indexes vector. When the user advances this iterator, it will filter out any // indexes that were not ready at the OperationContext's read timestamp. @@ -282,28 +266,17 @@ std::unique_ptr<IndexCatalog::IndexIterator> IndexCatalogImpl::getIndexIterator( opCtx, _readyIndexes.begin(), _readyIndexes.end()); } - // If the caller doesn't only want the ready indexes, for simplicity of implementation, we copy - // the pointers to a new vector. The vector's ownership is passed to the iterator. The query - // code path from an external client is not expected to hit this case so the cost isn't paid by - // the important code path. + // If the caller wants all indexes, for simplicity of implementation, we copy the pointers to + // a new vector. The vector's ownership is passed to the iterator. The query code path from an + // external client is not expected to hit this case so the cost isn't paid by the important + // code path. auto allIndexes = std::make_unique<std::vector<IndexCatalogEntry*>>(); - - if (inclusionPolicy & InclusionPolicy::kReady) { - for (auto it = _readyIndexes.begin(); it != _readyIndexes.end(); ++it) { - allIndexes->push_back(it->get()); - } + for (auto it = _readyIndexes.begin(); it != _readyIndexes.end(); ++it) { + allIndexes->push_back(it->get()); } - if (inclusionPolicy & InclusionPolicy::kUnfinished) { - for (auto it = _buildingIndexes.begin(); it != _buildingIndexes.end(); ++it) { - allIndexes->push_back(it->get()); - } - } - - if (inclusionPolicy & InclusionPolicy::kFrozen) { - for (auto it = _frozenIndexes.begin(); it != _frozenIndexes.end(); ++it) { - allIndexes->push_back(it->get()); - } + for (auto it = _buildingIndexes.begin(); it != _buildingIndexes.end(); ++it) { + allIndexes->push_back(it->get()); } return std::make_unique<AllIndexesIterator>(opCtx, std::move(allIndexes)); @@ -377,7 +350,6 @@ void IndexCatalogImpl::_logInternalState(OperationContext* opCtx, "numIndexesInCollectionCatalogEntry"_attr = numIndexesInCollectionCatalogEntry, "numReadyIndexes"_attr = _readyIndexes.size(), "numBuildingIndexes"_attr = _buildingIndexes.size(), - "numFrozenIndexes"_attr = _frozenIndexes.size(), "indexNamesToDrop"_attr = indexNamesToDrop); // Report the ready indexes. @@ -453,8 +425,7 @@ StatusWith<BSONObj> IndexCatalogImpl::prepareSpecForCreate( } // First check against only the ready indexes for conflicts. - status = - _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, InclusionPolicy::kReady); + status = _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, false); if (!status.isOK()) { return status; } @@ -470,12 +441,7 @@ StatusWith<BSONObj> IndexCatalogImpl::prepareSpecForCreate( // The index catalog cannot currently iterate over only in-progress indexes. So by previously // checking against only ready indexes without error, we know that any errors encountered // checking against all indexes occurred due to an in-progress index. - status = _doesSpecConflictWithExisting(opCtx, - collection, - validatedSpec, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + status = _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, true); if (!status.isOK()) { if (ErrorCodes::IndexAlreadyExists == status.code()) { // Callers need to be able to distinguish conflicts against ready indexes versus @@ -504,11 +470,8 @@ std::vector<BSONObj> IndexCatalogImpl::removeExistingIndexesNoChecks( // _doesSpecConflictWithExisting currently does more work than we require here: we are only // interested in the index already exists error. if (ErrorCodes::IndexAlreadyExists == - _doesSpecConflictWithExisting(opCtx, - collection, - spec, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished)) { + _doesSpecConflictWithExisting( + opCtx, collection, spec, true /*includeUnfinishedIndexes*/)) { continue; } @@ -571,20 +534,19 @@ IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx, engine->getEngine()->alterIdentMetadata(opCtx, ident, desc, isForceUpdateMetadata); } - if (!frozen) { - const auto& collOptions = collection->getCollectionOptions(); - std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface( - opCtx, collection->ns(), collOptions, ident, desc); - std::unique_ptr<IndexAccessMethod> accessMethod = - IndexAccessMethod::make(entry.get(), std::move(sdi)); - entry->setAccessMethod(std::move(accessMethod)); - } + const auto& collOptions = collection->getCollectionOptions(); + std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface( + opCtx, collection->ns(), collOptions, ident, desc); + + std::unique_ptr<IndexAccessMethod> accessMethod = + IndexAccessMethodFactory::get(opCtx)->make(entry.get(), std::move(sdi)); + + entry->init(std::move(accessMethod)); + IndexCatalogEntry* save = entry.get(); if (isReadyIndex) { _readyIndexes.add(std::move(entry)); - } else if (frozen) { - _frozenIndexes.add(std::move(entry)); } else { _buildingIndexes.add(std::move(entry)); } @@ -623,7 +585,7 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont boost::optional<UUID> buildUUID = boost::none; IndexBuildBlock indexBuildBlock( collection->ns(), spec, IndexBuildMethod::kForeground, buildUUID); - status = indexBuildBlock.init(opCtx, collection, /*forRecovery=*/false); + status = indexBuildBlock.init(opCtx, collection); if (!status.isOK()) return status; @@ -911,11 +873,10 @@ Status IndexCatalogImpl::_isSpecOk(OperationContext* opCtx, } const std::unique_ptr<MatchExpression> filterExpr = std::move(statusWithMatcher.getValue()); - Status status = _checkValidFilterExpressions( - filterExpr.get(), - !serverGlobalParams.featureCompatibility.isVersionInitialized() || - feature_flags::gTimeseriesMetricIndexes.isEnabled( - serverGlobalParams.featureCompatibility)); + Status status = + _checkValidFilterExpressions(filterExpr.get(), + feature_flags::gTimeseriesMetricIndexes.isEnabled( + serverGlobalParams.featureCompatibility)); if (!status.isOK()) { return status; } @@ -990,7 +951,7 @@ Status IndexCatalogImpl::_isSpecOk(OperationContext* opCtx, Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx, const CollectionPtr& collection, const BSONObj& spec, - InclusionPolicy inclusionPolicy) const { + const bool includeUnfinishedIndexes) const { StringData name = spec.getStringField(IndexDescriptor::kIndexNameFieldName); invariant(name[0]); @@ -1004,7 +965,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx, { // Check whether an index with the specified candidate name already exists in the catalog. - const IndexDescriptor* desc = findIndexByName(opCtx, name, inclusionPolicy); + const IndexDescriptor* desc = findIndexByName(opCtx, name, includeUnfinishedIndexes); if (desc) { // Index already exists with same name. Check whether the options are the same as well. @@ -1057,7 +1018,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx, { // No index with the candidate name exists. Check for an index with conflicting options. const IndexDescriptor* desc = - findIndexByKeyPatternAndOptions(opCtx, key, spec, inclusionPolicy); + findIndexByKeyPatternAndOptions(opCtx, key, spec, includeUnfinishedIndexes); if (desc) { LOGV2_DEBUG(20353, @@ -1108,7 +1069,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx, string pluginName = IndexNames::findPluginName(key); if (pluginName == IndexNames::TEXT) { vector<const IndexDescriptor*> textIndexes; - findIndexByType(opCtx, IndexNames::TEXT, textIndexes, inclusionPolicy); + findIndexByType(opCtx, IndexNames::TEXT, textIndexes, includeUnfinishedIndexes); if (textIndexes.size() > 0) { return Status(ErrorCodes::CannotCreateIndex, str::stream() << "only one text index per collection allowed, " @@ -1149,10 +1110,7 @@ void IndexCatalogImpl::dropIndexes(OperationContext* opCtx, vector<string> indexNamesToDrop; { int seen = 0; - auto ii = getIndexIterator(opCtx, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, true); while (ii->more()) { seen++; const IndexDescriptor* desc = ii->next()->descriptor(); @@ -1167,11 +1125,7 @@ void IndexCatalogImpl::dropIndexes(OperationContext* opCtx, for (size_t i = 0; i < indexNamesToDrop.size(); i++) { string indexName = indexNamesToDrop[i]; - const IndexDescriptor* desc = findIndexByName( - opCtx, - indexName, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + const IndexDescriptor* desc = findIndexByName(opCtx, indexName, true); invariant(desc); LOGV2_DEBUG(20355, 1, "\t dropAllIndexes dropping: {desc}", "desc"_attr = *desc); IndexCatalogEntry* entry = desc->getEntry(); @@ -1230,73 +1184,6 @@ Status IndexCatalogImpl::dropIndex(OperationContext* opCtx, return dropIndexEntry(opCtx, collection, entry); } -Status IndexCatalogImpl::resetUnfinishedIndexForRecovery(OperationContext* opCtx, - Collection* collection, - const IndexDescriptor* desc) { - invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X)); - invariant(opCtx->lockState()->inAWriteUnitOfWork()); - - IndexCatalogEntry* entry = desc->getEntry(); - const std::string indexName = entry->descriptor()->indexName(); - - // Only indexes that aren't ready can be reset. - invariant(!collection->isIndexReady(indexName)); - - auto released = [&] { - if (auto released = _readyIndexes.release(entry->descriptor())) { - invariant(!released, "Cannot reset a ready index"); - } - if (auto released = _buildingIndexes.release(entry->descriptor())) { - return released; - } - if (auto released = _frozenIndexes.release(entry->descriptor())) { - return released; - } - MONGO_UNREACHABLE; - }(); - - LOGV2(6987700, - "Resetting unfinished index", - logAttrs(collection->ns()), - "index"_attr = indexName, - "ident"_attr = released->getIdent()); - - invariant(released.get() == entry); - - // Drop the ident if it exists. The storage engine will return OK if the ident is not found. - auto engine = opCtx->getServiceContext()->getStorageEngine(); - const std::string ident = released->getIdent(); - Status status = engine->getEngine()->dropIdent(opCtx->recoveryUnit(), ident); - if (!status.isOK()) { - return status; - } - - // Recreate the ident on-disk. DurableCatalog::createIndex() will lookup the ident internally - // using the catalogId and index name. - status = DurableCatalog::get(opCtx)->createIndex(opCtx, - collection->getCatalogId(), - collection->ns(), - collection->getCollectionOptions(), - released->descriptor()); - if (!status.isOK()) { - return status; - } - - // Update the index entry state in preparation to rebuild the index. - if (!released->accessMethod()) { - std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface( - opCtx, collection->ns(), collection->getCollectionOptions(), ident, desc); - std::unique_ptr<IndexAccessMethod> accessMethod = - IndexAccessMethod::make(released.get(), std::move(sdi)); - released->setAccessMethod(std::move(accessMethod)); - } - - released->setIsFrozen(false); - _buildingIndexes.add(std::move(released)); - - return Status::OK(); -} - Status IndexCatalogImpl::dropUnfinishedIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) { @@ -1364,26 +1251,25 @@ Status IndexCatalogImpl::dropIndexEntry(OperationContext* opCtx, audit::logDropIndex(opCtx->getClient(), indexName, collection->ns()); - auto released = [&] { - if (auto released = _readyIndexes.release(entry->descriptor())) { - return released; - } - if (auto released = _buildingIndexes.release(entry->descriptor())) { - return released; - } - if (auto released = _frozenIndexes.release(entry->descriptor())) { - return released; - } - MONGO_UNREACHABLE; - }(); - - invariant(released.get() == entry); - opCtx->recoveryUnit()->registerChange( - std::make_unique<IndexRemoveChange>(opCtx, - collection->ns(), - collection->uuid(), - std::move(released), - collection->getSharedDecorations())); + auto released = _readyIndexes.release(entry->descriptor()); + if (released) { + invariant(released.get() == entry); + opCtx->recoveryUnit()->registerChange( + std::make_unique<IndexRemoveChange>(opCtx, + collection->ns(), + collection->uuid(), + std::move(released), + collection->getSharedDecorations())); + } else { + released = _buildingIndexes.release(entry->descriptor()); + invariant(released.get() == entry); + opCtx->recoveryUnit()->registerChange( + std::make_unique<IndexRemoveChange>(opCtx, + collection->ns(), + collection->uuid(), + std::move(released), + collection->getSharedDecorations())); + } CollectionQueryInfo::get(collection).rebuildIndexData(opCtx, collection); CollectionIndexUsageTrackerDecoration::get(collection->getSharedDecorations()) @@ -1403,11 +1289,7 @@ void IndexCatalogImpl::_deleteIndexFromDisk(OperationContext* opCtx, Collection* collection, const string& indexName, std::shared_ptr<Ident> ident) { - invariant(!findIndexByName(opCtx, - indexName, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen)); + invariant(!findIndexByName(opCtx, indexName, true /* includeUnfinishedIndexes*/)); catalog::removeIndex(opCtx, indexName, collection, std::move(ident)); } @@ -1437,7 +1319,7 @@ int IndexCatalogImpl::numIndexesTotal(OperationContext* opCtx) const { int IndexCatalogImpl::numIndexesReady(OperationContext* opCtx) const { std::vector<const IndexDescriptor*> itIndexes; - auto ii = getIndexIterator(opCtx, InclusionPolicy::kReady); + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, /*includeUnfinished*/ false); while (ii->more()) { itIndexes.push_back(ii->next()->descriptor()); } @@ -1449,7 +1331,7 @@ bool IndexCatalogImpl::haveIdIndex(OperationContext* opCtx) const { } const IndexDescriptor* IndexCatalogImpl::findIdIndex(OperationContext* opCtx) const { - auto ii = getIndexIterator(opCtx, InclusionPolicy::kReady); + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, false); while (ii->more()) { const IndexDescriptor* desc = ii->next()->descriptor(); if (desc->isIdIndex()) @@ -1460,8 +1342,8 @@ const IndexDescriptor* IndexCatalogImpl::findIdIndex(OperationContext* opCtx) co const IndexDescriptor* IndexCatalogImpl::findIndexByName(OperationContext* opCtx, StringData name, - InclusionPolicy inclusionPolicy) const { - auto ii = getIndexIterator(opCtx, inclusionPolicy); + bool includeUnfinishedIndexes) const { + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes); while (ii->more()) { const IndexDescriptor* desc = ii->next()->descriptor(); if (desc->indexName() == name) @@ -1474,8 +1356,8 @@ const IndexDescriptor* IndexCatalogImpl::findIndexByKeyPatternAndOptions( OperationContext* opCtx, const BSONObj& key, const BSONObj& indexSpec, - InclusionPolicy inclusionPolicy) const { - auto ii = getIndexIterator(opCtx, inclusionPolicy); + bool includeUnfinishedIndexes) const { + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes); IndexDescriptor needle(_getAccessMethodName(key), indexSpec); while (ii->more()) { const auto* entry = ii->next(); @@ -1489,10 +1371,10 @@ const IndexDescriptor* IndexCatalogImpl::findIndexByKeyPatternAndOptions( void IndexCatalogImpl::findIndexesByKeyPattern(OperationContext* opCtx, const BSONObj& key, - InclusionPolicy inclusionPolicy, + bool includeUnfinishedIndexes, std::vector<const IndexDescriptor*>* matches) const { invariant(matches); - auto ii = getIndexIterator(opCtx, inclusionPolicy); + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes); while (ii->more()) { const IndexDescriptor* desc = ii->next()->descriptor(); if (SimpleBSONObjComparator::kInstance.evaluate(desc->keyPattern() == key)) { @@ -1504,8 +1386,8 @@ void IndexCatalogImpl::findIndexesByKeyPattern(OperationContext* opCtx, void IndexCatalogImpl::findIndexByType(OperationContext* opCtx, const string& type, vector<const IndexDescriptor*>& matches, - InclusionPolicy inclusionPolicy) const { - auto ii = getIndexIterator(opCtx, inclusionPolicy); + bool includeUnfinishedIndexes) const { + std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes); while (ii->more()) { const IndexDescriptor* desc = ii->next()->descriptor(); if (IndexNames::findPluginName(desc->keyPattern()) == type) { @@ -1746,10 +1628,7 @@ Status IndexCatalogImpl::indexRecords(OperationContext* opCtx, for (const MultikeyPathInfo& newPath : newPaths) { invariant(newPath.nss == coll->ns()); - auto idx = findIndexByName(opCtx, - newPath.indexName, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished); + auto idx = findIndexByName(opCtx, newPath.indexName, /*includeUnfinishedIndexes=*/true); if (!idx) { return Status(ErrorCodes::IndexNotFound, str::stream() @@ -1852,10 +1731,7 @@ Status IndexCatalogImpl::compactIndexes(OperationContext* opCtx) const { } std::string::size_type IndexCatalogImpl::getLongestIndexNameLength(OperationContext* opCtx) const { - auto it = getIndexIterator(opCtx, - IndexCatalog::InclusionPolicy::kReady | - IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + std::unique_ptr<IndexIterator> it = getIndexIterator(opCtx, true); std::string::size_type longestIndexNameLength = 0; while (it->more()) { auto thisLength = it->next()->descriptor()->indexName().length(); @@ -1903,11 +1779,7 @@ void IndexCatalogImpl::indexBuildSuccess(OperationContext* opCtx, invariant(releasedEntry.get() == index); _readyIndexes.add(std::move(releasedEntry)); - // Wait to unset the interceptor until the index actually commits. If a write conflict is - // encountered and the index commit process is restated, the multikey information from the - // interceptor may still be needed. - opCtx->recoveryUnit()->onCommit( - [index](boost::optional<Timestamp>) { index->setIndexBuildInterceptor(nullptr); }); + index->setIndexBuildInterceptor(nullptr); index->setIsReady(true); } diff --git a/src/mongo/db/catalog/index_catalog_impl.h b/src/mongo/db/catalog/index_catalog_impl.h index 223399e5bf0..259732da61c 100644 --- a/src/mongo/db/catalog/index_catalog_impl.h +++ b/src/mongo/db/catalog/index_catalog_impl.h @@ -96,10 +96,9 @@ public: * * @return null if cannot find */ - const IndexDescriptor* findIndexByName( - OperationContext* opCtx, - StringData name, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override; + const IndexDescriptor* findIndexByName(OperationContext* opCtx, + StringData name, + bool includeUnfinishedIndexes = false) const override; /** * Find index by matching key pattern and options. The key pattern, collation spec, and partial @@ -111,7 +110,7 @@ public: OperationContext* opCtx, const BSONObj& key, const BSONObj& indexSpec, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override; + bool includeUnfinishedIndexes = false) const override; /** * Find indexes with a matching key pattern, putting them into the vector 'matches'. The key @@ -121,12 +120,12 @@ public: */ void findIndexesByKeyPattern(OperationContext* opCtx, const BSONObj& key, - InclusionPolicy inclusionPolicy, + bool includeUnfinishedIndexes, std::vector<const IndexDescriptor*>* matches) const override; void findIndexByType(OperationContext* opCtx, const std::string& type, std::vector<const IndexDescriptor*>& matches, - InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override; + bool includeUnfinishedIndexes = false) const override; /** @@ -154,7 +153,7 @@ public: using IndexIterator = IndexCatalog::IndexIterator; std::unique_ptr<IndexIterator> getIndexIterator(OperationContext* opCtx, - InclusionPolicy inclusionPolicy) const override; + bool includeUnfinishedIndexes) const override; // ---- index set modifiers ------ @@ -200,9 +199,6 @@ public: Status dropIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) override; - Status resetUnfinishedIndexForRecovery(OperationContext* opCtx, - Collection* collection, - const IndexDescriptor* desc) override; Status dropUnfinishedIndex(OperationContext* opCtx, Collection* collection, const IndexDescriptor* desc) override; @@ -381,7 +377,7 @@ private: Status _doesSpecConflictWithExisting(OperationContext* opCtx, const CollectionPtr& collection, const BSONObj& spec, - InclusionPolicy inclusionPolicy) const; + bool includeUnfinishedIndexes) const; /** * Returns true if the replica set member's config has {buildIndexes:false} set, which means @@ -397,6 +393,5 @@ private: IndexCatalogEntryContainer _readyIndexes; IndexCatalogEntryContainer _buildingIndexes; - IndexCatalogEntryContainer _frozenIndexes; }; } // namespace mongo diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp index 64b2dfe40ba..55e35b744f8 100644 --- a/src/mongo/db/catalog/index_consistency.cpp +++ b/src/mongo/db/catalog/index_consistency.cpp @@ -38,7 +38,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_repair.h" #include "mongo/db/catalog/validate_gen.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/index/index_descriptor.h" @@ -106,7 +106,7 @@ IndexConsistency::IndexConsistency(OperationContext* opCtx, void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, IndexInfo* indexInfo) { auto hash = _hashKeyString(ks, indexInfo->indexNameHash); - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(6208500, "[validate](multikeyMetadataPath) Adding with the hash", "hash"_attr = hash, @@ -118,7 +118,7 @@ void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, Index void IndexConsistency::removeMultikeyMetadataPath(const KeyString::Value& ks, IndexInfo* indexInfo) { auto hash = _hashKeyString(ks, indexInfo->indexNameHash); - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(6208501, "[validate](multikeyMetadataPath) Removing with the hash", "hash"_attr = hash, @@ -132,26 +132,9 @@ size_t IndexConsistency::getMultikeyMetadataPathCount(IndexInfo* indexInfo) { } bool IndexConsistency::haveEntryMismatch() const { - bool haveMismatch = - std::any_of(_indexKeyBuckets.begin(), - _indexKeyBuckets.end(), - [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; }); - - if (haveMismatch && _validateState->logDiagnostics()) { - for (size_t i = 0; i < _indexKeyBuckets.size(); i++) { - if (_indexKeyBuckets[i].indexKeyCount == 0) { - continue; - } - - LOGV2(7404500, - "[validate](bucket entry mismatch)", - "hash"_attr = i, - "indexKeyCount"_attr = _indexKeyBuckets[i].indexKeyCount, - "bucketBytesSize"_attr = _indexKeyBuckets[i].bucketSizeBytes); - } - } - - return haveMismatch; + return std::any_of(_indexKeyBuckets.begin(), + _indexKeyBuckets.end(), + [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; }); } void IndexConsistency::setSecondPhase() { @@ -209,7 +192,7 @@ void IndexConsistency::repairMissingIndexEntries(OperationContext* opCtx, } } -void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResults* results) { +void IndexConsistency::addIndexEntryErrors(ValidateResults* results) { invariant(!_firstPhase); // We'll report up to 1MB for extra index entry errors and missing index entry errors. @@ -223,26 +206,11 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu numExtraIndexEntryErrors += item.second.size(); } - // Sort missing index entries by size so we can process in order of increasing size and return - // as many as possible within memory limits. - using MissingIt = decltype(_missingIndexEntries)::const_iterator; - std::vector<MissingIt> missingIndexEntriesBySize; - missingIndexEntriesBySize.reserve(_missingIndexEntries.size()); - for (auto it = _missingIndexEntries.begin(); it != _missingIndexEntries.end(); ++it) { - missingIndexEntriesBySize.push_back(it); - } - std::sort(missingIndexEntriesBySize.begin(), - missingIndexEntriesBySize.end(), - [](const MissingIt& a, const MissingIt& b) { - return a->second.keyString.getSize() < b->second.keyString.getSize(); - }); - // Inform which indexes have inconsistencies and add the BSON objects of the inconsistent index // entries to the results vector. bool missingIndexEntrySizeLimitWarning = false; - bool first = true; - for (const auto& missingIndexEntry : missingIndexEntriesBySize) { - const IndexEntryInfo& entryInfo = missingIndexEntry->second; + for (const auto& missingIndexEntry : _missingIndexEntries) { + const IndexEntryInfo& entryInfo = missingIndexEntry.second; KeyString::Value ks = entryInfo.keyString; auto indexKey = KeyString::toBsonSafe(ks.getBuffer(), ks.getSize(), entryInfo.ord, ks.getTypeBits()); @@ -253,9 +221,8 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu entryInfo.idKey); numMissingIndexEntriesSizeBytes += entry.objsize(); - if (first || numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) { + if (numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) { results->missingIndexEntries.push_back(entry); - first = false; } else if (!missingIndexEntrySizeLimitWarning) { StringBuilder ss; ss << "Not all missing index entry inconsistencies are listed due to size limitations."; @@ -264,8 +231,6 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu missingIndexEntrySizeLimitWarning = true; } - _printMetadata(opCtx, results, entryInfo); - std::string indexName = entry["indexName"].String(); if (!results->indexResultsMap.at(indexName).valid) { continue; @@ -278,58 +243,33 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu results->indexResultsMap.at(indexName).valid = false; } - // Sort extra index entries by size so we can process in order of increasing size and return as - // many as possible within memory limits. - using ExtraIt = SimpleBSONObjSet::const_iterator; - std::vector<ExtraIt> extraIndexEntriesBySize; - // Since the extra entries are stored in a map of sets, we have to iterate the entries in the - // map and sum the size of the sets in order to get the total number. Given that we can have at - // most 64 indexes per collection, and the total number of entries could potentially be in the - // millions, we expect that iterating the map will be much less costly than the additional - // allocations and copies that could result from not calling 'reserve' on the vector. - size_t totalExtraIndexEntriesCount = - std::accumulate(_extraIndexEntries.begin(), - _extraIndexEntries.end(), - 0, - [](size_t total, const std::pair<IndexKey, SimpleBSONObjSet>& set) { - return total + set.second.size(); - }); - extraIndexEntriesBySize.reserve(totalExtraIndexEntriesCount); + bool extraIndexEntrySizeLimitWarning = false; for (const auto& extraIndexEntry : _extraIndexEntries) { const SimpleBSONObjSet& entries = extraIndexEntry.second; - for (auto it = entries.begin(); it != entries.end(); ++it) { - extraIndexEntriesBySize.push_back(it); - } - } - std::sort(extraIndexEntriesBySize.begin(), - extraIndexEntriesBySize.end(), - [](const ExtraIt& a, const ExtraIt& b) { return a->objsize() < b->objsize(); }); + for (const auto& entry : entries) { + numExtraIndexEntriesSizeBytes += entry.objsize(); + if (numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) { + results->extraIndexEntries.push_back(entry); + } else if (!extraIndexEntrySizeLimitWarning) { + StringBuilder ss; + ss << "Not all extra index entry inconsistencies are listed due to size " + "limitations."; + results->errors.push_back(ss.str()); + + extraIndexEntrySizeLimitWarning = true; + } + + std::string indexName = entry["indexName"].String(); + if (!results->indexResultsMap.at(indexName).valid) { + continue; + } - bool extraIndexEntrySizeLimitWarning = false; - for (const auto& entry : extraIndexEntriesBySize) { - numExtraIndexEntriesSizeBytes += entry->objsize(); - if (first || numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) { - results->extraIndexEntries.push_back(*entry); - first = false; - } else if (!extraIndexEntrySizeLimitWarning) { StringBuilder ss; - ss << "Not all extra index entry inconsistencies are listed due to size " - "limitations."; + ss << "Index with name '" << indexName << "' has inconsistencies."; results->errors.push_back(ss.str()); - extraIndexEntrySizeLimitWarning = true; + results->indexResultsMap.at(indexName).valid = false; } - - std::string indexName = (*entry)["indexName"].String(); - if (!results->indexResultsMap.at(indexName).valid) { - continue; - } - - StringBuilder ss; - ss << "Index with name '" << indexName << "' has inconsistencies."; - results->errors.push_back(ss.str()); - - results->indexResultsMap.at(indexName).valid = false; } // Inform how many inconsistencies were detected. @@ -362,8 +302,7 @@ void IndexConsistency::addDocumentMultikeyPaths(IndexInfo* indexInfo, void IndexConsistency::addDocKey(OperationContext* opCtx, const KeyString::Value& ks, IndexInfo* indexInfo, - RecordId recordId, - ValidateResults* results) { + RecordId recordId) { auto rawHash = ks.hash(indexInfo->indexNameHash); auto hashLower = rawHash % kNumHashBuckets; auto hashUpper = (rawHash / kNumHashBuckets) % kNumHashBuckets; @@ -379,7 +318,7 @@ void IndexConsistency::addDocKey(OperationContext* opCtx, upper.bucketSizeBytes += ks.getSize(); indexInfo->numRecords++; - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(4666602, "[validate](record) Adding with hashes", "hashUpper"_attr = hashUpper, @@ -409,6 +348,9 @@ void IndexConsistency::addDocKey(OperationContext* opCtx, invariant(_missingIndexEntries.count(key) == 0); _missingIndexEntries.insert( std::make_pair(key, IndexEntryInfo(*indexInfo, recordId, idKeyBuilder.obj(), ks))); + + // Prints the collection document's metadata. + _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx, recordId); } } @@ -432,7 +374,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx, upper.bucketSizeBytes += ks.getSize(); indexInfo->numKeys++; - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(4666603, "[validate](index) Adding with hashes", "hashUpper"_attr = hashUpper, @@ -481,12 +423,9 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx, SimpleBSONObjSet infoSet = {info}; _extraIndexEntries.insert(std::make_pair(key, infoSet)); - // Prints the collection document's and index entry's metadata. - _validateState->getCollection()->getRecordStore()->printRecordMetadata( - opCtx, recordId, &(results->recordTimestamps)); - indexInfo->accessMethod->asSortedData() - ->getSortedDataInterface() - ->printIndexEntryMetadata(opCtx, ks); + // Prints the collection document's metadata. + _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx, + recordId); return; } search->second.insert(info); @@ -499,8 +438,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx, bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) { invariant(!_firstPhase); - const uint64_t maxMemoryUsageBytes = - static_cast<uint64_t>(maxValidateMemoryUsageMB.load()) * 1024 * 1024; + const uint32_t maxMemoryUsageBytes = maxValidateMemoryUsageMB.load() * 1024 * 1024; const uint64_t totalMemoryNeededBytes = std::accumulate(_indexKeyBuckets.begin(), _indexKeyBuckets.end(), @@ -515,57 +453,48 @@ bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) { return true; } - // At this point we know we'll exceed the memory limit, and will pare back some of the buckets. - // First we'll see what the smallest bucket is, and if that's over the limit by itself, then - // we can zero out all the other buckets. Otherwise we'll keep as many buckets as we can. + bool hasNonZeroBucket = false; + uint64_t memoryUsedSoFarBytes = 0; + uint32_t smallestBucketBytes = std::numeric_limits<uint32_t>::max(); + // Zero out any nonzero buckets that would put us over maxMemoryUsageBytes. + std::for_each(_indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) { + if (bucket.indexKeyCount == 0) { + return; + } - auto smallestBucketWithAnInconsistency = std::min_element( - _indexKeyBuckets.begin(), - _indexKeyBuckets.end(), - [](const IndexKeyBucket& lhs, const IndexKeyBucket& rhs) { - if (lhs.indexKeyCount != 0) { - return rhs.indexKeyCount == 0 || lhs.bucketSizeBytes < rhs.bucketSizeBytes; - } - return false; - }); - invariant(smallestBucketWithAnInconsistency->indexKeyCount != 0); - - if (smallestBucketWithAnInconsistency->bucketSizeBytes > maxMemoryUsageBytes) { - // We're going to just keep the smallest bucket, and zero everything else. - std::for_each( - _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) { - if (&bucket == &(*smallestBucketWithAnInconsistency)) { - // We keep the smallest bucket. - return; - } + smallestBucketBytes = std::min(smallestBucketBytes, bucket.bucketSizeBytes); + if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) { + // Including this bucket would put us over the memory limit, so zero this bucket. We + // don't want to keep any entry that will exceed the memory limit in the second phase so + // we don't double the 'maxMemoryUsageBytes' here. + bucket.indexKeyCount = 0; + return; + } + memoryUsedSoFarBytes += bucket.bucketSizeBytes; + hasNonZeroBucket = true; + }); - bucket.indexKeyCount = 0; - }); - } else { - // We're going to scan through the buckets and keep as many as we can. - std::uint32_t memoryUsedSoFarBytes = 0; - std::for_each( - _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) { - if (bucket.indexKeyCount == 0) { - return; - } + StringBuilder memoryLimitMessage; + memoryLimitMessage << "Memory limit for validation is currently set to " + << maxValidateMemoryUsageMB.load() + << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter."; - if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) { - // Including this bucket would put us over the memory limit, so zero this - // bucket. We don't want to keep any entry that will exceed the memory limit in - // the second phase so we don't double the 'maxMemoryUsageBytes' here. - bucket.indexKeyCount = 0; - return; - } - memoryUsedSoFarBytes += bucket.bucketSizeBytes; - }); + if (!hasNonZeroBucket) { + const uint32_t minMemoryNeededMB = (smallestBucketBytes / (1024 * 1024)) + 1; + StringBuilder ss; + ss << "Unable to report index entry inconsistencies due to memory limitations. Need at " + "least " + << minMemoryNeededMB << "MB to report at least one index entry inconsistency. " + << memoryLimitMessage.str(); + result->errors.push_back(ss.str()); + result->valid = false; + + return false; } StringBuilder ss; - ss << "Not all index entry inconsistencies are reported due to memory limitations. Memory " - "limit for validation is currently set to " - << maxValidateMemoryUsageMB.load() - << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter."; + ss << "Not all index entry inconsistencies are reported due to memory limitations. " + << memoryLimitMessage.str(); result->errors.push_back(ss.str()); result->valid = false; @@ -611,16 +540,4 @@ uint32_t IndexConsistency::_hashKeyString(const KeyString::Value& ks, uint32_t indexNameHash) const { return ks.hash(indexNameHash); } - -void IndexConsistency::_printMetadata(OperationContext* opCtx, - ValidateResults* results, - const IndexEntryInfo& entryInfo) { - _validateState->getCollection()->getRecordStore()->printRecordMetadata( - opCtx, entryInfo.recordId, &(results->recordTimestamps)); - getIndexInfo(entryInfo.indexName) - .accessMethod->asSortedData() - ->getSortedDataInterface() - ->printIndexEntryMetadata(opCtx, entryInfo.keyString); -} - } // namespace mongo diff --git a/src/mongo/db/catalog/index_consistency.h b/src/mongo/db/catalog/index_consistency.h index f4eab27660b..dab1d2d8d97 100644 --- a/src/mongo/db/catalog/index_consistency.h +++ b/src/mongo/db/catalog/index_consistency.h @@ -108,8 +108,7 @@ public: void addDocKey(OperationContext* opCtx, const KeyString::Value& ks, IndexInfo* indexInfo, - RecordId recordId, - ValidateResults* results); + RecordId recordId); /** * During the first phase of validation, given the index entry's KeyString, decrement the @@ -169,7 +168,7 @@ public: * Records the errors gathered from the second phase of index validation into the provided * ValidateResultsMap and ValidateResults. */ - void addIndexEntryErrors(OperationContext* opCtx, ValidateResults* results); + void addIndexEntryErrors(ValidateResults* results); /** * Sets up this IndexConsistency object to limit memory usage in the second phase of index @@ -180,8 +179,8 @@ public: private: struct IndexKeyBucket { - uint32_t indexKeyCount = 0; - uint32_t bucketSizeBytes = 0; + uint32_t indexKeyCount; + uint32_t bucketSizeBytes; }; IndexConsistency() = delete; @@ -242,11 +241,5 @@ private: */ uint32_t _hashKeyString(const KeyString::Value& ks, uint32_t indexNameHash) const; - /** - * Prints the collection document's and index entry's metadata. - */ - void _printMetadata(OperationContext* opCtx, - ValidateResults* results, - const IndexEntryInfo& info); }; // IndexConsistency } // namespace mongo diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp index 20dd9f37fc0..d6f70219594 100644 --- a/src/mongo/db/catalog/index_key_validate.cpp +++ b/src/mongo/db/catalog/index_key_validate.cpp @@ -58,7 +58,7 @@ namespace mongo { namespace index_key_validate { -std::function<void(std::map<StringData, std::set<IndexType>>&)> filterAllowedIndexFieldNames; +std::function<void(std::set<StringData>&)> filterAllowedIndexFieldNames; using IndexVersion = IndexDescriptor::IndexVersion; @@ -68,10 +68,6 @@ namespace { // specification. MONGO_FAIL_POINT_DEFINE(skipIndexCreateFieldNameValidation); -// When the skipTTLIndexNaNExpireAfterSecondsValidation failpoint is enabled, validation for -// TTL index 'expireAfterSeconds' will be disabled. -MONGO_FAIL_POINT_DEFINE(skipTTLIndexNaNExpireAfterSecondsValidation); - static const std::set<StringData> allowedIdIndexFieldNames = { IndexDescriptor::kCollationFieldName, IndexDescriptor::kIndexNameFieldName, @@ -108,16 +104,12 @@ Status isIndexVersionAllowedForCreation(IndexVersion indexVersion, const BSONObj BSONObj buildRepairedIndexSpec( const NamespaceString& ns, const BSONObj& indexSpec, - const std::map<StringData, std::set<IndexType>>& allowedFieldNames, + const std::set<StringData>& allowedFieldNames, std::function<void(const BSONElement&, BSONObjBuilder*)> indexSpecHandleFn) { - const auto key = indexSpec.getObjectField(IndexDescriptor::kKeyPatternFieldName); - const auto indexName = IndexNames::nameToType(IndexNames::findPluginName(key)); BSONObjBuilder builder; for (const auto& indexSpecElem : indexSpec) { StringData fieldName = indexSpecElem.fieldNameStringData(); - auto it = allowedFieldNames.find(fieldName); - if (it != allowedFieldNames.end() && - (it->second.empty() || it->second.count(indexName) != 0)) { + if (allowedFieldNames.count(fieldName)) { indexSpecHandleFn(indexSpecElem, &builder); } else { LOGV2_WARNING(23878, @@ -271,9 +263,9 @@ BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec) BSONObj repairIndexSpec(const NamespaceString& ns, const BSONObj& indexSpec, - const std::map<StringData, std::set<IndexType>>& allowedFieldNames) { - auto fixIndexSpecFn = [&indexSpec, &ns](const BSONElement& indexSpecElem, - BSONObjBuilder* builder) { + const std::set<StringData>& allowedFieldNames) { + auto fixBoolIndexSpecFn = [&indexSpec, &ns](const BSONElement& indexSpecElem, + BSONObjBuilder* builder) { StringData fieldName = indexSpecElem.fieldNameStringData(); if ((IndexDescriptor::kBackgroundFieldName == fieldName || IndexDescriptor::kUniqueFieldName == fieldName || @@ -287,28 +279,17 @@ BSONObj repairIndexSpec(const NamespaceString& ns, "fieldName"_attr = redact(fieldName), "indexSpec"_attr = redact(indexSpec)); builder->appendBool(fieldName, true); - } else if (IndexDescriptor::kExpireAfterSecondsFieldName == fieldName && - !(indexSpecElem.isNumber() && !indexSpecElem.isNaN())) { - LOGV2_WARNING(6835900, - "Fixing expire field from TTL index spec", - "namespace"_attr = redact(ns.toString()), - "fieldName"_attr = redact(fieldName), - "indexSpec"_attr = redact(indexSpec)); - builder->appendNumber(fieldName, - durationCount<Seconds>(kExpireAfterSecondsForInactiveTTLIndex)); } else { builder->append(indexSpecElem); } }; - - return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, fixIndexSpecFn); + return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, fixBoolIndexSpecFn); } StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& indexSpec) { bool hasKeyPatternField = false; bool hasIndexNameField = false; bool hasNamespaceField = false; - bool isTTLIndexWithNaNExpireAfterSeconds = false; bool hasVersionField = false; bool hasCollationField = false; bool hasWeightsField = false; @@ -497,9 +478,6 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in << "' is only allowed when '" << IndexDescriptor::kKeyPatternFieldName << "' is {\"$**\": ±1}"}; } - if (key.nFields() != 1) { - return {ErrorCodes::CannotCreateIndex, "wildcard indexes do not allow compounding"}; - } if (indexSpecElem.embeddedObject().isEmpty()) { return {ErrorCodes::FailedToParse, @@ -553,15 +531,12 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in IndexDescriptor::kTextVersionFieldName == indexSpecElemFieldName || IndexDescriptor::k2dIndexBitsFieldName == indexSpecElemFieldName || IndexDescriptor::k2dIndexMinFieldName == indexSpecElemFieldName || - IndexDescriptor::k2dIndexMaxFieldName == indexSpecElemFieldName || - IndexDescriptor::kBucketSizeFieldName == indexSpecElemFieldName) && + IndexDescriptor::k2dIndexMaxFieldName == indexSpecElemFieldName) && !indexSpecElem.isNumber()) { return {ErrorCodes::TypeMismatch, str::stream() << "The field '" << indexSpecElemFieldName << "' must be a number, but got " << typeName(indexSpecElem.type())}; - } else if (IndexDescriptor::kExpireAfterSecondsFieldName == indexSpecElemFieldName) { - isTTLIndexWithNaNExpireAfterSeconds = indexSpecElem.isNaN(); } else { // We can assume field name is valid at this point. Validation of fieldname is handled // prior to this in validateIndexSpecFieldNames(). @@ -633,19 +608,6 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in modifiedSpec = modifiedSpec.removeField(IndexDescriptor::kNamespaceFieldName); } - if (isTTLIndexWithNaNExpireAfterSeconds && - !skipTTLIndexNaNExpireAfterSecondsValidation.shouldFail()) { - // We create a new index specification with the 'expireAfterSeconds' field set as - // kExpireAfterSecondsForInactiveTTLIndex if the current value is NaN. A similar - // treatment is done in repairIndexSpec(). This rewrites the 'expireAfterSeconds' - // value to be compliant with the 'safeInt' IDL type for the listIndexes response. - BSONObjBuilder builder; - builder.appendNumber(IndexDescriptor::kExpireAfterSecondsFieldName, - durationCount<Seconds>(kExpireAfterSecondsForInactiveTTLIndex)); - auto obj = builder.obj(); - modifiedSpec = modifiedSpec.addField(obj.firstElement()); - } - if (!hasVersionField) { // We create a new index specification with the 'v' field set as 'defaultIndexVersion' if // the field was omitted. @@ -806,8 +768,7 @@ StatusWith<BSONObj> validateIndexSpecCollation(OperationContext* opCtx, return indexSpec; } -Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds, - ValidateExpireAfterSecondsMode mode) { +Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds) { if (expireAfterSeconds < 0) { return {ErrorCodes::InvalidOptions, str::stream() << "TTL index '" << IndexDescriptor::kExpireAfterSecondsFieldName @@ -818,31 +779,16 @@ Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds, << "TTL index '" << IndexDescriptor::kExpireAfterSecondsFieldName << "' option must be within an acceptable range, try a lower number"; - if (mode == ValidateExpireAfterSecondsMode::kSecondaryTTLIndex) { - // Relax epoch restriction on TTL indexes. This allows us to export and import existing - // TTL indexes with large values or NaN for the 'expireAfterSeconds' field. - // Additionally, the 'expireAfterSeconds' for TTL indexes is defined as safeInt (int32_t) - // in the IDL for listIndexes and collMod. See list_indexes.idl and coll_mod.idl. - if (expireAfterSeconds > std::numeric_limits<std::int32_t>::max()) { - return {ErrorCodes::InvalidOptions, tooLargeErr}; - } - } else { - // Clustered collections with TTL. - // Note that 'expireAfterSeconds' is defined as safeInt64 in the IDL for the create and - // collMod commands. See create.idl and coll_mod.idl. - // There are two cases where we can encounter an issue here. - // The first case is when we try to cast to millseconds from seconds, which could cause an - // overflow. The second case is where 'expireAfterSeconds' is larger than the current epoch - // time. This isn't necessarily problematic for the general case, but for the specific case - // of time series collections, we cluster the collection by an OID value, where the - // timestamp portion is only a 32-bit unsigned integer offset of seconds since the epoch. - if (expireAfterSeconds > std::numeric_limits<std::int64_t>::max() / 1000) { - return {ErrorCodes::InvalidOptions, tooLargeErr}; - } - auto expireAfterMillis = duration_cast<Milliseconds>(Seconds(expireAfterSeconds)); - if (expireAfterMillis > Date_t::now().toDurationSinceEpoch()) { - return {ErrorCodes::InvalidOptions, tooLargeErr}; - } + // There are two cases where we can encounter an issue here. + // The first case is when we try to cast to millseconds from seconds, which could cause an + // overflow. The second case is where 'expireAfterSeconds' is larger than the current epoch + // time. + if (expireAfterSeconds > std::numeric_limits<std::int64_t>::max() / 1000) { + return {ErrorCodes::InvalidOptions, tooLargeErr}; + } + auto expireAfterMillis = duration_cast<Milliseconds>(Seconds(expireAfterSeconds)); + if (expireAfterMillis > Date_t::now().toDurationSinceEpoch()) { + return {ErrorCodes::InvalidOptions, tooLargeErr}; } return Status::OK(); } @@ -866,9 +812,7 @@ Status validateIndexSpecTTL(const BSONObj& indexSpec) { << "'. Index spec: " << indexSpec}; } - if (auto status = - validateExpireAfterSeconds(expireAfterSecondsElt.safeNumberLong(), - ValidateExpireAfterSecondsMode::kSecondaryTTLIndex); + if (auto status = validateExpireAfterSeconds(expireAfterSecondsElt.safeNumberLong()); !status.isOK()) { return {ErrorCodes::CannotCreateIndex, str::stream() << status.reason() << ". Index spec: " << indexSpec}; diff --git a/src/mongo/db/catalog/index_key_validate.h b/src/mongo/db/catalog/index_key_validate.h index 45b383fee1c..8f226749c21 100644 --- a/src/mongo/db/catalog/index_key_validate.h +++ b/src/mongo/db/catalog/index_key_validate.h @@ -43,51 +43,35 @@ class StatusWith; namespace index_key_validate { -// TTL indexes with 'expireAfterSeconds' are repaired with this duration, which is chosen to be -// the largest possible value for the 'safeInt' type that can be returned in the listIndexes -// response. -constexpr auto kExpireAfterSecondsForInactiveTTLIndex = - Seconds(std::numeric_limits<int32_t>::max()); - -/** - * Describe which field names are considered valid options when creating an index. If the set - * associated with the field name is empty, the option is always valid, otherwise it will be allowed - * only when creating the set of index types listed in the set. - */ -static std::map<StringData, std::set<IndexType>> allowedFieldNames = { - {IndexDescriptor::k2dIndexBitsFieldName, {IndexType::INDEX_2D}}, - {IndexDescriptor::k2dIndexMaxFieldName, {IndexType::INDEX_2D}}, - {IndexDescriptor::k2dIndexMinFieldName, {IndexType::INDEX_2D}}, - {IndexDescriptor::k2dsphereCoarsestIndexedLevel, {IndexType::INDEX_2DSPHERE}}, - {IndexDescriptor::k2dsphereFinestIndexedLevel, {IndexType::INDEX_2DSPHERE}}, - {IndexDescriptor::k2dsphereVersionFieldName, - {IndexType::INDEX_2DSPHERE, IndexType::INDEX_2DSPHERE_BUCKET}}, - {IndexDescriptor::kBackgroundFieldName, {}}, - {IndexDescriptor::kCollationFieldName, {}}, - {IndexDescriptor::kDefaultLanguageFieldName, {}}, - {IndexDescriptor::kDropDuplicatesFieldName, {}}, - {IndexDescriptor::kExpireAfterSecondsFieldName, {}}, - {IndexDescriptor::kHiddenFieldName, {}}, - {IndexDescriptor::kIndexNameFieldName, {}}, - {IndexDescriptor::kIndexVersionFieldName, {}}, - {IndexDescriptor::kKeyPatternFieldName, {}}, - {IndexDescriptor::kLanguageOverrideFieldName, {}}, - {IndexDescriptor::kNamespaceFieldName, {}}, - {IndexDescriptor::kPartialFilterExprFieldName, {}}, - {IndexDescriptor::kPathProjectionFieldName, {IndexType::INDEX_WILDCARD}}, - {IndexDescriptor::kSparseFieldName, {}}, - {IndexDescriptor::kStorageEngineFieldName, {}}, - {IndexDescriptor::kTextVersionFieldName, {IndexType::INDEX_TEXT}}, - {IndexDescriptor::kUniqueFieldName, {}}, - {IndexDescriptor::kWeightsFieldName, {IndexType::INDEX_TEXT}}, - {IndexDescriptor::kOriginalSpecFieldName, {}}, - {IndexDescriptor::kPrepareUniqueFieldName, {}}, +static std::set<StringData> allowedFieldNames = { + IndexDescriptor::k2dIndexBitsFieldName, + IndexDescriptor::k2dIndexMaxFieldName, + IndexDescriptor::k2dIndexMinFieldName, + IndexDescriptor::k2dsphereCoarsestIndexedLevel, + IndexDescriptor::k2dsphereFinestIndexedLevel, + IndexDescriptor::k2dsphereVersionFieldName, + IndexDescriptor::kBackgroundFieldName, + IndexDescriptor::kCollationFieldName, + IndexDescriptor::kDefaultLanguageFieldName, + IndexDescriptor::kDropDuplicatesFieldName, + IndexDescriptor::kExpireAfterSecondsFieldName, + IndexDescriptor::kHiddenFieldName, + IndexDescriptor::kIndexNameFieldName, + IndexDescriptor::kIndexVersionFieldName, + IndexDescriptor::kKeyPatternFieldName, + IndexDescriptor::kLanguageOverrideFieldName, + IndexDescriptor::kNamespaceFieldName, + IndexDescriptor::kPartialFilterExprFieldName, + IndexDescriptor::kPathProjectionFieldName, + IndexDescriptor::kSparseFieldName, + IndexDescriptor::kStorageEngineFieldName, + IndexDescriptor::kTextVersionFieldName, + IndexDescriptor::kUniqueFieldName, + IndexDescriptor::kWeightsFieldName, + IndexDescriptor::kOriginalSpecFieldName, + IndexDescriptor::kPrepareUniqueFieldName, // Index creation under legacy writeMode can result in an index spec with an _id field. - {"_id", {}}, - // TODO SERVER-76108: Field names are not validated to match index type. This was used for the - // removed 'geoHaystack' index type, but users could have set it for other index types as well. - // We need to keep allowing it until FCV upgrade is implemented to clean this up. - {"bucketSize"_sd, {}}}; + "_id"}; /** * Checks if the key is valid for building an index according to the validation rules for the given @@ -108,12 +92,12 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec); /** - * Returns a new index spec with boolean values in correct types and unknown field names removed. + * Returns a new index spec with boolean values in correct types and unkown field names removed. */ -BSONObj repairIndexSpec(const NamespaceString& ns, - const BSONObj& indexSpec, - const std::map<StringData, std::set<IndexType>>& allowedFieldNames = - index_key_validate::allowedFieldNames); +BSONObj repairIndexSpec( + const NamespaceString& ns, + const BSONObj& indexSpec, + const std::set<StringData>& allowedFieldNames = index_key_validate::allowedFieldNames); /** * Performs additional validation for _id index specifications. This should be called after @@ -137,14 +121,9 @@ StatusWith<BSONObj> validateIndexSpecCollation(OperationContext* opCtx, const CollatorInterface* defaultCollator); /** - * Validates the the 'expireAfterSeconds' value for a TTL index or clustered collection. + * Validates the the 'expireAfterSeconds' value for a TTL index.. */ -enum class ValidateExpireAfterSecondsMode { - kSecondaryTTLIndex, - kClusteredTTLIndex, -}; -Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds, - ValidateExpireAfterSecondsMode mode); +Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds); /** * Returns true if 'indexSpec' refers to a TTL index. @@ -166,7 +145,7 @@ bool isIndexAllowedInAPIVersion1(const IndexDescriptor& indexDesc); * Optional filtering function to adjust allowed index field names at startup. * Set it in a MONGO_INITIALIZER with 'FilterAllowedIndexFieldNames' as a dependant. */ -extern std::function<void(std::map<StringData, std::set<IndexType>>& allowedIndexFieldNames)> +extern std::function<void(std::set<StringData>& allowedIndexFieldNames)> filterAllowedIndexFieldNames; } // namespace index_key_validate diff --git a/src/mongo/db/catalog/index_key_validate_test.cpp b/src/mongo/db/catalog/index_key_validate_test.cpp index 659a0bfb7b2..2c25d1b8791 100644 --- a/src/mongo/db/catalog/index_key_validate_test.cpp +++ b/src/mongo/db/catalog/index_key_validate_test.cpp @@ -318,100 +318,5 @@ TEST(IndexKeyValidateTest, Background) { nullptr, fromjson("{key: {a: 1}, name: 'index', background: []}"))); } -TEST(IndexKeyValidateTest, RemoveUnkownFieldsFromIndexSpecs) { - ASSERT(fromjson("{key: {a: 1}, name: 'index'}") - .binaryEqual(index_key_validate::removeUnknownFields( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', safe: true, force: true}")))); -} - -TEST(IndexKeyValidateTest, UpdateTTLIndexNaNExpireAfterSeconds) { - ASSERT(BSON("key" << BSON("a" << 1) << "name" - << "index" - << "expireAfterSeconds" << std::numeric_limits<int32_t>::max() - << IndexDescriptor::kIndexVersionFieldName << IndexVersion::kV2) - .binaryEqual(unittest::assertGet(index_key_validate::validateIndexSpec( - nullptr, fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: NaN}"))))); -} - -TEST(IndexKeyValidateTest, ValidateAfterSecondsAcceptsFloatingPointNumber) { - auto spec = unittest::assertGet(index_key_validate::validateIndexSpec( - nullptr, fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: 123.456}"))); - - // TTLMonitor extracts 'expireAfterSeconds' using BSONElement::safeNumberLong(). - ASSERT_EQUALS(spec["expireAfterSeconds"].safeNumberLong(), 123LL); -} - -TEST(IndexKeyValidateTest, RepairIndexSpecs) { - ASSERT(fromjson("{key: {a: 1}, name: 'index'}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', safe: true, force: true}")))); - - ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', sparse: 'true'}")))); - - ASSERT(fromjson("{key: {a: 1}, name: 'index', background: true}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', background: '1'}")))); - - ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true, background: true}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', sparse: 'true', background: '1'}")))); - - ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true, background: true}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', sparse: 'true', background: '1', safe: " - "true, force: true}")))); - - ASSERT(fromjson("{key: {a: 1}, name: 'index'}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', weights: {key: 1, name: 1}}")))); - - ASSERT(fromjson("{key: {'a': 'text'}, name: 'index', weights: {key: 1, name: 1}}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {'a': 'text'}, name: 'index', weights: {key: 1, name: 1}}")))); - - ASSERT(fromjson("{key: {'$**': 'text'}, name: 'index', weights: {key: 1, name: 1}}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {'$**': 'text'}, name: 'index', weights: {key: 1, name: 1}}")))); - - ASSERT(fromjson("{key: {'data.loc' : '2dsphere_bucket'}, 'name': 'loc_2dsphere', " - "'2dsphereIndexVersion' : 3}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {'data.loc' : '2dsphere_bucket'}, 'name': 'loc_2dsphere', " - "'2dsphereIndexVersion' : 3}")))); - - ASSERT( - fromjson("{key: {a: 1, 'name': 'text'}, name: 'index', weights: {key: 1, name: 1}}") - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson( - "{key: {a: 1, 'name': 'text'}, name: 'index', weights: {key: 1, name: 1}}")))); - - ASSERT(BSON("key" << BSON("a" << 1) << "name" - << "index" - << "expireAfterSeconds" << std::numeric_limits<int32_t>::max()) - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: NaN}")))); - - ASSERT(BSON("key" << BSON("a" << 1) << "name" - << "index" - << "expireAfterSeconds" << std::numeric_limits<int32_t>::max()) - .binaryEqual(index_key_validate::repairIndexSpec( - NamespaceString("coll"), - fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: '123'}")))); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/catalog/index_repair.cpp b/src/mongo/db/catalog/index_repair.cpp index 4e863cec74c..6effefcd909 100644 --- a/src/mongo/db/catalog/index_repair.cpp +++ b/src/mongo/db/catalog/index_repair.cpp @@ -31,7 +31,7 @@ #include "mongo/base/status_with.h" #include "mongo/db/catalog/validate_state.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_access_method.h" #include "mongo/logv2/log_debug.h" diff --git a/src/mongo/db/catalog/list_indexes.cpp b/src/mongo/db/catalog/list_indexes.cpp index 660c4068f83..ce1515a4c13 100644 --- a/src/mongo/db/catalog/list_indexes.cpp +++ b/src/mongo/db/catalog/list_indexes.cpp @@ -38,7 +38,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/catalog/clustered_collection_util.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop_failpoint_helpers.h" #include "mongo/db/db_raii.h" #include "mongo/db/namespace_string.h" diff --git a/src/mongo/db/catalog/local_oplog_info.cpp b/src/mongo/db/catalog/local_oplog_info.cpp index d48a5277288..d7e51eb2892 100644 --- a/src/mongo/db/catalog/local_oplog_info.cpp +++ b/src/mongo/db/catalog/local_oplog_info.cpp @@ -38,7 +38,6 @@ #include "mongo/db/storage/recovery_unit.h" #include "mongo/db/vector_clock_mutable.h" #include "mongo/util/assert_util.h" -#include "mongo/util/timer.h" namespace mongo { namespace { @@ -112,8 +111,6 @@ std::vector<OplogSlot> LocalOplogInfo::getNextOpTimes(OperationContext* opCtx, s invariant(_oplog); fassert(28560, _oplog->getRecordStore()->oplogDiskLocRegister(opCtx, ts, orderedCommit)); } - - Timer oplogSlotDurationTimer; std::vector<OplogSlot> oplogSlots(count); for (std::size_t i = 0; i < count; i++) { oplogSlots[i] = {Timestamp(ts.asULL() + i), term}; @@ -122,25 +119,8 @@ std::vector<OplogSlot> LocalOplogInfo::getNextOpTimes(OperationContext* opCtx, s // If we abort a transaction that has reserved an optime, we should make sure to update the // stable timestamp if necessary, since this oplog hole may have been holding back the stable // timestamp. - opCtx->recoveryUnit()->onRollback([replCoord, oplogSlotDurationTimer]() { - replCoord->attemptToAdvanceStableTimestamp(); - - // Transactions can commit on a different thread with a different client and opCtx. - if (auto opCtx = cc().getOperationContext()) { - // Sum the oplog slot durations. An operation may participate in multiple transactions. - CurOp::get(opCtx)->debug().totalOplogSlotDurationMicros += - Microseconds(oplogSlotDurationTimer.elapsed()); - } - }); - - opCtx->recoveryUnit()->onCommit([oplogSlotDurationTimer](boost::optional<Timestamp>) { - // Transactions can commit on a different thread with a different Client and opCtx. - if (auto opCtx = cc().getOperationContext()) { - // Sum the oplog slot durations. An operation may participate in multiple transactions. - CurOp::get(opCtx)->debug().totalOplogSlotDurationMicros += - Microseconds(oplogSlotDurationTimer.elapsed()); - } - }); + opCtx->recoveryUnit()->onRollback( + [replCoord]() { replCoord->attemptToAdvanceStableTimestamp(); }); return oplogSlots; } diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp index a0e8382959e..324f6e489ae 100644 --- a/src/mongo/db/catalog/multi_index_block.cpp +++ b/src/mongo/db/catalog/multi_index_block.cpp @@ -41,7 +41,7 @@ #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/multi_index_block_gen.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/multikey_paths.h" #include "mongo/db/multi_key_path_tracker.h" #include "mongo/db/op_observer.h" @@ -185,7 +185,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(OperationContext* opCtx, const BSONObj& spec, OnInitFn onInit) { const auto indexes = std::vector<BSONObj>(1, spec); - return init(opCtx, collection, indexes, onInit, /*forRecovery=*/false, boost::none); + return init(opCtx, collection, indexes, onInit, boost::none); } StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( @@ -193,7 +193,6 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( CollectionWriter& collection, const std::vector<BSONObj>& indexSpecs, OnInitFn onInit, - bool forRecovery, const boost::optional<ResumeIndexInfo>& resumeInfo) { invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X), str::stream() << "Collection " << collection->ns() << " with UUID " @@ -247,31 +246,27 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( for (size_t i = 0; i < indexSpecs.size(); i++) { BSONObj info = indexSpecs[i]; - if (!forRecovery) { - // We skip this step when initializing unfinished index builds during startup - // recovery as they are already in the index catalog. - StatusWith<BSONObj> statusWithInfo = - collection->getIndexCatalog()->prepareSpecForCreate( - opCtx, collection.get(), info, resumeInfo); - Status status = statusWithInfo.getStatus(); - if (!status.isOK()) { - // If we were given two identical indexes to build, we will run into an error - // trying to set up the same index a second time in this for-loop. This is the - // only way to encounter this error because callers filter out ready/in-progress - // indexes and start the build while holding a lock throughout. - if (status == ErrorCodes::IndexBuildAlreadyInProgress) { - invariant(indexSpecs.size() > 1, - str::stream() << "Collection: " << collection->ns() << " (" - << _collectionUUID - << "), Index spec: " << indexSpecs.front()); - return {ErrorCodes::OperationFailed, - "Cannot build two identical indexes. Try again without duplicate " - "indexes."}; - } - return status; + StatusWith<BSONObj> statusWithInfo = + collection->getIndexCatalog()->prepareSpecForCreate( + opCtx, collection.get(), info, resumeInfo); + Status status = statusWithInfo.getStatus(); + if (!status.isOK()) { + // If we were given two identical indexes to build, we will run into an error trying + // to set up the same index a second time in this for-loop. This is the only way to + // encounter this error because callers filter out ready/in-progress indexes and + // start the build while holding a lock throughout. + if (status == ErrorCodes::IndexBuildAlreadyInProgress) { + invariant(indexSpecs.size() > 1, + str::stream() + << "Collection: " << collection->ns() << " (" << _collectionUUID + << "), Index spec: " << indexSpecs.front()); + return { + ErrorCodes::OperationFailed, + "Cannot build two identical indexes. Try again without duplicate indexes."}; } - info = statusWithInfo.getValue(); + return status; } + info = statusWithInfo.getValue(); indexInfoObjs.push_back(info); boost::optional<TimeseriesOptions> options = collection->getTimeseriesOptions(); @@ -307,7 +302,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init( status = index.block->initForResume( opCtx, collection.getWritableCollection(), *stateInfo, resumeInfo->getPhase()); } else { - status = index.block->init(opCtx, collection.getWritableCollection(), forRecovery); + status = index.block->init(opCtx, collection.getWritableCollection()); } if (!status.isOK()) return status; @@ -443,7 +438,7 @@ Status MultiIndexBlock::insertAllDocumentsInCollection( // Unlock before hanging so replication recognizes we've completed. collection.yield(); Locker::LockSnapshot lockInfo; - opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); + invariant(opCtx->lockState()->saveLockStateAndUnlock(&lockInfo)); LOGV2(4585201, "Hanging index build with no locks due to " @@ -563,7 +558,7 @@ Status MultiIndexBlock::insertAllDocumentsInCollection( // Unlock before hanging so replication recognizes we've completed. collection.yield(); Locker::LockSnapshot lockInfo; - opCtx->lockState()->saveLockStateAndUnlock(&lockInfo); + invariant(opCtx->lockState()->saveLockStateAndUnlock(&lockInfo)); LOGV2(20390, "Hanging index build with no locks due to " @@ -690,10 +685,10 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx, // collection to have it. if (_containsIndexBuildOnTimeseriesMeasurement && *collection->getTimeseriesBucketsMayHaveMixedSchemaData()) { - auto docHasMixedSchemaData = + bool docHasMixedSchemaData = collection->doesTimeseriesBucketsDocContainMixedSchemaData(doc); - if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue()) { + if (docHasMixedSchemaData) { LOGV2(6057700, "Detected mixed-schema data in time-series bucket collection", logAttrs(collection->ns()), @@ -709,8 +704,7 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx, auto replCoord = repl::ReplicationCoordinator::get(opCtx); const bool replSetAndNotPrimary = !replCoord->canAcceptWritesFor(opCtx, collection->ns()); - if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue() && - !replSetAndNotPrimary) { + if (docHasMixedSchemaData && !replSetAndNotPrimary) { return timeseriesMixedSchemaDataFailure(collection.get()); } } @@ -727,6 +721,7 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx, try { idxStatus = _indexes[i].bulk->insert(opCtx, collection, + _indexes[i].block->getPooledBuilder(), doc, loc, _indexes[i].options, @@ -978,21 +973,15 @@ Status MultiIndexBlock::commit(OperationContext* opCtx, onCommit(); - // We can't update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag here as it - // requires the change to be driven by the router role. It means that subsequent index builds - // and other systems needs to treat this collection as-if it contains mixed-schema data even if - // it might not. We log a warning that can be used to initiate changing the flag. Note: just - // because this node doesn't contain mixed-schema it doesn't mean that other shards can't have - // mixed schema data. This flag needs to be consistent across the shards. + // Update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag to false in order to + // allow subsequent index builds to skip checking bucket documents for mixed-schema data. if (_containsIndexBuildOnTimeseriesMeasurement && !_timeseriesBucketContainsMixedSchemaData) { boost::optional<bool> mayContainMixedSchemaData = collection->getTimeseriesBucketsMayHaveMixedSchemaData(); invariant(mayContainMixedSchemaData); if (*mayContainMixedSchemaData) { - LOGV2_WARNING(9301400, - "Index build finished for time-series collection marked as containing " - "mixed schema buckets without detecting any buckets with mixed schema."); + collection->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, false); } } diff --git a/src/mongo/db/catalog/multi_index_block.h b/src/mongo/db/catalog/multi_index_block.h index 5220f8dd6f0..840770595cf 100644 --- a/src/mongo/db/catalog/multi_index_block.h +++ b/src/mongo/db/catalog/multi_index_block.h @@ -114,7 +114,6 @@ public: CollectionWriter& collection, const std::vector<BSONObj>& specs, OnInitFn onInit, - bool forRecovery, const boost::optional<ResumeIndexInfo>& resumeInfo = boost::none); StatusWith<std::vector<BSONObj>> init(OperationContext* opCtx, CollectionWriter& collection, diff --git a/src/mongo/db/catalog/multi_index_block_test.cpp b/src/mongo/db/catalog/multi_index_block_test.cpp index e9153ba35dc..2fb9caf7371 100644 --- a/src/mongo/db/catalog/multi_index_block_test.cpp +++ b/src/mongo/db/catalog/multi_index_block_test.cpp @@ -90,11 +90,8 @@ TEST_F(MultiIndexBlockTest, CommitWithoutInsertingDocuments) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init(operationContext(), - coll, - std::vector<BSONObj>(), - MultiIndexBlock::kNoopOnInitFn, - /*forRecovery=*/false)); + auto specs = unittest::assertGet(indexer->init( + operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK(indexer->dumpInsertsFromBulk(operationContext(), coll.get())); @@ -116,11 +113,8 @@ TEST_F(MultiIndexBlockTest, CommitAfterInsertingSingleDocument) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init(operationContext(), - coll, - std::vector<BSONObj>(), - MultiIndexBlock::kNoopOnInitFn, - /*forRecovery=*/false)); + auto specs = unittest::assertGet(indexer->init( + operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK( @@ -152,11 +146,8 @@ TEST_F(MultiIndexBlockTest, AbortWithoutCleanupAfterInsertingSingleDocument) { AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X); CollectionWriter coll(operationContext(), autoColl); - auto specs = unittest::assertGet(indexer->init(operationContext(), - coll, - std::vector<BSONObj>(), - MultiIndexBlock::kNoopOnInitFn, - /*forRecovery=*/false)); + auto specs = unittest::assertGet(indexer->init( + operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn)); ASSERT_EQUALS(0U, specs.size()); ASSERT_OK( indexer->insertSingleDocumentForInitialSyncOrRecovery(operationContext(), diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index 01bc87f7e2b..186e4a38330 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -43,8 +43,8 @@ #include "mongo/db/catalog/list_indexes.h" #include "mongo/db/catalog/local_oplog_info.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/concurrency/lock_state.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" @@ -116,10 +116,7 @@ Status checkSourceAndTargetNamespaces(OperationContext* opCtx, str::stream() << "Source collection " << source.ns() << " does not exist"); } - if (sourceColl->getCollectionOptions().encryptedFieldConfig && - !AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::setUserWriteBlockMode)) { + if (sourceColl->getCollectionOptions().encryptedFieldConfig) { return Status(ErrorCodes::IllegalOperation, "Cannot rename an encrypted collection"); } @@ -132,10 +129,7 @@ Status checkSourceAndTargetNamespaces(OperationContext* opCtx, return Status(ErrorCodes::NamespaceExists, str::stream() << "a view already exists with that name: " << target); } else { - if (targetColl->getCollectionOptions().encryptedFieldConfig && - !AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::setUserWriteBlockMode)) { + if (targetColl->getCollectionOptions().encryptedFieldConfig) { return Status(ErrorCodes::IllegalOperation, "Cannot rename to an existing encrypted collection"); } @@ -339,28 +333,6 @@ Status renameCollectionWithinDB(OperationContext* opCtx, if (!status.isOK()) return status; - if (options.originalCollectionOptions) { - // Check target collection options match expected. - const BSONObj collectionOptions = - targetColl ? targetColl->getCollectionOptions().toBSON() : BSONObj(); - status = checkTargetCollectionOptionsMatch( - target, options.originalCollectionOptions.get(), collectionOptions); - if (!status.isOK()) { - return status; - } - } - if (options.originalIndexes) { - // Check target collection indexes match expected. - const auto currentIndexes = - listIndexesEmptyListIfMissing(opCtx, target, ListIndexesInclude::Nothing); - status = checkTargetCollectionIndexesMatch( - target, options.originalIndexes.get(), currentIndexes); - - if (!status.isOK()) { - return status; - } - } - AutoStatsTracker statsTracker( opCtx, source, @@ -650,10 +622,7 @@ Status renameBetweenDBs(OperationContext* opCtx, // Copy the index descriptions from the source collection. std::vector<BSONObj> indexesToCopy; - for (auto sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator( - opCtx, - IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished | - IndexCatalog::InclusionPolicy::kFrozen); + for (auto sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator(opCtx, true); sourceIndIt->more();) { auto descriptor = sourceIndIt->next()->descriptor(); if (descriptor->isIdIndex()) { @@ -795,44 +764,44 @@ Status renameBetweenDBs(OperationContext* opCtx, void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx, const NamespaceString& sourceNs, const NamespaceString& targetNs, - const RenameCollectionOptions& options) { - // Pass in originalIndexes and originalCollectionOptions to be evaluated later, - // under a collection lock in renameCollectionWithinDB. - validateAndRunRenameCollection(opCtx, sourceNs, targetNs, options); -} + const RenameCollectionOptions& options, + std::list<BSONObj> originalIndexes, + BSONObj originalCollectionOptions) { + AutoGetDb dbLock(opCtx, targetNs.db(), MODE_X); + auto collection = dbLock.getDb() + ? CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, targetNs) + : nullptr; + BSONObj collectionOptions = {}; + if (collection) { + // We do not include the UUID field in the options comparison. It is ok if the target + // collection was dropped and recreated, as long as the new target collection has the same + // options and indexes as the original one did. This is mainly to support concurrent $out + // to the same collection. + collectionOptions = collection->getCollectionOptions().toBSON().removeField("uuid"); + } -Status checkTargetCollectionOptionsMatch(const NamespaceString& targetNss, - const BSONObj& expectedOptions, - const BSONObj& currentOptions) { - // We do not include the UUID field in the options comparison. It is ok if the target collection - // was dropped and recreated, as long as the new target collection has the same options and - // indexes as the original one did. This is mainly to support concurrent $out to the same - // collection. - if (SimpleBSONObjComparator::kInstance.evaluate(expectedOptions.removeField("uuid") != - currentOptions.removeField("uuid"))) { - return Status(ErrorCodes::CommandFailed, - str::stream() - << "collection options of target collection " << targetNss.toString() - << " changed during processing. Original options: " << expectedOptions - << ", new options: " << currentOptions); - }; - return Status::OK(); -} + uassert(ErrorCodes::CommandFailed, + str::stream() << "collection options of target collection " << targetNs.ns() + << " changed during processing. Original options: " + << originalCollectionOptions << ", new options: " << collectionOptions, + SimpleBSONObjComparator::kInstance.evaluate( + originalCollectionOptions.removeField("uuid") == collectionOptions)); + + auto currentIndexes = + listIndexesEmptyListIfMissing(opCtx, targetNs, ListIndexesInclude::Nothing); -Status checkTargetCollectionIndexesMatch(const NamespaceString& targetNss, - const std::list<BSONObj>& expectedIndexes, - const std::list<BSONObj>& currentIndexes) { UnorderedFieldsBSONObjComparator comparator; - if (expectedIndexes.size() != currentIndexes.size() || - !(std::equal(expectedIndexes.begin(), - expectedIndexes.end(), - currentIndexes.begin(), - [&](auto& lhs, auto& rhs) { return comparator.compare(lhs, rhs) == 0; }))) { - return Status(ErrorCodes::CommandFailed, - str::stream() << "indexes of target collection " << targetNss.toString() - << " changed during processing."); - } - return Status::OK(); + uassert( + ErrorCodes::CommandFailed, + str::stream() << "indexes of target collection " << targetNs.ns() + << " changed during processing.", + originalIndexes.size() == currentIndexes.size() && + std::equal(originalIndexes.begin(), + originalIndexes.end(), + currentIndexes.begin(), + [&](auto& lhs, auto& rhs) { return comparator.compare(lhs, rhs) == 0; })); + + validateAndRunRenameCollection(opCtx, sourceNs, targetNs, options); } void validateNamespacesForRenameCollection(OperationContext* opCtx, @@ -886,21 +855,8 @@ void validateNamespacesForRenameCollection(OperationContext* opCtx, !source.isSystemDotViews() && !target.isSystemDotViews()); uassert(ErrorCodes::IllegalOperation, - "renaming system.users collection or renaming to system.users is not allowed", - !source.isSystemDotUsers() && !target.isSystemDotUsers()); - - if (source.isTimeseriesBucketsCollection()) { - uassert(ErrorCodes::IllegalOperation, - "Renaming system.buckets collections is not allowed", - AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::setUserWriteBlockMode)); - - uassert(ErrorCodes::IllegalOperation, - str::stream() << "Cannot rename time-series buckets collection {" << source.ns() - << "} to a non-time-series buckets namespace {" << target.ns() << "}", - target.isTimeseriesBucketsCollection()); - } + "Renaming system.buckets collections is not allowed", + !source.isTimeseriesBucketsCollection()); } void validateAndRunRenameCollection(OperationContext* opCtx, diff --git a/src/mongo/db/catalog/rename_collection.h b/src/mongo/db/catalog/rename_collection.h index ef0d4f8d9d8..253a358f89c 100644 --- a/src/mongo/db/catalog/rename_collection.h +++ b/src/mongo/db/catalog/rename_collection.h @@ -53,35 +53,14 @@ struct RenameCollectionOptions { bool markFromMigrate = false; boost::optional<UUID> expectedSourceUUID; boost::optional<UUID> expectedTargetUUID; - boost::optional<UUID> newTargetCollectionUuid; - boost::optional<std::list<BSONObj>> originalIndexes; - boost::optional<BSONObj> originalCollectionOptions; }; void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx, const NamespaceString& sourceNs, const NamespaceString& targetNs, - const RenameCollectionOptions& options); - -/** - * Checks that CollectionOptions 'expectedOptions' and 'currentOptions' are equal, except for the - * 'uuid' field. Returns a CommandFailed status otherwise. - * To be used by doLocalRenameIfOptionsAndIndexesHaveNotChanged and also its sharding-aware - * equivalent in RenameCollectionCoordinator. - */ -Status checkTargetCollectionOptionsMatch(const NamespaceString& targetNss, - const BSONObj& expectedOptions, - const BSONObj& currentOptions); - -/** - * Checks that the lists of index specs 'expectedIndexes' and 'currentIndexes' are equal. - * To be used by doLocalRenameIfOptionsAndIndexesHaveNotChanged and also its sharding-aware - * equivalent in RenameCollectionCoordinator. Returns a CommandFailed status if indexes do not - * match. - */ -Status checkTargetCollectionIndexesMatch(const NamespaceString& targetNss, - const std::list<BSONObj>& expectedIndexes, - const std::list<BSONObj>& currentIndexes); + const RenameCollectionOptions& options, + std::list<BSONObj> originalIndexes, + BSONObj collectionOptions); Status renameCollection(OperationContext* opCtx, const NamespaceString& source, diff --git a/src/mongo/db/catalog/rename_collection_test.cpp b/src/mongo/db/catalog/rename_collection_test.cpp index b092cf4fffe..215069fd9b7 100644 --- a/src/mongo/db/catalog/rename_collection_test.cpp +++ b/src/mongo/db/catalog/rename_collection_test.cpp @@ -40,7 +40,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/index_builds_coordinator.h" diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp deleted file mode 100644 index 66ed55b6e57..00000000000 --- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright (C) 2024-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include <algorithm> -#include <cstring> -#include <fmt/format.h> -#include <pcrecpp.h> - -#include "mongo/base/string_data.h" -#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" -#include "mongo/util/ctype.h" - -namespace mongo { - -const static StaticImmortal<pcrecpp::RE> appMetadataRegex( - R"re(((?<=^|,)\s*(?:app_metadata|\"app_metadata\")\s*[=:]\s*[({[]\s*))re"); - -static pcrecpp::RE flagMatchRegex(StringData flagName) { - // This check is overly strict, but it suffices for now and ensures that both: - // - The flag name is a valid WiredTiger identifier, and - // - It can be used in the regular expression without needing to escape it - invariant(std::all_of(flagName.begin(), flagName.end(), ctype::isAlpha)); - - // Some examples of possible matches: - // `flag=false` - // `flag:true` - // `flag` - // ` "flag" = false ` - // ` "flag" ` - return pcrecpp::RE(fmt::format( - R"re(((?<=[,({{[])\s*(?:{0}|\"{0}\")(?:\s*[=:]\s*(true|false))?\s*(?=[,)}}\]])))re", - flagName)); -} - -static std::map<StringData, boost::optional<bool>> getFlagsFromWtConfigStringAppMetadata( - const std::string& configString, const std::vector<StringData>& flagNames) { - std::map<StringData, boost::optional<bool>> flags; - - for (const auto& flagName : flagNames) { - auto flagRegex = flagMatchRegex(flagName); - pcrecpp::StringPiece fullMatch, flagValueStr; - auto matches = flagRegex.PartialMatch(configString, &fullMatch, &flagValueStr); - flags.emplace(flagName, - matches ? boost::optional<bool>(flagValueStr == "" || flagValueStr == "true") - : boost::none); - } - - return flags; -} - -std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson( - const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames) { - auto configString = WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions); - return getFlagsFromWtConfigStringAppMetadata(configString.value_or(""), flagNames); -} - -boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions, - StringData flagName) { - return getFlagsFromStorageEngineBson(storageEngineOptions, {flagName})[flagName]; -} - -// Finds or adds the 'app_metadata=(...)' struct inside a WiredTiger config string -// Returns the position inside the struct (after the delimiter, before the first key-value) -static size_t findOrAddAppMetadataStructToConfigString(std::string& configString) { - pcrecpp::StringPiece fullMatch; - auto matches = appMetadataRegex->PartialMatch(configString, &fullMatch); - if (!matches) - configString += configString.empty() ? "app_metadata=()" : ",app_metadata=()"; - return matches ? (fullMatch.data() + fullMatch.size() - configString.data()) - : configString.size() - 1; -} - -// Expand a [pos, len) range inside a config string to include a leading or trailing comma separator -static void expandRangeToIncludeSeparator(const std::string& configString, - size_t& pos, - size_t& len) { - if (pos > 0 && configString[pos - 1] == ',') { - pos--; - len++; - } else if (pos + len < configString.size() && configString[pos + len] == ',') { - len++; - } -} - -static void setFlagsToWtConfigStringAppMetadata( - std::string& configString, const std::map<StringData, boost::optional<bool>>& flags) { - auto metadataPos = findOrAddAppMetadataStructToConfigString(configString); - - for (const auto& [flagName, flagValue] : flags) { - auto flagRegex = flagMatchRegex(flagName); - pcrecpp::StringPiece fullMatch; - // "- 1" allows the positive lookbehind (?<=) at the start of the regex to work - auto matches = flagRegex.PartialMatch(&configString[metadataPos - 1], &fullMatch); - if (matches) { - size_t pos = fullMatch.data() - configString.data(), len = fullMatch.size(); - - if (flagValue.has_value()) { // Replace existing flag - auto flagItem = fmt::format("{}={}", flagName, *flagValue); - configString.replace(pos, len, flagItem); - } else { // Unset existing flag - expandRangeToIncludeSeparator(configString, pos, len); - configString.erase(pos, len); - } - } else if (flagValue.has_value()) { // Add new flag - auto metadataEmpty = strchr(")]}", configString[metadataPos]) != nullptr; - auto flagItem = fmt::format("{}={}{}", flagName, *flagValue, metadataEmpty ? "" : ","); - configString.insert(metadataPos, flagItem); - } - } -} - -BSONObj setFlagsToStorageEngineBson(const BSONObj& storageEngineOptions, - const std::map<StringData, boost::optional<bool>>& flags) { - auto configString = - WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions).value_or(""); - setFlagsToWtConfigStringAppMetadata(configString, flags); - - // Both for safety, and because the regex-based parser can not handle some theoretical cases, - // sanity check that the resulting string is a valid WiredTiger configuration string - auto configStringObj = BSON(WiredTigerUtil::kConfigStringField << configString); - tassert(9218600, - "The resulting WiredTiger configuration string is not valid", - WiredTigerUtil::checkTableCreationOptions(configStringObj.firstElement()).isOK()); - - return WiredTigerUtil::setConfigStringToStorageOptions(storageEngineOptions, configString); -} - -BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions, - StringData flagName, - boost::optional<bool> flagValue) { - return setFlagsToStorageEngineBson(storageEngineOptions, {{flagName, flagValue}}); -} - -} // namespace mongo diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h deleted file mode 100644 index cb4a1b20e2f..00000000000 --- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Copyright (C) 2024-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include <boost/optional/optional.hpp> -#include <map> -#include <vector> - -#include "mongo/base/string_data.h" -#include "mongo/bson/bsonobj.h" - -namespace mongo { - -/** - * Utility functions to get or set boolean flags from/to a storage engine options object - * (see `CollectionOptions::storageEngine`). - * - * The idea is that for exceptional (workaround) purposes, we can use the storage engine - * options object as a flexible structure where new fields can be added retroactively, - * unlike the other parts of the catalog which generally have non-flexible / strict validations. - * For more information, see: SERVER-91195, SERVER-92186. - */ - -std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson( - const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames); - -boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions, - StringData flagName); - -[[nodiscard]] BSONObj setFlagsToStorageEngineBson( - const BSONObj& storageEngineOptions, const std::map<StringData, boost::optional<bool>>& flags); - -[[nodiscard]] BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions, - StringData flagName, - boost::optional<bool> flagValue); - -} // namespace mongo diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp deleted file mode 100644 index 5305668e8fc..00000000000 --- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp +++ /dev/null @@ -1,250 +0,0 @@ -/** - * Copyright (C) 2024-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" -#include "mongo/unittest/bson_test_util.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -namespace { - -static BSONObj makeStorageEngineWithConfigString(StringData configString) { - return BSON(kWiredTigerEngineName << BSON(WiredTigerUtil::kConfigStringField << configString)); -} - -static BSONObj addExtraFields(const BSONObj& storageEngineOptions) { - auto wtObj = storageEngineOptions[kWiredTigerEngineName].Obj(); - return storageEngineOptions.addFields(BSON("dummy1" - << "value1" << kWiredTigerEngineName - << wtObj.addFields(BSON("dummy2" - << "value2")))); -} - -TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenNoWiredTigerConfigString) { - auto options = BSONObj(); - - auto flag = getFlagFromStorageEngineBson(options, "flagA"); - - ASSERT_EQ(boost::none, flag); -} - -TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenOptionsDoesNotContainMetadata) { - auto options = makeStorageEngineWithConfigString("access_pattern_hint=random"); - - auto flag = getFlagFromStorageEngineBson(options, "flagA"); - - ASSERT_EQ(boost::none, flag); -} - -TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataDoesNotContainTheFlag) { - auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1)"); - - auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); - - ASSERT_EQ(boost::none, flags["flagA"]); - ASSERT_EQ(boost::none, flags["flagB"]); -} - -TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsASingleFlag) { - auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=true)"); - - auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); - - ASSERT_EQ(true, flags["flagA"]); - ASSERT_EQ(boost::none, flags["flagB"]); -} - -TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsMultipleFlags) { - auto options = makeStorageEngineWithConfigString("app_metadata=(flagB=true,flagA=false)"); - - auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); - - ASSERT_EQ(false, flags["flagA"]); - ASSERT_EQ(true, flags["flagB"]); -} - -TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataContainsAnInvalidValue) { - auto options = - makeStorageEngineWithConfigString("app_metadata=(flagB=(hello=world),flagA=true)"); - - auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); - - ASSERT_EQ(true, flags["flagA"]); - ASSERT_EQ(boost::none, flags["flagB"]); -} - -TEST(StorageEngineFlagsParserTest, GetTrueWhenMetadataContainsAKeyWithNoValue) { - auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)"); - - auto flag = getFlagFromStorageEngineBson(options, "flagA"); - - ASSERT_EQ(true, flag); -} - -TEST(StorageEngineFlagsParserTest, GetIgnoresUnknownStorageEngineFields) { - auto options = addExtraFields(makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); - - auto flag = getFlagFromStorageEngineBson(options, "flagA"); - - ASSERT_EQ(true, flag); -} - -TEST(StorageEngineFlagsParserTest, GetHandlesTrickyFormatting) { - auto options = addExtraFields(makeStorageEngineWithConfigString( - " access_pattern_hint = random , \"app_metadata\" : [ x=y , " - "\"flagB\": true , z : t ]")); - - auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"}); - - ASSERT_EQ(boost::none, flags["flagA"]); - ASSERT_EQ(true, flags["flagB"]); -} - -TEST(StorageEngineFlagsParserTest, AddFlagToEmptyStorageEngineBson) { - auto options = BSONObj(); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); - - ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); -} - -TEST(StorageEngineFlagsParserTest, AddFlagToEmptyConfigString) { - auto options = makeStorageEngineWithConfigString(""); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); - - ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)")); -} - -TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithNoMetadata) { - auto options = makeStorageEngineWithConfigString("access_pattern_hint=random"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)")); -} - -TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithEmptyMetadata) { - auto options = makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=()"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)")); -} - -TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithOtherFlags) { - auto options = makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(hello2=world2,flagB=true)"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(flagA=false,hello2=world2,flagB=true)")); -} - -TEST(StorageEngineFlagsParserTest, SetExistingFlag) { - auto options = - makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", true); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)")); -} - -TEST(StorageEngineFlagsParserTest, RemoveExistingFlag) { - auto options = makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(flagB=true,x=y,z=t)"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagB", boost::none); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(x=y,z=t)")); -} - -TEST(StorageEngineFlagsParserTest, SetMultipleFlags) { - auto options = makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t,flagC=true)"); - - auto newOptions = setFlagsToStorageEngineBson( - options, {{"flagB", false}, {"flagA", true}, {"flagC", boost::none}}); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(flagA=true,x=y,flagB=false,z=t)")); -} - -TEST(StorageEngineFlagsParserTest, SetFlagWhenMetadataContainsAKeyWithNoValue) { - auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)"); - - auto newOptions = setFlagToStorageEngineBson(options, "flagA", false); - - ASSERT_BSONOBJ_EQ( - newOptions, - makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=false)")); -} - -TEST(StorageEngineFlagsParserTest, SetPreservesUnknownStorageEngineFields) { - auto options = addExtraFields(makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(x=y,flagB=false,z=t)")); - - auto newOptions = setFlagToStorageEngineBson(options, "flagB", true); - - auto expected = addExtraFields(makeStorageEngineWithConfigString( - "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t)")); - ASSERT_BSONOBJ_EQ(newOptions, expected); -} - -TEST(StorageEngineFlagsParserTest, SetHandlesTrickyFormatting) { - auto options = addExtraFields(makeStorageEngineWithConfigString( - " access_pattern_hint = random , \"app_metadata\" : [ x=y , " - "\"flagB\": false , z : t , flagC : true ]")); - - auto newOptions = setFlagsToStorageEngineBson( - options, {{"flagA", false}, {"flagB", true}, {"flagC", boost::none}}); - - auto expected = addExtraFields(makeStorageEngineWithConfigString( - " access_pattern_hint = random , \"app_metadata\" : [ flagA=false,x=y " - " ,flagB=true, z : t ]")); - ASSERT_BSONOBJ_EQ(newOptions, expected); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/catalog/uncommitted_catalog_updates.cpp b/src/mongo/db/catalog/uncommitted_catalog_updates.cpp index 86424f3d71a..67491ba5319 100644 --- a/src/mongo/db/catalog/uncommitted_catalog_updates.cpp +++ b/src/mongo/db/catalog/uncommitted_catalog_updates.cpp @@ -117,7 +117,7 @@ void UncommittedCatalogUpdates::_createCollection(OperationContext* opCtx, // This will throw when registering a namespace which is already in use. CollectionCatalog::write(opCtx, [&, coll = createdColl](CollectionCatalog& catalog) { - catalog.registerCollection(opCtx, coll); + catalog.registerCollection(opCtx, uuid, coll); }); opCtx->recoveryUnit()->onRollback([opCtx, uuid]() { diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp index 9dc4a8dc71b..acbef39ba9c 100644 --- a/src/mongo/db/catalog/validate_adaptor.cpp +++ b/src/mongo/db/catalog/validate_adaptor.cpp @@ -41,7 +41,7 @@ #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog/index_consistency.h" #include "mongo/db/catalog/throttle_cursor.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/index/index_descriptor.h" @@ -53,10 +53,6 @@ #include "mongo/db/storage/execution_context.h" #include "mongo/db/storage/key_string.h" #include "mongo/db/storage/record_store.h" -#include "mongo/db/storage/storage_parameters_gen.h" -#include "mongo/db/timeseries/flat_bson.h" -#include "mongo/db/timeseries/timeseries_constants.h" -#include "mongo/db/timeseries/timeseries_options.h" #include "mongo/logv2/log.h" #include "mongo/rpc/object_check.h" #include "mongo/util/fail_point.h" @@ -67,8 +63,6 @@ namespace mongo { namespace { MONGO_FAIL_POINT_DEFINE(crashOnMultikeyValidateFailure); -MONGO_FAIL_POINT_DEFINE(failIndexKeyOrdering); -MONGO_FAIL_POINT_DEFINE(failRecordStoreTraversal); // Set limit for size of corrupted records that will be reported. const long long kMaxErrorSizeBytes = 1 * 1024 * 1024; @@ -78,9 +72,6 @@ const long long kInterruptIntervalNumBytes = 50 * 1024 * 1024; // 50MB. static constexpr const char* kSchemaValidationFailedReason = "Detected one or more documents not compliant with the collection's schema. Check logs for log " "id 5363500."; -static constexpr const char* kTimeseriesValidationInconsistencyReason = - "Detected one or more documents in this collection incompatible with time-series " - "specifications. For more info, see logs with log id 6698300."; /** * Validate that for each record in a clustered RecordStore the record key (RecordId) matches the @@ -127,197 +118,32 @@ void schemaValidationFailed(CollectionValidation::ValidateState* state, state->setCollectionSchemaViolated(); - if (result != Collection::SchemaValidationResult::kPass) { + // TODO SERVER-65078: remove the testing proctor check. + // When testing is enabled, only warn about non-compliant documents to prevent test failures. + if (TestingProctor::instance().isEnabled() || + Collection::SchemaValidationResult::kWarn == result) { results->warnings.push_back(kSchemaValidationFailedReason); + } else if (Collection::SchemaValidationResult::kError == result) { + results->errors.push_back(kSchemaValidationFailedReason); + results->valid = false; } } -/** - * Checks the value of the bucket's version and if it matches the types of 'data' fields. - */ -Status _validateTimeseriesControlVersion(const BSONObj& recordBson) { - int controlVersion = recordBson.getField(timeseries::kBucketControlFieldName) - .Obj() - .getField(timeseries::kBucketControlVersionFieldName) - .Number(); - if (controlVersion != 1 && controlVersion != 2) { - return Status( - ErrorCodes::BadValue, - fmt::format("Invalid value for 'control.version'. Expected 1 or 2, but got {}.", - controlVersion)); - } - auto dataType = controlVersion == 1 ? BSONType::Object : BSONType::BinData; - // In addition to checking dataType, make sure that closed buckets have BinData Column subtype - auto isCorrectType = [&](BSONElement el) { - if (controlVersion == 1) { - return el.type() == BSONType::Object; - } else { - return el.type() == BSONType::BinData && el.binDataType() == BinDataType::Column; - } - }; - BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); - for (BSONObjIterator bi(data); bi.more();) { - BSONElement e = bi.next(); - if (!isCorrectType(e)) { - return Status(ErrorCodes::TypeMismatch, - fmt::format("Mismatch between time-series schema version and data field " - "type. Expected type {}, but got {}.", - mongo::typeName(dataType), - mongo::typeName(e.type()))); - } - } - return Status::OK(); -} - -/** - * Checks the equivalence between the min and max fields in 'control' for a bucket and - * the corresponding value in 'data'. - */ -Status _validateTimeseriesMinMax(const BSONObj& recordBson, const CollectionPtr& coll) { - BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj(); - BSONObj control = recordBson.getField(timeseries::kBucketControlFieldName).Obj(); - BSONObj controlMin = control.getField(timeseries::kBucketControlMinFieldName).Obj(); - BSONObj controlMax = control.getField(timeseries::kBucketControlMaxFieldName).Obj(); - - auto dataFields = data.getFieldNames<std::set<std::string>>(); - auto controlMinFields = controlMin.getFieldNames<std::set<std::string>>(); - auto controlMaxFields = controlMax.getFieldNames<std::set<std::string>>(); - - // Checks that the number of 'control.min' and 'control.max' fields agrees with number of 'data' - // fields. - if (dataFields.size() != controlMinFields.size() || - dataFields.size() != controlMaxFields.size()) { - return Status( - ErrorCodes::BadValue, - fmt::format( - "Mismatch between the number of time-series control fields and the number " - "of data fields. " - "Control had {} min fields and {} max fields, but observed data had {} fields.", - controlMinFields.size(), - controlMaxFields.size(), - dataFields.size())); - }; - - // Used when checking min timestamp, which is rounded down by granularity. - auto granularity = coll->getTimeseriesOptions()->getGranularity(); - - // Validates that the 'control.min' and 'control.max' field values agree with 'data' field - // values. - for (auto fieldName : dataFields) { - timeseries::MinMax minmax; - auto field = data.getField(fieldName); - - for (BSONElement el : field.Obj()) { - minmax.update(el.wrap(fieldName), boost::none, coll->getDefaultCollator()); - } - - auto controlFieldMin = controlMin.getField(fieldName); - auto controlFieldMax = controlMax.getField(fieldName); - auto min = minmax.min(); - auto max = minmax.max(); - - // Checks whether the min and max values between 'control' and 'data' match, taking - // timestamp granularity into account. - auto checkMinAndMaxMatch = [&]() { - if (fieldName == coll->getTimeseriesOptions()->getTimeField()) { - return controlFieldMin.Date() == - timeseries::roundTimestampToGranularity(min.getField(fieldName).Date(), - granularity) && - controlFieldMax.Date() == max.getField(fieldName).Date(); - } else { - return controlFieldMin.wrap().woCompare(min) == 0 && - controlFieldMax.wrap().woCompare(max) == 0; - } - }; - - if (!checkMinAndMaxMatch()) { - return Status( - ErrorCodes::BadValue, - fmt::format( - "Mismatch between time-series control and observed min or max for field {}. " - "Control had min {} and max {}, but observed data had min {} and max {}.", - fieldName, - controlFieldMin.toString(), - controlFieldMax.toString(), - min.toString(), - max.toString())); - } - } - - return Status::OK(); -} - -/** - * Validates the consistency of a time-series bucket. - */ -Status _validateTimeSeriesBucketRecord(const CollectionPtr& collection, - const BSONObj& recordBson, - ValidateResults* results) { - - if (Status status = _validateTimeseriesControlVersion(recordBson); !status.isOK()) { - return status; - } - - int version = recordBson.getField(timeseries::kBucketControlFieldName) - .Obj() - .getField(timeseries::kBucketControlVersionFieldName) - .Number(); - - // TODO(SERVER-67023): Check closed bucket as part of validation. - if (version == 1) { - if (Status status = _validateTimeseriesMinMax(recordBson, collection); !status.isOK()) { - return status; - } - } - - - return Status::OK(); -} - - -void _timeseriesValidationFailed(CollectionValidation::ValidateState* state, - ValidateResults* results) { - if (state->isTimeseriesDataInconsistent()) { - // Only report the warning message once. - return; - } - state->setTimeseriesDataInconsistent(); - - results->warnings.push_back(kTimeseriesValidationInconsistencyReason); -} - -BSONObj rehydrateKey(const BSONObj& keyPattern, const BSONObj& indexKey) { - // We need to rehydrate the indexKey for improved readability. - // {"": ObjectId(...)} -> {"_id": ObjectId(...)} - auto keysIt = keyPattern.begin(); - auto valuesIt = indexKey.begin(); - - BSONObjBuilder b; - while (keysIt != keyPattern.end()) { - // keysIt and valuesIt must have the same number of elements. - invariant(valuesIt != indexKey.end()); - b.appendAs(*valuesIt, keysIt->fieldName()); - keysIt++; - valuesIt++; - } - return b.obj(); -} } // namespace Status ValidateAdaptor::validateRecord(OperationContext* opCtx, const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results, - ValidationVersion validationVersion) { - const Status status = validateBSON(record.data(), record.size(), validationVersion); + ValidateResults* results) { + const Status status = validateBSON(record.data(), record.size()); if (!status.isOK()) return status; BSONObj recordBson = record.toBson(); *dataSize = recordBson.objsize(); - if (MONGO_unlikely(_validateState->logDiagnostics())) { + if (MONGO_unlikely(_validateState->extraLoggingForTest())) { LOGV2(4666601, "[validate]", "recordId"_attr = recordId, "recordData"_attr = recordBson); } @@ -361,26 +187,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, {multikeyMetadataKeys->begin(), multikeyMetadataKeys->end()}, *documentMultikeyPaths); - auto printMultikeyMetadata = [&]() { - LOGV2(7556100, - "Index is not multikey but document has multikey data", - "indexName"_attr = descriptor->indexName(), - "recordId"_attr = recordId, - "record"_attr = redact(recordBson)); - for (auto& key : *documentKeySet) { - auto indexKey = KeyString::toBsonSafe(key.getBuffer(), - key.getSize(), - iam->getSortedDataInterface()->getOrdering(), - key.getTypeBits()); - const BSONObj rehydratedKey = rehydrateKey(descriptor->keyPattern(), indexKey); - LOGV2(7556101, - "Index key for document with multikey inconsistency", - "indexName"_attr = descriptor->indexName(), - "recordId"_attr = recordId, - "indexKey"_attr = redact(rehydratedKey)); - } - }; - if (!index->isMultikey(opCtx, coll) && shouldBeMultikey) { if (_validateState->fixErrors()) { writeConflictRetry(opCtx, "setIndexAsMultikey", coll->ns().ns(), [&] { @@ -398,17 +204,10 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, << " set to multikey."); results->repaired = true; } else { - printMultikeyMetadata(); - auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()]; - const std::string msg = fmt::format( - "Index {} is not multikey but document with RecordId({}) and {} has multikey " - "data, " - "{} key(s)", - descriptor->indexName(), - recordId.toString(), - recordBson.getField("_id").toString(), - documentKeySet->size()); + std::string msg = str::stream() << "Index " << descriptor->indexName() + << " is not multikey but has more than one" + << " key in document " << recordId; curRecordResults.errors.push_back(msg); curRecordResults.valid = false; if (crashOnMultikeyValidateFailure.shouldFail()) { @@ -436,8 +235,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, << " multikey paths updated."); results->repaired = true; } else { - printMultikeyMetadata(); - std::string msg = str::stream() << "Index " << descriptor->indexName() << " multikey paths do not cover a document. RecordId: " << recordId; @@ -470,7 +267,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx, for (const auto& keyString : *documentKeySet) { try { _totalIndexKeys++; - _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId, results); + _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId); } catch (...) { return exceptionToStatus(); } @@ -483,16 +280,15 @@ namespace { // Ensures that index entries are in increasing or decreasing order. void _validateKeyOrder(OperationContext* opCtx, const IndexCatalogEntry* index, - const KeyStringEntry& currKey, - const KeyStringEntry& prevKey, + const KeyString::Value& currKey, + const KeyString::Value& prevKey, IndexValidateResults* results) { auto descriptor = index->descriptor(); bool unique = descriptor->unique(); // KeyStrings will be in strictly increasing order because all keys are sorted and they are in // the format (Key, RID), and all RecordIDs are unique. - if (currKey.keyString.compare(prevKey.keyString) <= 0 || - MONGO_unlikely(failIndexKeyOrdering.shouldFail())) { + if (currKey.compare(prevKey) <= 0) { if (results && results->valid) { results->errors.push_back(str::stream() << "index '" << descriptor->indexName() @@ -506,20 +302,21 @@ void _validateKeyOrder(OperationContext* opCtx, if (unique) { // Unique indexes must not have duplicate keys. - int cmp = currKey.loc.isLong() - ? currKey.keyString.compareWithoutRecordIdLong(prevKey.keyString) - : currKey.keyString.compareWithoutRecordIdStr(prevKey.keyString); + int cmp = currKey.compareWithoutRecordIdLong(prevKey); if (cmp != 0) { return; } if (results && results->valid) { - auto bsonKey = - KeyString::toBson(currKey.keyString, Ordering::make(descriptor->keyPattern())); + auto bsonKey = KeyString::toBson(currKey, Ordering::make(descriptor->keyPattern())); + auto firstRecordId = + KeyString::decodeRecordIdLongAtEnd(prevKey.getBuffer(), prevKey.getSize()); + auto secondRecordId = + KeyString::decodeRecordIdLongAtEnd(currKey.getBuffer(), currKey.getSize()); results->errors.push_back(str::stream() << "Unique index '" << descriptor->indexName() << "' has duplicate key: " << bsonKey - << ", first record: " << prevKey.loc - << ", second record: " << currKey.loc); + << ", first record: " << firstRecordId + << ", second record: " << secondRecordId); } if (results) { results->valid = false; @@ -538,6 +335,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, IndexInfo& indexInfo = _indexConsistency->getIndexInfo(indexName); int64_t numKeys = 0; + bool isFirstEntry = true; + // The progress meter will be inactive after traversing the record store to allow the message // and the total to be set to different values. if (!_progress->isActive()) { @@ -552,7 +351,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, KeyString::Builder firstKeyStringBuilder( version, BSONObj(), indexInfo.ord, KeyString::Discriminator::kExclusiveBefore); KeyString::Value firstKeyString = firstKeyStringBuilder.getValueCopy(); - boost::optional<KeyStringEntry> prevIndexKeyStringEntry; + KeyString::Value prevIndexKeyStringValue; // Ensure that this index has an open index cursor. const auto indexCursorIt = _validateState->getIndexCursors().find(indexName); @@ -583,8 +382,9 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, bool foundOldUniqueIndexKeys = false; while (indexEntry) { - if (prevIndexKeyStringEntry) { - _validateKeyOrder(opCtx, index, *indexEntry, *prevIndexKeyStringEntry, &indexResults); + if (!isFirstEntry) { + _validateKeyOrder( + opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, &indexResults); } if (!foundOldUniqueIndexKeys && !descriptor->isIdIndex() && descriptor->unique() && @@ -615,7 +415,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, _progress->hit(); numKeys++; - prevIndexKeyStringEntry = indexEntry; + isFirstEntry = false; + prevIndexKeyStringValue = indexEntry->keyString; if (numKeys % kInterruptIntervalNumRecords == 0) { // Periodically checks for interrupts and yields. @@ -631,7 +432,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, "Error advancing index cursor", "error"_attr = ex.toString(), "index"_attr = indexName, - "prevKey"_attr = prevIndexKeyStringEntry->keyString.toString()); + "prevKey"_attr = prevIndexKeyStringValue.toString()); } throw; } @@ -717,8 +518,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx, void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output, - ValidationVersion validationVersion) { + BSONObjBuilder* output) { _numRecords = 0; // need to reset it because this function can be called more than once. long long dataSizeTotal = 0; long long interruptIntervalNumBytes = 0; @@ -742,10 +542,9 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // Because the progress meter is intended as an approximation, it's sufficient to get the number // of records when we begin traversing, even if this number may deviate from the final number. - const auto& coll = _validateState->getCollection(); const char* curopMessage = "Validate: scanning documents"; - const auto totalRecords = coll->getRecordStore()->numRecords(opCtx); - const auto rs = coll->getRecordStore(); + const auto totalRecords = _validateState->getCollection()->getRecordStore()->numRecords(opCtx); + const auto rs = _validateState->getCollection()->getRecordStore(); { stdx::unique_lock<Client> lk(*opCtx->getClient()); _progress.set(CurOp::get(opCtx)->setProgress_inlock(curopMessage, totalRecords)); @@ -756,9 +555,6 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, return; } - bool bucketMixedSchemaDataError = false; - bool bucketMinMaxMalformedError = false; - bool bucketMixedSchemaDataWarning = false; bool corruptRecordsSizeLimitWarning = false; const std::unique_ptr<SeekableRecordThrottleCursor>& traverseRecordStoreCursor = _validateState->getTraverseRecordStoreCursor(); @@ -772,25 +568,11 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, interruptIntervalNumBytes += dataSize; dataSizeTotal += dataSize; size_t validatedSize = 0; - Status status = validateRecord( - opCtx, record->id, record->data, &validatedSize, results, validationVersion); - - // Log the out-of-order entries as errors. - // - // Validate uses a DataCorruptionDetectionMode::kLogAndContinue mode such that data - // corruption errors are logged without throwing, so certain checks must be duplicated here - // as well. - if ((prevRecordId.isValid() && prevRecordId > record->id) || - MONGO_unlikely(failRecordStoreTraversal.shouldFail())) { - // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list and - // not care about deduplication. - static constexpr auto kErrorMessage = "Detected out-of-order documents. See logs."; - if (results->valid || - std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == - results->errors.end()) { - results->errors.push_back(kErrorMessage); - results->valid = false; - } + Status status = validateRecord(opCtx, record->id, record->data, &validatedSize, results); + + // RecordStores are required to return records in RecordId order. + if (prevRecordId.isValid()) { + invariant(prevRecordId < record->id); } // validatedSize = dataSize is not a general requirement as some storage engines may use @@ -821,14 +603,8 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, results->numRemovedCorruptRecords++; _numRecords--; } else { - // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list - // and not care about deduplication. - static constexpr auto kErrorMessage = - "Detected one or more invalid documents. See logs."; - if (results->valid || - std::find(results->errors.begin(), results->errors.end(), kErrorMessage) == - results->errors.end()) { - results->errors.push_back(kErrorMessage); + if (results->valid) { + results->errors.push_back("Detected one or more invalid documents. See logs."); results->valid = false; } @@ -847,76 +623,18 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, // If the document is not corrupted, validate the document against this collection's // schema validator. Don't treat invalid documents as errors since documents can bypass // document validation when being inserted or updated. - auto result = coll->checkValidation(opCtx, record->data.toBson()); + auto result = + _validateState->getCollection()->checkValidation(opCtx, record->data.toBson()); if (result.first != Collection::SchemaValidationResult::kPass) { LOGV2_WARNING(5363500, "Document is not compliant with the collection's schema", - logAttrs(coll->ns()), + logAttrs(_validateState->getCollection()->ns()), "recordId"_attr = record->id, "reason"_attr = result.second); nNonCompliantDocuments++; schemaValidationFailed(_validateState, result.first, results); - } else if (coll->getTimeseriesOptions()) { - // Checks for time-series collection consistency. - Status bucketStatus = - _validateTimeSeriesBucketRecord(coll, record->data.toBson(), results); - - // This log id should be kept in sync with the associated warning messages that are - // returned to the client. - if (!bucketStatus.isOK()) { - LOGV2_WARNING(6698300, - "Document is not compliant with time-series specifications", - logAttrs(coll->ns()), - "recordId"_attr = record->id, - "reason"_attr = bucketStatus); - nNonCompliantDocuments++; - _timeseriesValidationFailed(_validateState, results); - } else { - auto containsMixedSchemaDataResponse = - coll->doesTimeseriesBucketsDocContainMixedSchemaData(record->data.toBson()); - if (!containsMixedSchemaDataResponse.isOK() && !bucketMinMaxMalformedError) { - bucketMinMaxMalformedError = true; - LOGV2_WARNING(8469900, - "Detected a time-series bucket with malformed min/max values", - logAttrs(coll->ns()), - "bucketId"_attr = record->id, - "error"_attr = containsMixedSchemaDataResponse.getStatus()); - results->errors.push_back( - str::stream() - << "Detected a time-series bucket with malformed min/max values"); - results->valid = false; - } else if (containsMixedSchemaDataResponse.isOK() && - containsMixedSchemaDataResponse.getValue()) { - bool mixedSchemaAllowed = - coll->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(true); - if (mixedSchemaAllowed && !bucketMixedSchemaDataWarning) { - bucketMixedSchemaDataWarning = true; - LOGV2_WARNING(8469901, - "Detected a time-series bucket with mixed schema data", - logAttrs(coll->ns()), - "bucketId"_attr = record->id); - results->warnings.push_back( - str::stream() - << "Detected a time-series bucket with mixed schema data"); - } else if (!mixedSchemaAllowed && !bucketMixedSchemaDataError) { - bucketMixedSchemaDataError = true; - LOGV2_WARNING(8469902, - "Detected a time-series bucket with mixed schema data " - "when timeseriesBucketsMayHaveMixedSchemaData is false. " - "You can run the collMod command to set this flag", - logAttrs(coll->ns()), - "bucketId"_attr = record->id); - results->errors.push_back( - str::stream() - << "Detected a time-series bucket with mixed schema data when " - "timeseriesBucketsMayHaveMixedSchemaData is false. You can run " - "the collMod command to set this flag"); - results->valid = false; - } - } - } } } @@ -939,18 +657,20 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx, << " invalid documents."); } - const auto fastCount = coll->numRecords(opCtx); + const auto fastCount = _validateState->getCollection()->numRecords(opCtx); if (_validateState->shouldEnforceFastCount() && fastCount != _numRecords) { - results->errors.push_back( - str::stream() << "fast count (" << fastCount << ") does not match number of records (" - << _numRecords << ") for collection '" << coll->ns() << "'"); + results->errors.push_back(str::stream() << "fast count (" << fastCount + << ") does not match number of records (" + << _numRecords << ") for collection '" + << _validateState->getCollection()->ns() << "'"); results->valid = false; } // Do not update the record store stats if we're in the background as we've validated a // checkpoint and it may not have the most up-to-date changes. if (results->valid && !_validateState->isBackground()) { - coll->getRecordStore()->updateStatsAfterRepair(opCtx, _numRecords, dataSizeTotal); + _validateState->getCollection()->getRecordStore()->updateStatsAfterRepair( + opCtx, _numRecords, dataSizeTotal); } } diff --git a/src/mongo/db/catalog/validate_adaptor.h b/src/mongo/db/catalog/validate_adaptor.h index 99c0a2fceb1..ba548ca6358 100644 --- a/src/mongo/db/catalog/validate_adaptor.h +++ b/src/mongo/db/catalog/validate_adaptor.h @@ -57,8 +57,7 @@ public: const RecordId& recordId, const RecordData& record, size_t* dataSize, - ValidateResults* results, - ValidationVersion validationVersion = currentValidationVersion); + ValidateResults* results); /** * Traverses the index getting index entries to validate them and keep track of the index keys @@ -75,8 +74,7 @@ public: */ void traverseRecordStore(OperationContext* opCtx, ValidateResults* results, - BSONObjBuilder* output, - ValidationVersion validationVersion); + BSONObjBuilder* output); /** * Validates that the number of document keys matches the number of index keys previously diff --git a/src/mongo/db/catalog/validate_results.h b/src/mongo/db/catalog/validate_results.h index c02235a68bc..baa6d78ad42 100644 --- a/src/mongo/db/catalog/validate_results.h +++ b/src/mongo/db/catalog/validate_results.h @@ -30,7 +30,6 @@ #pragma once #include <map> -#include <set> #include <string> #include <vector> @@ -45,6 +44,7 @@ struct IndexValidateResults { std::vector<std::string> errors; std::vector<std::string> warnings; int64_t keysTraversed = 0; + int64_t keysTraversedFromFullValidate = 0; int64_t keysRemovedFromRecordStore = 0; }; @@ -60,9 +60,6 @@ struct ValidateResults { std::vector<BSONObj> extraIndexEntries; std::vector<BSONObj> missingIndexEntries; std::vector<RecordId> corruptRecords; - // Timestamps (startTs, startDurable, stopTs, stopDurableTs) related to records - // with validation errors. See WiredTigerRecordStore::printRecordMetadata(). - std::set<Timestamp> recordTimestamps; long long numRemovedCorruptRecords = 0; long long numRemovedExtraIndexEntries = 0; long long numInsertedMissingIndexEntries = 0; diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp index 01f80ac61cc..434950b9540 100644 --- a/src/mongo/db/catalog/validate_state.cpp +++ b/src/mongo/db/catalog/validate_state.cpp @@ -54,13 +54,12 @@ ValidateState::ValidateState(OperationContext* opCtx, const NamespaceString& nss, ValidateMode mode, RepairMode repairMode, - const AdditionalOptions& additionalOptions, - bool logDiagnostics) + bool turnOnExtraLoggingForTest) : _nss(nss), _mode(mode), _repairMode(repairMode), _dataThrottle(opCtx), - _logDiagnostics(logDiagnostics) { + _extraLoggingForTest(turnOnExtraLoggingForTest) { // Subsequent re-locks will use the UUID when 'background' is true. if (isBackground()) { @@ -81,26 +80,13 @@ ValidateState::ValidateState(OperationContext* opCtx, _collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss); if (!_collection) { - auto view = CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss); - if (!view) { - uasserted(ErrorCodes::NamespaceNotFound, - str::stream() << "Collection '" << _nss << "' does not exist to validate."); - } else { - // Uses the bucket collection in place of the time-series collection view. - if (!view->timeseries()) { - uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view"); - } - _nss = _nss.makeTimeseriesBucketsNamespace(); - if (isBackground()) { - _collectionLock.emplace(opCtx, _nss, MODE_IS); - } else { - _collectionLock.emplace(opCtx, _nss, MODE_X); - } - _collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss); + if (CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss)) { + uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view"); } - } - _validationVersion = additionalOptions.validationVersion; + uasserted(ErrorCodes::NamespaceNotFound, + str::stream() << "Collection '" << _nss << "' does not exist to validate."); + } // RepairMode is incompatible with the ValidateModes kBackground and // kForegroundFullEnforceFastCount. @@ -252,16 +238,15 @@ void ValidateState::initializeCursors(OperationContext* opCtx) { const IndexCatalog* indexCatalog = _collection->getIndexCatalog(); // The index iterator for ready indexes is timestamp-aware and will only return indexes that // are visible at our read time. - const auto it = indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + const std::unique_ptr<IndexCatalog::IndexIterator> it = + indexCatalog->getIndexIterator(opCtx, /*includeUnfinished*/ false); while (it->more()) { const IndexCatalogEntry* entry = it->next(); const IndexDescriptor* desc = entry->descriptor(); auto iam = entry->accessMethod()->asSortedData(); - if (!iam) { - _skippedIndexes.emplace(desc->indexName()); + if (!iam) continue; - } _indexCursors.emplace( desc->indexName(), diff --git a/src/mongo/db/catalog/validate_state.h b/src/mongo/db/catalog/validate_state.h index bb8f431d2d6..df796c686ce 100644 --- a/src/mongo/db/catalog/validate_state.h +++ b/src/mongo/db/catalog/validate_state.h @@ -55,12 +55,15 @@ class ValidateState { ValidateState& operator=(const ValidateState&) = delete; public: + /** + * 'turnOnExtraLoggingForTest' turns on extra logging for test debugging. This parameter is for + * unit testing only. + */ ValidateState(OperationContext* opCtx, const NamespaceString& nss, ValidateMode mode, RepairMode repairMode, - const AdditionalOptions& additionalOptions, - bool logDiagnostics); + bool turnOnExtraLoggingForTest = false); const NamespaceString& nss() const { return _nss; @@ -93,13 +96,6 @@ public: _collectionSchemaViolated = true; } - bool isTimeseriesDataInconsistent() { - return _timeseriesDataInconsistency; - } - void setTimeseriesDataInconsistent() { - _timeseriesDataInconsistency = true; - } - bool fixErrors() const { return _repairMode == RepairMode::kFixErrors; } @@ -127,10 +123,6 @@ public: return _indexes; } - const StringSet& getSkippedIndexes() const { - return _skippedIndexes; - } - /** * Map of index names to index cursors. */ @@ -170,13 +162,11 @@ public: /** * Indicates whether extra logging should occur during validation. + * + * This is for unit testing only. Intended to improve diagnosibility. */ - bool logDiagnostics() { - return _logDiagnostics; - } - - ValidationVersion validationVersion() const { - return _validationVersion; + bool extraLoggingForTest() { + return _extraLoggingForTest; } boost::optional<Timestamp> getValidateTimestamp() { @@ -221,8 +211,6 @@ private: ValidateMode _mode; RepairMode _repairMode; bool _collectionSchemaViolated = false; - bool _timeseriesDataInconsistency = false; - ValidationVersion _validationVersion = currentValidationVersion; boost::optional<ShouldNotConflictWithSecondaryBatchApplicationBlock> _noPBWM; boost::optional<Lock::GlobalLock> _globalLock; @@ -247,10 +235,6 @@ private: std::unique_ptr<SeekableRecordThrottleCursor> _traverseRecordStoreCursor; std::unique_ptr<SeekableRecordThrottleCursor> _seekRecordStoreCursor; - // Stores the set of indexes that will not be validated for some reason, e.g. they are not - // ready. - StringSet _skippedIndexes; - RecordId _firstRecordId; DataThrottle _dataThrottle; @@ -258,8 +242,8 @@ private: // Used to detect when the catalog is re-opened while yielding locks. uint64_t _catalogGeneration; - // Can be set to obtain better insight into what validate sees/does. - bool _logDiagnostics; + // Can be set by unit tests to obtain better insight into what validate sees/does. + bool _extraLoggingForTest; boost::optional<Timestamp> _validateTs = boost::none; }; diff --git a/src/mongo/db/catalog/validate_state_test.cpp b/src/mongo/db/catalog/validate_state_test.cpp index 3e4222da54b..e8e4bfbbbd9 100644 --- a/src/mongo/db/catalog/validate_state_test.cpp +++ b/src/mongo/db/catalog/validate_state_test.cpp @@ -157,9 +157,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError CollectionValidation::ValidateState(opCtx, kNss, CollectionValidation::ValidateMode::kForeground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false), + CollectionValidation::RepairMode::kNone), AssertionException, ErrorCodes::NamespaceNotFound); @@ -167,9 +165,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError CollectionValidation::ValidateState(opCtx, kNss, CollectionValidation::ValidateMode::kBackground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false), + CollectionValidation::RepairMode::kNone), AssertionException, ErrorCodes::NamespaceNotFound); } @@ -188,9 +184,7 @@ TEST_F(ValidateStateTest, UncheckpointedCollectionShouldBeAbleToInitializeCursor opCtx, kNss, CollectionValidation::ValidateMode::kBackground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); // Assert that cursors are able to created on the new collection. validateState.initializeCursors(opCtx); // There should only be a first record id if cursors were initialized successfully. @@ -217,9 +211,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) { opCtx, kNss, CollectionValidation::ValidateMode::kForeground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); validateState.initializeCursors(opCtx); // Make sure all of the indexes were found and cursors opened against them. Including the @@ -237,9 +229,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) { opCtx, kNss, CollectionValidation::ValidateMode::kForeground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); validateState.initializeCursors(opCtx); ASSERT_EQ(validateState.getIndexes().size(), 5); } @@ -266,9 +256,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexesWithBackground) { opCtx, kNss, CollectionValidation::ValidateMode::kBackground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); validateState.initializeCursors(opCtx); // We should be able to open a cursor on each index. @@ -305,9 +293,7 @@ TEST_F(ValidateStateTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatWereL opCtx, kNss, CollectionValidation::ValidateMode::kBackground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); validateState.initializeCursors(opCtx); ASSERT_EQ(validateState.getIndexes().size(), 3); } @@ -320,9 +306,7 @@ TEST_F(ValidateStateTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatWereL opCtx, kNss, CollectionValidation::ValidateMode::kBackground, - CollectionValidation::RepairMode::kNone, - /*additionalOptions=*/{}, - /*logDiagnostics=*/false); + CollectionValidation::RepairMode::kNone); validateState.initializeCursors(opCtx); ASSERT_EQ(validateState.getIndexes().size(), 3); } diff --git a/src/mongo/db/catalog/views_for_database.cpp b/src/mongo/db/catalog/views_for_database.cpp index 404297c11a3..776cf5e3266 100644 --- a/src/mongo/db/catalog/views_for_database.cpp +++ b/src/mongo/db/catalog/views_for_database.cpp @@ -61,8 +61,47 @@ std::shared_ptr<const ViewDefinition> ViewsForDatabase::lookup(const NamespaceSt } Status ViewsForDatabase::reload(OperationContext* opCtx) { + auto reloadCallback = [&](const BSONObj& view) -> Status { + BSONObj collationSpec = view.hasField("collation") ? view["collation"].Obj() : BSONObj(); + auto collator = parseCollator(opCtx, collationSpec); + if (!collator.isOK()) { + return collator.getStatus(); + } + + NamespaceString viewName(view["_id"].str()); + + auto pipeline = view["pipeline"].Obj(); + for (auto&& stage : pipeline) { + if (BSONType::Object != stage.type()) { + return Status(ErrorCodes::InvalidViewDefinition, + str::stream() << "View 'pipeline' entries must be objects, but " + << viewName.toString() + << " has a pipeline element of type " << stage.type()); + } + } + + auto viewDef = std::make_shared<ViewDefinition>(viewName.db(), + viewName.coll(), + view["viewOn"].str(), + pipeline, + std::move(collator.getValue())); + + if (!viewName.isOnInternalDb() && !viewName.isSystem()) { + if (viewDef->timeseries()) { + stats.userTimeseries += 1; + } else { + stats.userViews += 1; + } + } else { + stats.internal += 1; + } + + viewMap[viewName.ns()] = std::move(viewDef); + return Status::OK(); + }; + try { - durable->iterate(opCtx, [&](const BSONObj& view) { return _insert(opCtx, view); }); + durable->iterate(opCtx, reloadCallback); } catch (const DBException& ex) { auto status = ex.toStatus(); LOGV2(22547, @@ -71,60 +110,9 @@ Status ViewsForDatabase::reload(OperationContext* opCtx) { "error"_attr = status); return status; } - valid = true; - return Status::OK(); -} - -Status ViewsForDatabase::insert(OperationContext* opCtx, const BSONObj& view) { - auto status = _insert(opCtx, view); - if (!status.isOK()) { - LOGV2(5387000, - "Could not insert view", - "db"_attr = durable->getName(), - "error"_attr = status); - return status; - } valid = true; - return Status::OK(); -}; - -Status ViewsForDatabase::_insert(OperationContext* opCtx, const BSONObj& view) { - BSONObj collationSpec = view.hasField("collation") ? view["collation"].Obj() : BSONObj(); - auto collator = parseCollator(opCtx, collationSpec); - if (!collator.isOK()) { - return collator.getStatus(); - } - - NamespaceString viewName(view["_id"].str()); - - auto pipeline = view["pipeline"].Obj(); - for (auto&& stage : pipeline) { - if (BSONType::Object != stage.type()) { - return Status(ErrorCodes::InvalidViewDefinition, - str::stream() << "View 'pipeline' entries must be objects, but " - << viewName.toString() << " has a pipeline element of type " - << stage.type()); - } - } - - auto viewDef = std::make_shared<ViewDefinition>(viewName.db(), - viewName.coll(), - view["viewOn"].str(), - pipeline, - std::move(collator.getValue())); - - if (!viewName.isOnInternalDb() && !viewName.isSystem()) { - if (viewDef->timeseries()) { - stats.userTimeseries += 1; - } else { - stats.userViews += 1; - } - } else { - stats.internal += 1; - } - viewMap[viewName.ns()] = std::move(viewDef); return Status::OK(); } @@ -147,8 +135,7 @@ Status ViewsForDatabase::validateCollation(OperationContext* opCtx, Status ViewsForDatabase::upsertIntoGraph(OperationContext* opCtx, const ViewDefinition& viewDef, - const PipelineValidatorFn& validatePipeline, - const bool needsValidation) { + const PipelineValidatorFn& validatePipeline) { // Performs the insert into the graph. auto doInsert = [this, opCtx, &validatePipeline](const ViewDefinition& viewDef, bool needsValidation) -> Status { @@ -203,7 +190,7 @@ Status ViewsForDatabase::upsertIntoGraph(OperationContext* opCtx, // is simply a no-op. viewGraph.remove(viewDef.name()); - return doInsert(viewDef, needsValidation); + return doInsert(viewDef, true); } } // namespace mongo diff --git a/src/mongo/db/catalog/views_for_database.h b/src/mongo/db/catalog/views_for_database.h index ab4329bf9d9..914adf60df7 100644 --- a/src/mongo/db/catalog/views_for_database.h +++ b/src/mongo/db/catalog/views_for_database.h @@ -93,11 +93,6 @@ public: Status reload(OperationContext* opCtx); /** - * Inserts the view into the view map. - */ - Status insert(OperationContext* opCtx, const BSONObj& view); - - /** * Returns Status::OK if each view namespace in 'refs' has the same default collation as * 'view'. Otherwise, returns ErrorCodes::OptionNotSupportedOnView. */ @@ -108,17 +103,11 @@ public: /** * Parses the view definition pipeline, attempts to upsert into the view graph, and * refreshes the graph if necessary. Returns an error status if the resulting graph - * would be invalid. needsValidation can be set to false if the view already exists in the - * durable view catalog and skips checking that the resulting dependency graph is acyclic and - * within the maximum depth. + * would be invalid. */ Status upsertIntoGraph(OperationContext* opCtx, const ViewDefinition& viewDef, - const PipelineValidatorFn&, - bool needsValidation); - -private: - Status _insert(OperationContext* opCtx, const BSONObj& view); + const PipelineValidatorFn&); }; } // namespace mongo |
