summaryrefslogtreecommitdiff
path: root/src/mongo/db/catalog
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /src/mongo/db/catalog
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'src/mongo/db/catalog')
-rw-r--r--src/mongo/db/catalog/README.md24
-rw-r--r--src/mongo/db/catalog/SConscript83
-rw-r--r--src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp86
-rw-r--r--src/mongo/db/catalog/backwards_compatible_collection_options_util.h130
-rw-r--r--src/mongo/db/catalog/capped_utils.cpp7
-rw-r--r--src/mongo/db/catalog/catalog_control.cpp68
-rw-r--r--src/mongo/db/catalog/catalog_control.h9
-rw-r--r--src/mongo/db/catalog/catalog_control_test.cpp4
-rw-r--r--src/mongo/db/catalog/catalog_stats.cpp14
-rw-r--r--src/mongo/db/catalog/catalog_stats.h38
-rw-r--r--src/mongo/db/catalog/coll_mod.cpp170
-rw-r--r--src/mongo/db/catalog/coll_mod.h12
-rw-r--r--src/mongo/db/catalog/coll_mod_index.cpp29
-rw-r--r--src/mongo/db/catalog/coll_mod_test.cpp106
-rw-r--r--src/mongo/db/catalog/collection.h30
-rw-r--r--src/mongo/db/catalog/collection_catalog.cpp405
-rw-r--r--src/mongo/db/catalog/collection_catalog.h111
-rw-r--r--src/mongo/db/catalog/collection_catalog_bm.cpp122
-rw-r--r--src/mongo/db/catalog/collection_catalog_helper.cpp18
-rw-r--r--src/mongo/db/catalog/collection_catalog_test.cpp91
-rw-r--r--src/mongo/db/catalog/collection_compact.cpp13
-rw-r--r--src/mongo/db/catalog/collection_impl.cpp325
-rw-r--r--src/mongo/db/catalog/collection_impl.h20
-rw-r--r--src/mongo/db/catalog/collection_mock.h13
-rw-r--r--src/mongo/db/catalog/collection_operation_source.cpp4
-rw-r--r--src/mongo/db/catalog/collection_operation_source.h1
-rw-r--r--src/mongo/db/catalog/collection_test.cpp26
-rw-r--r--src/mongo/db/catalog/collection_validation.cpp216
-rw-r--r--src/mongo/db/catalog/collection_validation.h13
-rw-r--r--src/mongo/db/catalog/collection_validation_test.cpp35
-rw-r--r--src/mongo/db/catalog/collection_writer_test.cpp3
-rw-r--r--src/mongo/db/catalog/create_collection.cpp159
-rw-r--r--src/mongo/db/catalog/create_collection_test.cpp2
-rw-r--r--src/mongo/db/catalog/database.h7
-rw-r--r--src/mongo/db/catalog/database_holder_impl.cpp8
-rw-r--r--src/mongo/db/catalog/database_impl.cpp13
-rw-r--r--src/mongo/db/catalog/database_impl.h8
-rw-r--r--src/mongo/db/catalog/database_test.cpp4
-rw-r--r--src/mongo/db/catalog/drop_collection.cpp76
-rw-r--r--src/mongo/db/catalog/drop_collection.h8
-rw-r--r--src/mongo/db/catalog/drop_database.cpp18
-rw-r--r--src/mongo/db/catalog/drop_database_test.cpp2
-rw-r--r--src/mongo/db/catalog/drop_indexes.cpp57
-rw-r--r--src/mongo/db/catalog/health_log.cpp24
-rw-r--r--src/mongo/db/catalog/health_log.h44
-rw-r--r--src/mongo/db/catalog/health_log_interface.cpp54
-rw-r--r--src/mongo/db/catalog/health_log_interface.h92
-rw-r--r--src/mongo/db/catalog/index_build_block.cpp59
-rw-r--r--src/mongo/db/catalog/index_build_block.h11
-rw-r--r--src/mongo/db/catalog/index_build_entry_test.cpp3
-rw-r--r--src/mongo/db/catalog/index_builds_manager.cpp5
-rw-r--r--src/mongo/db/catalog/index_builds_manager.h1
-rw-r--r--src/mongo/db/catalog/index_catalog.h50
-rw-r--r--src/mongo/db/catalog/index_catalog_entry.h5
-rw-r--r--src/mongo/db/catalog/index_catalog_entry_impl.cpp11
-rw-r--r--src/mongo/db/catalog/index_catalog_entry_impl.h6
-rw-r--r--src/mongo/db/catalog/index_catalog_impl.cpp266
-rw-r--r--src/mongo/db/catalog/index_catalog_impl.h21
-rw-r--r--src/mongo/db/catalog/index_consistency.cpp235
-rw-r--r--src/mongo/db/catalog/index_consistency.h15
-rw-r--r--src/mongo/db/catalog/index_key_validate.cpp96
-rw-r--r--src/mongo/db/catalog/index_key_validate.h93
-rw-r--r--src/mongo/db/catalog/index_key_validate_test.cpp95
-rw-r--r--src/mongo/db/catalog/index_repair.cpp2
-rw-r--r--src/mongo/db/catalog/list_indexes.cpp2
-rw-r--r--src/mongo/db/catalog/local_oplog_info.cpp24
-rw-r--r--src/mongo/db/catalog/multi_index_block.cpp73
-rw-r--r--src/mongo/db/catalog/multi_index_block.h1
-rw-r--r--src/mongo/db/catalog/multi_index_block_test.cpp21
-rw-r--r--src/mongo/db/catalog/rename_collection.cpp126
-rw-r--r--src/mongo/db/catalog/rename_collection.h27
-rw-r--r--src/mongo/db/catalog/rename_collection_test.cpp2
-rw-r--r--src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp161
-rw-r--r--src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h62
-rw-r--r--src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp250
-rw-r--r--src/mongo/db/catalog/uncommitted_catalog_updates.cpp2
-rw-r--r--src/mongo/db/catalog/validate_adaptor.cpp390
-rw-r--r--src/mongo/db/catalog/validate_adaptor.h6
-rw-r--r--src/mongo/db/catalog/validate_results.h5
-rw-r--r--src/mongo/db/catalog/validate_state.cpp35
-rw-r--r--src/mongo/db/catalog/validate_state.h38
-rw-r--r--src/mongo/db/catalog/validate_state_test.cpp32
-rw-r--r--src/mongo/db/catalog/views_for_database.cpp97
-rw-r--r--src/mongo/db/catalog/views_for_database.h15
84 files changed, 1227 insertions, 3918 deletions
diff --git a/src/mongo/db/catalog/README.md b/src/mongo/db/catalog/README.md
index 5fe1a00de68..ddb5d479da2 100644
--- a/src/mongo/db/catalog/README.md
+++ b/src/mongo/db/catalog/README.md
@@ -432,30 +432,6 @@ See
[wtRcToStatus](https://github.com/mongodb/mongo/blob/c799851554dc01493d35b43701416e9c78b3665c/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp#L178-L183)
where we throw the exception in WiredTiger.
See [TemporarilyUnavailableException](https://github.com/mongodb/mongo/blob/c799851554dc01493d35b43701416e9c78b3665c/src/mongo/db/concurrency/temporarily_unavailable_exception.h#L39-L45).
-
-## TransactionTooLargeForCacheException
-
-A TransactionTooLargeForCacheException may be thrown inside the server to indicate that an operation
-was rolled-back and is unlikely to ever complete because the storage engine cache is insufficient,
-even in the absence of concurrent operations. This is determined by a simple heuristic wherein,
-after a rollback, a threshold on the proportion of total dirty cache bytes the running transaction
-can represent and still be considered fullfillable is checked. The threshold can be tuned with the
-`transactionTooLargeForCacheThreshold` parameter. Setting this threshold to its maximum value (1.0)
-causes the check to be skipped and TransactionTooLargeForCacheException to be disabled.
-
-On replica sets, if an operation succeeds on a primary, it should also succeed on a secondary. It
-would be possible to convert to both TemporarilyUnavailableException and WriteConflictException,
-as if TransactionTooLargeForCacheException was disabled. But on secondaries the only
-difference between the two is the rate at which the operation is retried. Hence,
-TransactionTooLargeForCacheException is always converted to a WriteConflictException, which retries
-faster, to avoid stalling replication longer than necessary.
-
-Prior to 6.3, or when TransactionTooLargeForCacheException is disabled, multi-document
-transactions always return a WriteConflictException, which may result in drivers retrying an
-operation indefinitely. For non-multi-document operations, there is a limited number of retries on
-TemporarilyUnavailableException, but it might still be beneficial to not retry operations which are
-unlikely to complete and are disruptive for concurrent operations.
-
## Collection and Index Writes
Collection write operations (inserts, updates, and deletes) perform storage engine writes to both
diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript
index e20adc6cf10..b116e2ad284 100644
--- a/src/mongo/db/catalog/SConscript
+++ b/src/mongo/db/catalog/SConscript
@@ -38,7 +38,6 @@ env.Library(
],
LIBDEPS=[
'$BUILD_DIR/mongo/db/pipeline/change_stream_pre_and_post_images_options',
- '$BUILD_DIR/mongo/db/query/query_shape/query_shape',
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
@@ -101,29 +100,17 @@ env.Library(
)
env.Library(
- target='health_log_interface',
- source=[
- 'health_log.idl',
- 'health_log_interface.cpp',
- ],
- LIBDEPS_PRIVATE=[
- '$BUILD_DIR/mongo/db/service_context',
- ],
-)
-
-env.Library(
target='health_log',
source=[
- 'health_log.cpp',
+ "health_log.cpp",
+ "health_log.idl",
],
LIBDEPS_PRIVATE=[
"$BUILD_DIR/mongo/base",
"$BUILD_DIR/mongo/db/concurrency/deferred_writer",
- "$BUILD_DIR/mongo/db/namespace_string",
"$BUILD_DIR/mongo/db/service_context",
"$BUILD_DIR/mongo/idl/idl_parser",
'collection_options',
- 'health_log_interface',
],
)
@@ -134,9 +121,9 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/db/catalog_raii',
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/index/index_access_method',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/storage/key_string',
'validate_state',
]
@@ -150,9 +137,9 @@ env.Library(
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/db/catalog_raii',
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/query/query_knobs',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/storage/record_store_base',
'$BUILD_DIR/mongo/db/storage/storage_repair_observer',
'index_repair',
@@ -182,7 +169,8 @@ env.Library(
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/db/common',
- '$BUILD_DIR/mongo/db/index/index_access_method',
+ '$BUILD_DIR/mongo/db/index/index_descriptor',
+ '$BUILD_DIR/mongo/db/index/key_generator',
'$BUILD_DIR/mongo/db/index_names',
'$BUILD_DIR/mongo/db/matcher/expressions',
'$BUILD_DIR/mongo/db/query/collation/collator_factory_interface',
@@ -222,6 +210,7 @@ env.Library(
'$BUILD_DIR/mongo/db/collection_index_usage_tracker',
'$BUILD_DIR/mongo/db/common',
'$BUILD_DIR/mongo/db/index/index_access_method',
+ '$BUILD_DIR/mongo/db/index/index_descriptor',
'$BUILD_DIR/mongo/db/index_names',
'$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
'$BUILD_DIR/mongo/db/ttl_collection_cache',
@@ -264,9 +253,9 @@ env.Library(
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/db/catalog_raii',
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/index/index_access_method',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
'$BUILD_DIR/mongo/db/resumable_index_builds_idl',
'$BUILD_DIR/mongo/db/service_context',
@@ -294,6 +283,7 @@ env.Library(
'views_for_database.cpp',
],
LIBDEPS_PRIVATE=[
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
'$BUILD_DIR/mongo/db/multitenancy',
'$BUILD_DIR/mongo/db/namespace_string',
'$BUILD_DIR/mongo/db/profile_filter',
@@ -316,7 +306,6 @@ env.Benchmark(
LIBDEPS=[
'$BUILD_DIR/mongo/db/concurrency/lock_manager',
'$BUILD_DIR/mongo/db/multitenancy',
- 'collection',
'collection_catalog',
],
)
@@ -331,8 +320,6 @@ env.Library(
'$BUILD_DIR/mongo/db/multitenancy',
'$BUILD_DIR/mongo/db/rebuild_indexes',
'$BUILD_DIR/mongo/db/service_context',
- '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range',
- 'catalog_stats',
'collection',
'collection_catalog',
'database_holder',
@@ -370,15 +357,16 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/bson/bson_validate',
'$BUILD_DIR/mongo/db/collection_index_usage_tracker',
'$BUILD_DIR/mongo/db/commands/server_status_core',
'$BUILD_DIR/mongo/db/concurrency/lock_manager',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/db_raii',
'$BUILD_DIR/mongo/db/index/index_access_method',
+ '$BUILD_DIR/mongo/db/index/index_access_method_factory',
+ '$BUILD_DIR/mongo/db/index/index_access_methods',
'$BUILD_DIR/mongo/db/multitenancy',
'$BUILD_DIR/mongo/db/op_observer',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/record_id_helpers',
'$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper',
'$BUILD_DIR/mongo/db/repl/oplog',
@@ -395,7 +383,6 @@ env.Library(
'$BUILD_DIR/mongo/db/storage/storage_util',
'$BUILD_DIR/mongo/db/system_index',
'$BUILD_DIR/mongo/db/timeseries/timeseries_conversion_util',
- '$BUILD_DIR/mongo/db/timeseries/timeseries_extended_range',
'$BUILD_DIR/mongo/db/transaction',
'$BUILD_DIR/mongo/db/ttl_collection_cache',
'$BUILD_DIR/mongo/db/vector_clock',
@@ -408,14 +395,13 @@ env.Library(
'collection_catalog',
'collection_options',
'database_holder',
- 'health_log_interface',
+ 'health_log',
'index_build_block',
'index_catalog',
'index_catalog_entry',
'index_key_validate',
'index_repair',
'local_oplog_info',
- 'storage_engine_collection_options_flags_parser',
'throttle_cursor',
'validate_idl',
'validate_state',
@@ -445,20 +431,17 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/bson/bson_validate',
'$BUILD_DIR/mongo/bson/util/bson_column',
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
+ '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/db_raii',
'$BUILD_DIR/mongo/db/index/index_access_method',
'$BUILD_DIR/mongo/db/index_names',
'$BUILD_DIR/mongo/db/multi_key_path_tracker',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/record_id_helpers',
"$BUILD_DIR/mongo/db/service_context",
'$BUILD_DIR/mongo/db/storage/execution_context',
'$BUILD_DIR/mongo/db/storage/key_string',
- '$BUILD_DIR/mongo/db/timeseries/bucket_catalog',
- '$BUILD_DIR/mongo/db/timeseries/timeseries_options',
'$BUILD_DIR/mongo/idl/basic_types',
'catalog_impl',
'collection_options',
@@ -475,8 +458,8 @@ env.Library(
'throttle_cursor.cpp',
],
LIBDEPS_PRIVATE=[
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/index/index_access_method',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/util/fail_point',
'validate_idl',
],
@@ -489,7 +472,6 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/db/logical_time',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/repl/optime',
'$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
'$BUILD_DIR/mongo/db/storage/flow_control',
@@ -502,7 +484,6 @@ env.Library(
env.Library(
target='catalog_helpers',
source=[
- 'backwards_compatible_collection_options_util.cpp',
'capped_utils.cpp',
'collection_catalog_helper.cpp',
'coll_mod.cpp',
@@ -518,7 +499,6 @@ env.Library(
],
LIBDEPS_PRIVATE=[
'$BUILD_DIR/mongo/base',
- '$BUILD_DIR/mongo/db/concurrency/exception_util',
'$BUILD_DIR/mongo/db/db_raii',
'$BUILD_DIR/mongo/db/index/index_access_method',
'$BUILD_DIR/mongo/db/index_builds_coordinator_interface',
@@ -564,8 +544,8 @@ env.Library(
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/db/collection_index_usage_tracker',
'$BUILD_DIR/mongo/db/concurrency/lock_manager',
+ '$BUILD_DIR/mongo/db/curop',
'$BUILD_DIR/mongo/db/fts/base_fts',
- '$BUILD_DIR/mongo/db/query/query_stats/query_stats',
'$BUILD_DIR/mongo/db/s/sharding_api_d',
'$BUILD_DIR/mongo/db/service_context',
'index_catalog',
@@ -620,21 +600,7 @@ env.Library(
)
if wiredtiger:
- wtEnv = env.Clone()
- wtEnv.InjectThirdParty(libraries=["wiredtiger"])
-
- wtEnv.Library(
- target="storage_engine_collection_options_flags_parser",
- source=[
- "storage_engine_collection_options_flags_parser.cpp",
- ],
- LIBDEPS_PRIVATE=[
- "$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger",
- "$BUILD_DIR/third_party/shim_pcrecpp",
- ],
- )
-
- wtEnv.CppUnitTest(
+ env.CppUnitTest(
target='db_catalog_test',
source=[
'capped_utils_test.cpp',
@@ -644,7 +610,6 @@ if wiredtiger:
'collection_test.cpp',
'collection_validation_test.cpp',
'collection_writer_test.cpp',
- 'coll_mod_test.cpp',
'commit_quorum_options_test.cpp',
'create_collection_test.cpp',
'database_test.cpp',
@@ -656,17 +621,14 @@ if wiredtiger:
'index_spec_validate_test.cpp',
'multi_index_block_test.cpp',
'rename_collection_test.cpp',
- 'storage_engine_collection_options_flags_parser_test.cpp',
'throttle_cursor_test.cpp',
'validate_state_test.cpp',
],
LIBDEPS=[
'$BUILD_DIR/mongo/db/auth/authmocks',
- '$BUILD_DIR/mongo/db/commands/create_command',
'$BUILD_DIR/mongo/db/commands/test_commands_enabled',
'$BUILD_DIR/mongo/db/concurrency/lock_manager',
'$BUILD_DIR/mongo/db/db_raii',
- '$BUILD_DIR/mongo/db/dbhelpers',
'$BUILD_DIR/mongo/db/index_builds_coordinator_mongod',
'$BUILD_DIR/mongo/db/matcher/expressions',
'$BUILD_DIR/mongo/db/multitenancy',
@@ -677,7 +639,6 @@ if wiredtiger:
'$BUILD_DIR/mongo/db/query/query_test_service_context',
'$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper',
'$BUILD_DIR/mongo/db/repl/oplog',
- '$BUILD_DIR/mongo/db/repl/oplog_application',
'$BUILD_DIR/mongo/db/repl/optime',
'$BUILD_DIR/mongo/db/repl/repl_coordinator_interface',
'$BUILD_DIR/mongo/db/repl/replmocks',
@@ -686,7 +647,6 @@ if wiredtiger:
'$BUILD_DIR/mongo/db/service_context_d_test_fixture',
'$BUILD_DIR/mongo/db/service_context_test_fixture',
'$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger',
- '$BUILD_DIR/mongo/db/timeseries/timeseries_collmod',
'$BUILD_DIR/mongo/db/timeseries/timeseries_options',
'$BUILD_DIR/mongo/unittest/unittest',
'$BUILD_DIR/mongo/util/clock_source_mock',
@@ -705,7 +665,6 @@ if wiredtiger:
'index_builds_manager',
'index_key_validate',
'multi_index_block',
- 'storage_engine_collection_options_flags_parser',
'throttle_cursor',
'validate_idl',
'validate_state',
diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp b/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp
deleted file mode 100644
index d966d6108c6..00000000000
--- a/src/mongo/db/catalog/backwards_compatible_collection_options_util.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-// TODO SERVER-92265 evaluate getting rid of this util
-
-#include "mongo/db/catalog/backwards_compatible_collection_options_util.h"
-#include "mongo/bson/bsonobj.h"
-#include "mongo/bson/simple_bsonobj_comparator.h"
-#include "mongo/db/repl/oplog_entry.h"
-
-namespace mongo {
-namespace backwards_compatible_collection_options {
-std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd) {
- const BSONObj collModCmdStrippedBackwardsIncompatibleParams =
- collModCmd.removeFields(kBackwardsCompatibleCollectionOptions);
- if (SimpleBSONObjComparator::kInstance.evaluate(collModCmdStrippedBackwardsIncompatibleParams ==
- collModCmd)) {
- return {collModCmd, BSONObj()};
- }
-
- const BSONObj backwardsIncompatibleFields = [&]() {
- BSONObjBuilder bob;
- for (auto [fieldName, elem] : collModCmd) {
- if (kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) {
- bob.append(elem);
- }
- }
- return bob.obj();
- }();
-
- return {collModCmdStrippedBackwardsIncompatibleParams, backwardsIncompatibleFields};
-}
-
-BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry) {
- uassert(ErrorCodes::IllegalOperation,
- str::stream() << "Can't extract `collMod` command from non-collMod oplog entry: "
- << entry.toBSONForLogging(),
- entry.getCommandType() == repl::OplogEntry::CommandType::kCollMod);
-
- if (!entry.getObject2()) {
- return entry.getObject();
- }
-
- BSONObj incompatibleFields = entry.getObject2()->getObjectField(additionalCollModO2Field);
- if (incompatibleFields.isEmpty()) {
- return entry.getObject();
- }
-
- // Only consider backwards incompatible fields supported in the current [sub-]version
- for (auto [fieldName, elem] : incompatibleFields) {
- if (!kBackwardsCompatibleCollectionOptions.count(fieldName.toString())) {
- incompatibleFields = incompatibleFields.removeField(fieldName);
- }
- }
-
- return entry.getObject().addFields(incompatibleFields);
-}
-
-} // namespace backwards_compatible_collection_options
-} // namespace mongo
diff --git a/src/mongo/db/catalog/backwards_compatible_collection_options_util.h b/src/mongo/db/catalog/backwards_compatible_collection_options_util.h
deleted file mode 100644
index d5c47b45e6d..00000000000
--- a/src/mongo/db/catalog/backwards_compatible_collection_options_util.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-// TODO SERVER-92265 evaluate getting rid of this file
-
-#include "mongo/db/repl/oplog_entry.h"
-
-namespace mongo {
-
-/**
- *
- * This utility is providing primitives to manage catalog parameters for which the actual value may
- * have been missing or incorrect in previous mongod [sub-]versions. It is only meant to fix catalog
- * issues in existing versions and must NOT be used for implementing new features.
- *
- * The functions under this namespace are offering an abstraction to work around the following
- * limitations:
- * - collMod command is strict (so can't simply add a parameter to previous mongod [sub-]versions)
- * - Collection options are strict (so can't simply add an option to previous mongod [sub-]versions)
- *
- */
-namespace backwards_compatible_collection_options {
-
-const std::string kTimeseriesBucketsMayHaveMixedSchemaData =
- "timeseriesBucketsMayHaveMixedSchemaData";
-
-const std::string additionalCollModO2Field = "backwardsIncompatibleCollModParameters";
-
-/**
- * Backwards incompatible catalog parameters for which the actual value may have been missing or
- * incorrect in previous mongod [sub-]versions.
- */
-const std::set<std::string> kBackwardsCompatibleCollectionOptions{
- kTimeseriesBucketsMayHaveMixedSchemaData};
-
-/**
- * Strips backwards incompatible fields from a collMod command and places them into a
- * different BSON object.
- *
- * Returns two BSON objects:
- * - A backwards compatible collMod oplog entry (not to generate crashes when applied by
- * incompatible mongod [sub-]versions).
- * - A field meant to be added to the `o2` sub-object (parsable by new mongod [sub-]versions).
- *
- * Example:
- *
- * - Original command:
- * {"collMod":"testdb.system.buckets.testcoll", "timeseriesBucketsMayHaveMixedSchemaData":true }
- *
- * - Expected oplog entry with `timeseriesBucketsMayHaveMixedSchemaData` backwards incompatible
- * collMod parameter.
- *
- * {
- * "oplogEntry":{
- * "op":"c",
- * "ns":"testdb.$cmd",
- * "ui":"UUID(""7302d025-cb9c-4a16-9222-0d5aeefbc039"")",
- * "o":{
- * "collMod":"system.buckets.testcoll"
- * },
- * "o2":{
- * "collectionOptions_old":{
- * "uuid": UUID("7302d025-cb9c-4a16-9222-0d5aeefbc039"),
- * "Validator":{ "...REDACTED..." },
- * "clusteredIndex":true,
- * "timeseries":{
- * "timeField":"t",
- * “granularity":"seconds",
- * "bucketMaxSpanSeconds":3600
- * }
- * },
- * "backwardsIncompatibleCollModParameters":{
- * "timeseriesBucketsMayHaveMixedSchemaData":true
- * }
- * },
- * "ts":Timestamp(1720003401,4),
- * "t":1,
- * "v":2,
- * "wall":new Date(1720003401165)
- * }
- * }
- *
- */
-std::pair<BSONObj, BSONObj> getCollModCmdAndAdditionalO2Field(const BSONObj& collModCmd);
-
-/**
- * Rebuilds a collMod command from an oplog entry.
- *
- * Returns a bson object:
- * - A collMod command inclusive of potential backwards incompatible fields present in the oplog
- * entry's `o2` sub-object.
- *
- * Example: collMod command parsed from the sample oplog entry documented above.
- *
- * {
- * "collMod":"system.buckets.testcoll",
- * "timeseriesBucketsMayHaveMixedSchemaData":true
- * }
- *
- */
-BSONObj parseCollModCmdFromOplogEntry(const repl::OplogEntry& entry);
-
-} // namespace backwards_compatible_collection_options
-} // namespace mongo
diff --git a/src/mongo/db/catalog/capped_utils.cpp b/src/mongo/db/catalog/capped_utils.cpp
index 77ccf110f97..945881e58c5 100644
--- a/src/mongo/db/catalog/capped_utils.cpp
+++ b/src/mongo/db/catalog/capped_utils.cpp
@@ -43,7 +43,7 @@
#include "mongo/db/catalog/rename_collection.h"
#include "mongo/db/catalog_raii.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/index_builds_coordinator.h"
#include "mongo/db/namespace_string.h"
@@ -245,7 +245,8 @@ void cloneCollectionAsCapped(OperationContext* opCtx,
insertStmt.oplogSlot = oplogSlots.front();
}
- uassertStatusOK(toCollection->insertDocument(opCtx, insertStmt, nullOpDebug, true));
+ uassertStatusOK(toCollection->insertDocument(
+ opCtx, InsertStatement(objToClone), nullOpDebug, true));
wunit.commit();
// Go to the next document
@@ -253,7 +254,7 @@ void cloneCollectionAsCapped(OperationContext* opCtx,
} catch (const WriteConflictException&) {
CurOp::get(opCtx)->debug().additiveMetrics.incrementWriteConflicts(1);
retries++; // logAndBackoff expects this to be 1 on first call.
- logWriteConflictAndBackoff(retries, "cloneCollectionAsCapped", fromNss.ns());
+ WriteConflictException::logAndBackoff(retries, "cloneCollectionAsCapped", fromNss.ns());
// Can't use writeConflictRetry since we need to save/restore exec around call to
// abandonSnapshot.
diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp
index 19f413fc1b6..dadfed598c0 100644
--- a/src/mongo/db/catalog/catalog_control.cpp
+++ b/src/mongo/db/catalog/catalog_control.cpp
@@ -35,7 +35,6 @@
#include "mongo/db/catalog/catalog_control.h"
-#include "mongo/db/catalog/catalog_stats.h"
#include "mongo/db/catalog/collection.h"
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/database.h"
@@ -44,18 +43,17 @@
#include "mongo/db/index_builds_coordinator.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/rebuild_indexes.h"
-#include "mongo/db/repl/oplog.h"
#include "mongo/db/tenant_database_name.h"
-#include "mongo/db/timeseries/timeseries_extended_range.h"
#include "mongo/logv2/log.h"
namespace mongo {
namespace catalog {
namespace {
-void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx,
- StorageEngine* storageEngine,
- const PreviousCatalogState& previousCatalogState,
- Timestamp stableTimestamp) {
+void reopenAllDatabasesAndReloadCollectionCatalog(
+ OperationContext* opCtx,
+ StorageEngine* storageEngine,
+ const MinVisibleTimestampMap& minVisibleTimestampMap,
+ Timestamp stableTimestamp) {
// Open all databases and repopulate the CollectionCatalog.
LOGV2(20276, "openCatalog: reopening all databases");
@@ -82,7 +80,7 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx,
str::stream()
<< "failed to get valid collection pointer for namespace " << collNss);
- if (previousCatalogState.minVisibleTimestampMap.count(collection->uuid()) > 0) {
+ if (minVisibleTimestampMap.count(collection->uuid()) > 0) {
// After rolling back to a stable timestamp T, the minimum visible timestamp for
// each collection must be reset to (at least) its value at T. Additionally, there
// cannot exist a minimum visible timestamp greater than lastApplied. This allows us
@@ -92,31 +90,14 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx,
// bound the minimum visible timestamp (where necessary) to the stable timestamp.
// The benefit of fine grained tracking is assumed to be low-value compared to the
// cost/effort.
- auto minVisible = std::min(
- stableTimestamp,
- previousCatalogState.minVisibleTimestampMap.find(collection->uuid())->second);
+ auto minVisible = std::min(stableTimestamp,
+ minVisibleTimestampMap.find(collection->uuid())->second);
auto writableCollection =
catalogWriter.get()->lookupCollectionByUUIDForMetadataWrite(opCtx,
collection->uuid());
writableCollection->setMinimumVisibleSnapshot(minVisible);
}
- if (collection->getTimeseriesOptions()) {
- bool extendedRangeSetting;
- if (auto it = previousCatalogState.requiresTimestampExtendedRangeSupportMap.find(
- collection->uuid());
- it != previousCatalogState.requiresTimestampExtendedRangeSupportMap.end()) {
- extendedRangeSetting = it->second;
- } else {
- extendedRangeSetting =
- timeseries::collectionMayRequireExtendedRangeSupport(opCtx, collection);
- }
-
- if (extendedRangeSetting) {
- collection->setRequiresTimeseriesExtendedRangeSupport(opCtx);
- }
- }
-
// If this is the oplog collection, re-establish the replication system's cached pointer
// to the oplog.
if (collNss.isOplog()) {
@@ -133,25 +114,28 @@ void reopenAllDatabasesAndReloadCollectionCatalog(OperationContext* opCtx,
// Opening CollectionCatalog: The collection catalog is now in sync with the storage engine
// catalog. Clear the pre-closing state.
- CollectionCatalog::write(opCtx, [](CollectionCatalog& catalog) { catalog.onOpenCatalog(); });
+ CollectionCatalog::write(opCtx,
+ [&](CollectionCatalog& catalog) { catalog.onOpenCatalog(opCtx); });
opCtx->getServiceContext()->incrementCatalogGeneration();
LOGV2(20278, "openCatalog: finished reloading collection catalog");
}
} // namespace
-PreviousCatalogState closeCatalog(OperationContext* opCtx) {
+MinVisibleTimestampMap closeCatalog(OperationContext* opCtx) {
invariant(opCtx->lockState()->isW());
IndexBuildsCoordinator::get(opCtx)->assertNoIndexBuildInProgress();
- PreviousCatalogState previousCatalogState;
+ MinVisibleTimestampMap minVisibleTimestampMap;
std::vector<TenantDatabaseName> allDbs =
opCtx->getServiceContext()->getStorageEngine()->listDatabases();
auto databaseHolder = DatabaseHolder::get(opCtx);
auto catalog = CollectionCatalog::get(opCtx);
for (auto&& tenantDbName : allDbs) {
- for (auto&& coll : catalog->range(tenantDbName)) {
+ for (auto collIt = catalog->begin(opCtx, tenantDbName); collIt != catalog->end(opCtx);
+ ++collIt) {
+ auto coll = *collIt;
if (!coll) {
break;
}
@@ -166,12 +150,7 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) {
"coll_ns"_attr = coll->ns(),
"uuid"_attr = coll->uuid(),
"minVisible"_attr = minVisible);
- previousCatalogState.minVisibleTimestampMap[coll->uuid()] = *minVisible;
- }
-
- if (coll->getTimeseriesOptions()) {
- previousCatalogState.requiresTimestampExtendedRangeSupportMap[coll->uuid()] =
- coll->getRequiresTimeseriesExtendedRangeSupport();
+ minVisibleTimestampMap[coll->uuid()] = *minVisible;
}
}
}
@@ -179,13 +158,14 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) {
// Need to mark the CollectionCatalog as open if we our closeAll fails, dismissed if successful.
ScopeGuard reopenOnFailure([opCtx] {
CollectionCatalog::write(opCtx,
- [](CollectionCatalog& catalog) { catalog.onOpenCatalog(); });
+ [&](CollectionCatalog& catalog) { catalog.onOpenCatalog(opCtx); });
});
// Closing CollectionCatalog: only lookupNSSByUUID will fall back to using pre-closing state to
// allow authorization for currently unknown UUIDs. This is needed because authorization needs
// to work before acquiring locks, and might otherwise spuriously regard a UUID as unknown
// while reloading the catalog.
- CollectionCatalog::write(opCtx, [](CollectionCatalog& catalog) { catalog.onCloseCatalog(); });
+ CollectionCatalog::write(opCtx,
+ [&](CollectionCatalog& catalog) { catalog.onCloseCatalog(opCtx); });
LOGV2_DEBUG(20270, 1, "closeCatalog: closing collection catalog");
@@ -197,16 +177,12 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx) {
LOGV2(20272, "closeCatalog: closing storage engine catalog");
opCtx->getServiceContext()->getStorageEngine()->closeCatalog(opCtx);
- // Reset the stats counter for extended range time-series collections. This is maintained
- // outside the catalog itself.
- catalog_stats::requiresTimeseriesExtendedRangeSupport.store(0);
-
reopenOnFailure.dismiss();
- return previousCatalogState;
+ return minVisibleTimestampMap;
}
void openCatalog(OperationContext* opCtx,
- const PreviousCatalogState& previousCatalogState,
+ const MinVisibleTimestampMap& minVisibleTimestampMap,
Timestamp stableTimestamp) {
invariant(opCtx->lockState()->isW());
@@ -278,7 +254,7 @@ void openCatalog(OperationContext* opCtx,
opCtx, reconcileResult.indexBuildsToRestart, reconcileResult.indexBuildsToResume);
reopenAllDatabasesAndReloadCollectionCatalog(
- opCtx, storageEngine, previousCatalogState, stableTimestamp);
+ opCtx, storageEngine, minVisibleTimestampMap, stableTimestamp);
}
diff --git a/src/mongo/db/catalog/catalog_control.h b/src/mongo/db/catalog/catalog_control.h
index f7bc3fbbdba..182f073c307 100644
--- a/src/mongo/db/catalog/catalog_control.h
+++ b/src/mongo/db/catalog/catalog_control.h
@@ -36,11 +36,6 @@ namespace catalog {
using MinVisibleTimestamp = Timestamp;
using MinVisibleTimestampMap = std::map<UUID, MinVisibleTimestamp>;
-using RequiresTimestampExtendedRangeSupportMap = std::map<UUID, bool>;
-struct PreviousCatalogState {
- MinVisibleTimestampMap minVisibleTimestampMap;
- RequiresTimestampExtendedRangeSupportMap requiresTimestampExtendedRangeSupportMap;
-};
/**
* Closes the catalog, destroying all associated in-memory data structures for all databases. After
@@ -48,7 +43,7 @@ struct PreviousCatalogState {
*
* Must be called with the global lock acquired in exclusive mode.
*/
-PreviousCatalogState closeCatalog(OperationContext* opCtx);
+MinVisibleTimestampMap closeCatalog(OperationContext* opCtx);
/**
* Restores the catalog and all in-memory state after a call to closeCatalog().
@@ -56,7 +51,7 @@ PreviousCatalogState closeCatalog(OperationContext* opCtx);
* Must be called with the global lock acquired in exclusive mode.
*/
void openCatalog(OperationContext* opCtx,
- const PreviousCatalogState& catalogState,
+ const MinVisibleTimestampMap& catalogState,
Timestamp stableTimestamp);
/**
diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 901c69450ee..4eca3df6a9a 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -75,8 +75,8 @@ TEST_F(CatalogControlTest, CloseAndOpenCatalog) {
ServiceContext::UniqueOperationContext opCtx = cc().makeOperationContext();
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- auto previousState = catalog::closeCatalog(opCtx.get());
- ASSERT_EQUALS(0U, previousState.minVisibleTimestampMap.size());
+ auto map = catalog::closeCatalog(opCtx.get());
+ ASSERT_EQUALS(0U, map.size());
catalog::openCatalog(opCtx.get(), {}, Timestamp());
}
diff --git a/src/mongo/db/catalog/catalog_stats.cpp b/src/mongo/db/catalog/catalog_stats.cpp
index d346e67f971..ea7b4ee2276 100644
--- a/src/mongo/db/catalog/catalog_stats.cpp
+++ b/src/mongo/db/catalog/catalog_stats.cpp
@@ -29,7 +29,7 @@
#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
-#include "mongo/db/catalog/catalog_stats.h"
+#include "mongo/platform/basic.h"
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/database_holder.h"
@@ -37,10 +37,7 @@
#include "mongo/db/db_raii.h"
#include "mongo/logv2/log.h"
-namespace mongo::catalog_stats {
-
-// Number of time-series collections requiring extended range support
-AtomicWord<int> requiresTimeseriesExtendedRangeSupport;
+namespace mongo {
namespace {
class CatalogStatsSSS : public ServerStatusSection {
@@ -61,7 +58,6 @@ public:
int timeseries = 0;
int internalCollections = 0;
int internalViews = 0;
- int timeseriesExtendedRange = 0;
void toBson(BSONObjBuilder* builder) const {
builder->append("collections", collections);
@@ -71,9 +67,6 @@ public:
builder->append("views", views);
builder->append("internalCollections", internalCollections);
builder->append("internalViews", internalViews);
- if (timeseriesExtendedRange > 0) {
- builder->append("timeseriesExtendedRange", timeseriesExtendedRange);
- }
}
};
@@ -87,7 +80,6 @@ public:
stats.capped = catalogStats.userCapped;
stats.clustered = catalogStats.userClustered;
stats.internalCollections = catalogStats.internal;
- stats.timeseriesExtendedRange = requiresTimeseriesExtendedRangeSupport.load();
const auto viewCatalogDbNames = catalog->getViewCatalogDbNames(opCtx);
for (const auto& tenantDbName : viewCatalogDbNames) {
@@ -114,4 +106,4 @@ public:
} catalogStatsSSS;
} // namespace
-} // namespace mongo::catalog_stats
+} // namespace mongo
diff --git a/src/mongo/db/catalog/catalog_stats.h b/src/mongo/db/catalog/catalog_stats.h
deleted file mode 100644
index 54cc04f0bb6..00000000000
--- a/src/mongo/db/catalog/catalog_stats.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/platform/atomic_word.h"
-
-namespace mongo::catalog_stats {
-
-extern AtomicWord<int> requiresTimeseriesExtendedRangeSupport;
-
-} // namespace mongo::catalog_stats
diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp
index 2c9f03053d0..8af0ba8efc9 100644
--- a/src/mongo/db/catalog/coll_mod.cpp
+++ b/src/mongo/db/catalog/coll_mod.cpp
@@ -33,7 +33,6 @@
#include "mongo/db/catalog/coll_mod.h"
-#include "mongo/db/stats/counters.h"
#include <boost/optional.hpp>
#include <memory>
@@ -45,7 +44,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_key_validate.h"
#include "mongo/db/coll_mod_gen.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop_failpoint_helpers.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -54,7 +53,6 @@
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/s/collection_sharding_state.h"
#include "mongo/db/s/database_sharding_state.h"
-#include "mongo/db/s/shard_key_index_util.h"
#include "mongo/db/server_options.h"
#include "mongo/db/service_context.h"
#include "mongo/db/storage/recovery_unit.h"
@@ -64,7 +62,6 @@
#include "mongo/db/views/view_catalog_helpers.h"
#include "mongo/idl/command_generic_argument.h"
#include "mongo/logv2/log.h"
-#include "mongo/s/grid.h"
#include "mongo/util/fail_point.h"
#include "mongo/util/version/releases.h"
#include "mongo/util/visit_helper.h"
@@ -112,13 +109,12 @@ struct ParsedCollModRequest {
boost::optional<Collection::Validator> collValidator;
boost::optional<ValidationActionEnum> collValidationAction;
boost::optional<ValidationLevelEnum> collValidationLevel;
- boost::optional<bool> recordPreImages;
+ bool recordPreImages = false;
boost::optional<ChangeStreamPreAndPostImagesOptions> changeStreamPreAndPostImagesOptions;
int numModifications = 0;
bool dryRun = false;
boost::optional<long long> cappedSize;
boost::optional<long long> cappedMax;
- boost::optional<bool> timeseriesBucketsMayHaveMixedSchemaData;
};
Status getNotSupportedOnViewError(StringData fieldName) {
@@ -254,8 +250,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
"TTL indexes are not supported for capped collections."};
}
if (auto status = index_key_validate::validateExpireAfterSeconds(
- *cmdIndex.getExpireAfterSeconds(),
- index_key_validate::ValidateExpireAfterSecondsMode::kSecondaryTTLIndex);
+ *cmdIndex.getExpireAfterSeconds());
!status.isOK()) {
return {ErrorCodes::InvalidOptions, status.reason()};
}
@@ -289,8 +284,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
}
} else {
std::vector<const IndexDescriptor*> indexes;
- coll->getIndexCatalog()->findIndexesByKeyPattern(
- opCtx, keyPattern, IndexCatalog::InclusionPolicy::kReady, &indexes);
+ coll->getIndexCatalog()->findIndexesByKeyPattern(opCtx, keyPattern, false, &indexes);
if (indexes.size() > 1) {
return {ErrorCodes::AmbiguousIndexKeyPattern,
@@ -343,7 +337,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
if (cmrIndex->idx->unique()) {
indexForOplog->setUnique(boost::none);
} else {
- // Disallow one-step unique conversion. The user has to set
+ // Disallow one-step unique convertion. The user has to set
// 'prepareUnique' to true first.
if (!cmrIndex->idx->prepareUnique()) {
return Status(ErrorCodes::InvalidOptions,
@@ -369,29 +363,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
return {ErrorCodes::BadValue, "can't hide _id index"};
}
- // If the index is not hidden and we are trying to hide it, check if it is possible
- // to drop the shard key index, so it could be possible to hide it.
- if (!cmrIndex->idx->hidden() && *cmdIndex.getHidden()) {
- if (auto catalogClient = Grid::get(opCtx)->catalogClient()) {
- try {
- auto shardedColl = catalogClient->getCollection(opCtx, nss);
-
- if (isLastNonHiddenRangedShardKeyIndex(
- opCtx,
- coll,
- coll->getIndexCatalog(),
- cmrIndex->idx->indexName(),
- shardedColl.getKeyPattern().toBSON())) {
- return {ErrorCodes::InvalidOptions,
- "Can't hide the only compatible index for this collection's "
- "shard key"};
- }
- } catch (ExceptionFor<ErrorCodes::NamespaceNotFound>&) {
- // The collection is unsharded or doesn't exist.
- }
- }
- }
-
// Hiding a hidden index or unhiding a visible index should be treated as a no-op.
if (cmrIndex->idx->hidden() == *cmdIndex.getHidden()) {
indexForOplog->setHidden(boost::none);
@@ -401,35 +372,12 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
}
if (cmdIndex.getPrepareUnique()) {
- // Check if prepareUnique is being set on a time-series collection.
- if (isTimeseries) {
- return {ErrorCodes::InvalidOptions,
- "cannot set 'prepareUnique' for indexes of a time-series collection."};
- }
parsed.numModifications++;
// Attempting to modify with the same value should be treated as a no-op.
if (cmrIndex->idx->prepareUnique() == *cmdIndex.getPrepareUnique() ||
cmrIndex->idx->unique()) {
indexForOplog->setPrepareUnique(boost::none);
} else {
- // Checks if the index key pattern conflicts with the shard key pattern.
- if (auto catalogClient = Grid::get(opCtx)->catalogClient()) {
- try {
- auto shardedColl = catalogClient->getCollection(opCtx, nss);
- const ShardKeyPattern shardKeyPattern(shardedColl.getKeyPattern());
- if (!shardKeyPattern.isIndexUniquenessCompatible(
- cmrIndex->idx->keyPattern())) {
- return {ErrorCodes::InvalidOptions,
- fmt::format(
- "cannot set 'prepareUnique' for index {} with shard key "
- "pattern {}",
- cmrIndex->idx->keyPattern().toString(),
- shardKeyPattern.toBSON().toString())};
- }
- } catch (ExceptionFor<ErrorCodes::NamespaceNotFound>&) {
- // The collection is unsharded or doesn't exist.
- }
- }
cmrIndex->indexPrepareUnique = cmdIndex.getPrepareUnique();
}
}
@@ -482,11 +430,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
validatorObj.getOwned(),
MatchExpressionParser::kDefaultSpecialFeatures,
maxFeatureCompatibilityVersion);
-
- // Increment counters to track the usage of schema validators.
- validatorCounters.incrementCounters(
- cmd.kCommandName, parsed.collValidator->validatorDoc, parsed.collValidator->isOK());
-
if (!parsed.collValidator->isOK()) {
return parsed.collValidator->getStatus();
}
@@ -590,9 +533,7 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
},
[&oplogEntryBuilder](std::int64_t value) {
oplogEntryBuilder.append(CollMod::kExpireAfterSecondsFieldName, value);
- return index_key_validate::validateExpireAfterSeconds(
- value,
- index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex);
+ return index_key_validate::validateExpireAfterSeconds(value);
},
},
*expireAfterSeconds);
@@ -611,17 +552,6 @@ StatusWith<std::pair<ParsedCollModRequest, BSONObj>> parseCollModRequest(Operati
timeseries->serialize(&subObjBuilder);
}
- if (auto mixedSchema = cmr.getTimeseriesBucketsMayHaveMixedSchemaData()) {
- if (!isTimeseries) {
- return getOnlySupportedOnTimeseriesError(
- CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName);
- }
-
- parsed.timeseriesBucketsMayHaveMixedSchemaData = mixedSchema;
- oplogEntryBuilder.append(CollMod::kTimeseriesBucketsMayHaveMixedSchemaDataFieldName,
- *mixedSchema);
- }
-
if (auto& dryRun = cmr.getDryRun()) {
parsed.dryRun = *dryRun;
// The dry run option should never be included in a collMod oplog entry.
@@ -666,8 +596,7 @@ void _setClusteredExpireAfterSeconds(
if (!oldExpireAfterSeconds) {
auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext());
opCtx->recoveryUnit()->onCommit([ttlCache, uuid = coll->uuid()](auto _) {
- ttlCache->registerTTLInfo(
- uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
+ ttlCache->registerTTLInfo(uuid, TTLCollectionCache::ClusteredId());
});
}
@@ -917,7 +846,7 @@ Status _collModInternal(OperationContext* opCtx,
cmrNew.recordPreImages = false;
}
- if (cmrNew.recordPreImages && *cmrNew.recordPreImages) {
+ if (cmrNew.recordPreImages) {
cmrNew.changeStreamPreAndPostImagesOptions =
ChangeStreamPreAndPostImagesOptions(false);
}
@@ -947,11 +876,6 @@ Status _collModInternal(OperationContext* opCtx,
*cmd.getExpireAfterSeconds());
}
- if (auto mixedSchema = cmrNew.timeseriesBucketsMayHaveMixedSchemaData) {
- coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(
- opCtx, mixedSchema);
- }
-
// Handle index modifications.
processCollModIndexRequest(
opCtx, &coll, cmrNew.indexRequest, &indexCollModInfo, result, mode);
@@ -969,9 +893,8 @@ Status _collModInternal(OperationContext* opCtx,
"Failed to set validationLevel");
}
- if (cmrNew.recordPreImages.has_value() &&
- *cmrNew.recordPreImages != oldCollOptions.recordPreImages) {
- coll.getWritableCollection(opCtx)->setRecordPreImages(opCtx, *cmrNew.recordPreImages);
+ if (cmrNew.recordPreImages != oldCollOptions.recordPreImages) {
+ coll.getWritableCollection(opCtx)->setRecordPreImages(opCtx, cmrNew.recordPreImages);
}
if (cmrNew.changeStreamPreAndPostImagesOptions.has_value() &&
@@ -991,9 +914,9 @@ Status _collModInternal(OperationContext* opCtx,
}
}
- // Fix any invalid index options for indexes belonging to this collection.
+ // Remove any invalid index options for indexes belonging to this collection.
std::vector<std::string> indexesWithInvalidOptions =
- coll.getWritableCollection(opCtx)->repairInvalidIndexOptions(opCtx);
+ coll.getWritableCollection(opCtx)->removeInvalidIndexOptions(opCtx);
for (const auto& indexWithInvalidOptions : indexesWithInvalidOptions) {
const IndexDescriptor* desc =
coll->getIndexCatalog()->findIndexByName(opCtx, indexWithInvalidOptions);
@@ -1007,24 +930,24 @@ Status _collModInternal(OperationContext* opCtx,
// (Generic FCV reference): TODO SERVER-60912: When kLastLTS is 6.0, remove this FCV-gated
// upgrade/downgrade code.
const auto currentVersion = serverGlobalParams.featureCompatibility.getVersion();
- if (coll->getTimeseriesOptions()) {
- if (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest) {
- // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest,
- // collMod is called as part of the upgrade process to add the
- // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series
- // collections that are missing the flag. This indicates that the time-series
- // collection existed in earlier server versions and may have mixed-schema data.
- coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx,
- true);
- } else if (currentVersion ==
- multiversion::GenericFCV::kDowngradingFromLatestToLastLTS) {
- // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called
- // as part of the downgrade process to remove the
- // 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag for time-series
- // collections that have the flag.
- coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(
- opCtx, boost::none);
- }
+ if (coll->getTimeseriesOptions() && !coll->getTimeseriesBucketsMayHaveMixedSchemaData() &&
+ (currentVersion == multiversion::GenericFCV::kUpgradingFromLastLTSToLatest ||
+ currentVersion == multiversion::GenericFCV::kLatest)) {
+ // (Generic FCV reference): While upgrading the FCV from kLastLTS to kLatest, collMod is
+ // called as part of the upgrade process to add the
+ // 'timeseriesBucketsMayHaveMixedSchemaData=true' catalog entry flag for time-series
+ // collections that are missing the flag. This indicates that the time-series collection
+ // existed in earlier server versions and may have mixed-schema data.
+ coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx,
+ true);
+ } else if (coll->getTimeseriesBucketsMayHaveMixedSchemaData() &&
+ (currentVersion == multiversion::GenericFCV::kDowngradingFromLatestToLastLTS ||
+ currentVersion == multiversion::GenericFCV::kLastLTS)) {
+ // (Generic FCV reference): While downgrading the FCV to kLastLTS, collMod is called as
+ // part of the downgrade process to remove the 'timeseriesBucketsMayHaveMixedSchemaData'
+ // catalog entry flag for time-series collections that have the flag.
+ coll.getWritableCollection(opCtx)->setTimeseriesBucketsMayHaveMixedSchemaData(
+ opCtx, boost::none);
}
// Only observe non-view collMods, as view operations are observed as operations on the
@@ -1040,39 +963,6 @@ Status _collModInternal(OperationContext* opCtx,
} // namespace
-bool isCollModIndexUniqueConversion(const CollModRequest& request) {
- auto index = request.getIndex();
- if (!index) {
- return false;
- }
- if (auto indexUnique = index->getUnique(); !indexUnique) {
- return false;
- }
- // Checks if the request is an actual unique conversion instead of a dry run.
- if (auto dryRun = request.getDryRun(); dryRun && *dryRun) {
- return false;
- }
- return true;
-}
-
-CollModRequest makeCollModDryRunRequest(const CollModRequest& request) {
- CollModRequest dryRunRequest;
- CollModIndex dryRunIndex;
- const auto& requestIndex = request.getIndex();
- dryRunIndex.setUnique(true);
- if (auto keyPattern = requestIndex->getKeyPattern()) {
- dryRunIndex.setKeyPattern(keyPattern);
- } else if (auto name = requestIndex->getName()) {
- dryRunIndex.setName(name);
- }
- if (auto uuid = request.getCollectionUUID()) {
- dryRunRequest.setCollectionUUID(uuid);
- }
- dryRunRequest.setIndex(dryRunIndex);
- dryRunRequest.setDryRun(true);
- return dryRunRequest;
-}
-
Status processCollModCommand(OperationContext* opCtx,
const NamespaceStringOrUUID& nsOrUUID,
const CollMod& cmd,
diff --git a/src/mongo/db/catalog/coll_mod.h b/src/mongo/db/catalog/coll_mod.h
index f2b0c1702d5..f08aca11444 100644
--- a/src/mongo/db/catalog/coll_mod.h
+++ b/src/mongo/db/catalog/coll_mod.h
@@ -48,18 +48,6 @@ class OperationContext;
void addCollectionUUIDs(OperationContext* opCtx);
/**
- * Checks if the collMod request is converting an index to unique.
- */
-bool isCollModIndexUniqueConversion(const CollModRequest& request);
-
-/**
- * Constructs a valid collMod dry-run request from the original request.
- * The 'dryRun' option can only be used with the index 'unique' option, so we assume 'request' must
- * have the 'unique' option. The function will also remove other options from the original request.
- */
-CollModRequest makeCollModDryRunRequest(const CollModRequest& request);
-
-/**
* Performs the collection modification described in "cmd" on the collection "ns".
*/
Status processCollModCommand(OperationContext* opCtx,
diff --git a/src/mongo/db/catalog/coll_mod_index.cpp b/src/mongo/db/catalog/coll_mod_index.cpp
index 2ffecf7f1e1..d56acc639f4 100644
--- a/src/mongo/db/catalog/coll_mod_index.cpp
+++ b/src/mongo/db/catalog/coll_mod_index.cpp
@@ -67,10 +67,8 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx,
// Do not refer to 'idx' within this commit handler as it may be be invalidated by
// IndexCatalog::refreshEntry().
opCtx->recoveryUnit()->onCommit(
- [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds](
- auto _) {
- ttlCache->registerTTLInfo(
- uuid, TTLCollectionCache::Info{indexName, /*isExpireAfterSecondsNaN=*/false});
+ [ttlCache, uuid = coll->uuid(), indexName = idx->indexName()](auto _) {
+ ttlCache->registerTTLInfo(uuid, indexName);
});
// Change the value of "expireAfterSeconds" on disk.
@@ -79,29 +77,6 @@ void _processCollModIndexRequestExpireAfterSeconds(OperationContext* opCtx,
return;
}
- // If the current `expireAfterSeconds` is NaN, it can never be equal to
- // 'indexExpireAfterSeconds'.
- if (oldExpireSecsElement.isNaN()) {
- // Setting *oldExpireSecs is mostly for informational purposes.
- // We could also use index_key_validate::kExpireAfterSecondsForInactiveTTLIndex but
- // 0 is more consistent with the previous safeNumberLong() behavior and avoids potential
- // showing the same value for the new and old values in the collMod response.
- *oldExpireSecs = 0;
-
- // Change the value of "expireAfterSeconds" on disk.
- autoColl->getWritableCollection(opCtx)->updateTTLSetting(
- opCtx, idx->indexName(), indexExpireAfterSeconds);
-
- // Keep the TTL information maintained by the TTLCollectionCache in sync so that we don't
- // try to fix up the TTL index during the next step-up.
- auto ttlCache = &TTLCollectionCache::get(opCtx->getServiceContext());
- const auto& coll = autoColl->getCollection();
- opCtx->recoveryUnit()->onCommit(
- [ttlCache, uuid = coll->uuid(), indexName = idx->indexName(), indexExpireAfterSeconds](
- auto _) { ttlCache->unsetTTLIndexExpireAfterSecondsNaN(uuid, indexName); });
- return;
- }
-
// This collection is already TTL. Compare the requested value against the existing setting
// before updating the catalog.
*oldExpireSecs = oldExpireSecsElement.safeNumberLong();
diff --git a/src/mongo/db/catalog/coll_mod_test.cpp b/src/mongo/db/catalog/coll_mod_test.cpp
deleted file mode 100644
index c6335aa1c67..00000000000
--- a/src/mongo/db/catalog/coll_mod_test.cpp
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Copyright (C) 2022-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/catalog/coll_mod.h"
-
-#include <boost/optional.hpp>
-
-#include "mongo/db/coll_mod_gen.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo {
-namespace {
-TEST(CollModOptionTest, isConvertingIndexToUnique) {
- IDLParserErrorContext ctx("collMod");
- auto requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}}");
- auto request = CollModRequest::parse(ctx, requestObj);
- ASSERT_TRUE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true, hidden: true}}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_TRUE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson(
- "{index: {keyPattern: {a: 1}, unique: true, hidden: true}, validationAction: 'warn'}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_TRUE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: true}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_FALSE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: false}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_TRUE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, prepareUnique: true}}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_FALSE(isCollModIndexUniqueConversion(request));
-
- requestObj = fromjson("{validationAction: 'warn'}");
- request = CollModRequest::parse(ctx, requestObj);
- ASSERT_FALSE(isCollModIndexUniqueConversion(request));
-}
-
-TEST(CollModOptionTest, makeDryRunRequest) {
- IDLParserErrorContext ctx("collMod");
- auto requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}}");
- auto request = CollModRequest::parse(ctx, requestObj);
- auto dryRunRequest = makeCollModDryRunRequest(request);
- ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}")));
- ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique());
- ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun());
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true, hidden: true}}");
- request = CollModRequest::parse(ctx, requestObj);
- dryRunRequest = makeCollModDryRunRequest(request);
- ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}")));
- ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique());
- ASSERT_FALSE(dryRunRequest.getIndex()->getHidden());
- ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun());
-
- requestObj = fromjson(
- "{index: {keyPattern: {a: 1}, unique: true, hidden: true}, validationAction: 'warn'}");
- request = CollModRequest::parse(ctx, requestObj);
- dryRunRequest = makeCollModDryRunRequest(request);
- ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}")));
- ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique());
- ASSERT_FALSE(dryRunRequest.getIndex()->getHidden());
- ASSERT_FALSE(dryRunRequest.getValidationAction());
- ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun());
-
- requestObj = fromjson("{index: {keyPattern: {a: 1}, unique: true}, dryRun: false}");
- request = CollModRequest::parse(ctx, requestObj);
- dryRunRequest = makeCollModDryRunRequest(request);
- ASSERT_TRUE(dryRunRequest.getIndex()->getKeyPattern()->binaryEqual(fromjson("{a: 1}")));
- ASSERT_TRUE(dryRunRequest.getIndex()->getUnique() && *dryRunRequest.getIndex()->getUnique());
- ASSERT_TRUE(dryRunRequest.getDryRun() && *dryRunRequest.getDryRun());
-}
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h
index 9be52a11a11..425b02dc9d5 100644
--- a/src/mongo/db/catalog/collection.h
+++ b/src/mongo/db/catalog/collection.h
@@ -97,9 +97,6 @@ struct CollectionUpdateArgs {
bool preImageRecordingEnabledForCollection = false;
bool changeStreamPreAndPostImagesEnabledForCollection = false;
- // Set if the diff insert operation needs to check for the field's existence.
- bool mustCheckExistenceForInsertOperations = false;
-
// Set if OpTimes were reserved for the update ahead of time.
std::vector<OplogSlot> oplogSlots;
};
@@ -560,28 +557,11 @@ public:
boost::optional<bool> setting) = 0;
/**
- * Returns true if the passed in time-series bucket document contains mixed-schema data. Returns
- * a non-OK status if the bucket's min/max is malformed.
+ * Returns true if the passed in time-series bucket document contains mixed-schema data.
*/
- virtual StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData(
+ virtual bool doesTimeseriesBucketsDocContainMixedSchemaData(
const BSONObj& bucketsDoc) const = 0;
- /**
- * Returns true if the time-series collection may have dates outside the standard range (roughly
- * 1970-2038). The value may be updated in the background by another thread between calls, even
- * if the caller holds a lock on the collection. The value may only transition from false to
- * true.
- */
- virtual bool getRequiresTimeseriesExtendedRangeSupport() const = 0;
-
- /**
- * Sets the in-memory flag for this collection. This value can be retrieved by
- * 'getRequiresTimeseriesExtendedRangeSupport'.
- *
- * Throws if this is not a time-series collection.
- */
- virtual void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const = 0;
-
/*
* Returns true if this collection is clustered. That is, its RecordIds store the value of the
* cluster key. If the collection is clustered on _id, there is no separate _id index.
@@ -640,10 +620,10 @@ public:
bool prepareUnique) = 0;
/**
- * Repairs invalid index options on all indexes in this collection. Returns a list of
- * index names that were repaired.
+ * Removes invalid index options on all indexes in this collection. Returns a list of index
+ * names that contained invalid index options.
*/
- virtual std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) = 0;
+ virtual std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) = 0;
/**
* Updates the 'temp' setting for this collection.
diff --git a/src/mongo/db/catalog/collection_catalog.cpp b/src/mongo/db/catalog/collection_catalog.cpp
index cdaef1c8e55..4e5df27b592 100644
--- a/src/mongo/db/catalog/collection_catalog.cpp
+++ b/src/mongo/db/catalog/collection_catalog.cpp
@@ -52,13 +52,10 @@ const ServiceContext::Decoration<LatestCollectionCatalog> getCatalog =
ServiceContext::declareDecoration<LatestCollectionCatalog>();
std::shared_ptr<CollectionCatalog> batchedCatalogWriteInstance;
-absl::flat_hash_set<Collection*> batchedCatalogClonedCollections;
const OperationContext::Decoration<std::shared_ptr<const CollectionCatalog>> stashedCatalog =
OperationContext::declareDecoration<std::shared_ptr<const CollectionCatalog>>();
-const auto maxUuid = UUID::parse("FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF").getValue();
-const auto minUuid = UUID::parse("00000000-0000-0000-0000-000000000000").getValue();
} // namespace
class IgnoreExternalViewChangesForDatabase {
@@ -94,12 +91,12 @@ public:
static void setCollectionInCatalog(CollectionCatalog& catalog,
std::shared_ptr<Collection> collection) {
- catalog._collections = catalog._collections.set(collection->ns(), collection);
- catalog._catalog = catalog._catalog.set(collection->uuid(), collection);
+ catalog._collections[collection->ns()] = collection;
+ catalog._catalog[collection->uuid()] = collection;
// TODO SERVER-64608 Use tenantID from ns
auto dbIdPair = std::make_pair(TenantDatabaseName(boost::none, collection->ns().db()),
collection->uuid());
- catalog._orderedCollections = catalog._orderedCollections.set(dbIdPair, collection);
+ catalog._orderedCollections[dbIdPair] = collection;
}
PublishCatalogUpdates(OperationContext* opCtx,
@@ -133,7 +130,7 @@ public:
case UncommittedCatalogUpdates::Entry::Action::kRenamedCollection: {
writeJobs.push_back(
[& from = entry.nss, &to = entry.renameTo](CollectionCatalog& catalog) {
- catalog._collections = catalog._collections.erase(from);
+ catalog._collections.erase(from);
auto fromStr = from.ns();
auto toStr = to.ns();
@@ -154,9 +151,10 @@ public:
break;
}
case UncommittedCatalogUpdates::Entry::Action::kRecreatedCollection: {
- writeJobs.push_back([opCtx = _opCtx, collection = entry.collection](
- CollectionCatalog& catalog) {
- catalog.registerCollection(opCtx, std::move(collection));
+ writeJobs.push_back([opCtx = _opCtx,
+ collection = entry.collection,
+ uuid = *entry.externalUUID](CollectionCatalog& catalog) {
+ catalog.registerCollection(opCtx, uuid, std::move(collection));
});
// Fallthrough to the createCollection case to finish committing the collection.
}
@@ -223,65 +221,90 @@ private:
UncommittedCatalogUpdates& _uncommittedCatalogUpdates;
};
-CollectionCatalog::iterator::iterator(const TenantDatabaseName& tenantDbName,
- OrderedCollectionMap::iterator it,
- const OrderedCollectionMap& map)
- : _map{map}, _mapIter{it}, _end(_map.upper_bound(std::make_pair(tenantDbName, maxUuid))) {
- _skipUncommitted();
+CollectionCatalog::iterator::iterator(OperationContext* opCtx,
+ const TenantDatabaseName& tenantDbName,
+ const CollectionCatalog& catalog)
+ : _opCtx(opCtx), _tenantDbName(tenantDbName), _catalog(&catalog) {
+ auto minUuid = UUID::parse("00000000-0000-0000-0000-000000000000").getValue();
+
+ _mapIter = _catalog->_orderedCollections.lower_bound(std::make_pair(_tenantDbName, minUuid));
+
+ // Start with the first collection that is visible outside of its transaction.
+ while (!_exhausted() && !_mapIter->second->isCommitted()) {
+ _mapIter++;
+ }
+
+ if (!_exhausted()) {
+ _uuid = _mapIter->first.second;
+ }
}
+CollectionCatalog::iterator::iterator(OperationContext* opCtx,
+ std::map<std::pair<TenantDatabaseName, UUID>,
+ std::shared_ptr<Collection>>::const_iterator mapIter,
+ const CollectionCatalog& catalog)
+ : _opCtx(opCtx), _mapIter(mapIter), _catalog(&catalog) {}
+
CollectionCatalog::iterator::value_type CollectionCatalog::iterator::operator*() {
- if (_mapIter == _map.end()) {
- return nullptr;
+ if (_exhausted()) {
+ return CollectionPtr();
}
- return _mapIter->second.get();
+
+ return {
+ _opCtx, _mapIter->second.get(), LookupCollectionForYieldRestore(_mapIter->second->ns())};
+}
+
+Collection* CollectionCatalog::iterator::getWritableCollection(OperationContext* opCtx) {
+ return CollectionCatalog::get(opCtx)->lookupCollectionByUUIDForMetadataWrite(
+ opCtx, operator*()->uuid());
+}
+
+boost::optional<UUID> CollectionCatalog::iterator::uuid() {
+ return _uuid;
}
CollectionCatalog::iterator CollectionCatalog::iterator::operator++() {
- invariant(_mapIter != _map.end());
- invariant(_mapIter != _end);
_mapIter++;
- _skipUncommitted();
- return *this;
-}
-bool CollectionCatalog::iterator::operator==(const iterator& other) const {
- invariant(_map == other._map);
+ // Skip any collections that are not yet visible outside of their respective transactions.
+ while (!_exhausted() && !_mapIter->second->isCommitted()) {
+ _mapIter++;
+ }
- if (other._mapIter == other._map.end()) {
- return _mapIter == _map.end();
- } else if (_mapIter == _map.end()) {
- return other._mapIter == other._map.end();
+ if (_exhausted()) {
+ // If the iterator is at the end of the map or now points to an entry that does not
+ // correspond to the correct database.
+ _mapIter = _catalog->_orderedCollections.end();
+ _uuid = boost::none;
+ return *this;
}
- return _mapIter->first.second == other._mapIter->first.second;
+ _uuid = _mapIter->first.second;
+ return *this;
}
-bool CollectionCatalog::iterator::operator!=(const iterator& other) const {
- return !(*this == other);
+CollectionCatalog::iterator CollectionCatalog::iterator::operator++(int) {
+ auto oldPosition = *this;
+ ++(*this);
+ return oldPosition;
}
-void CollectionCatalog::iterator::_skipUncommitted() {
- // Advance to the next collection that is visible outside of its transaction.
- while (_mapIter != _end && !_mapIter->second->isCommitted()) {
- ++_mapIter;
+bool CollectionCatalog::iterator::operator==(const iterator& other) const {
+ invariant(_catalog == other._catalog);
+ if (other._mapIter == _catalog->_orderedCollections.end()) {
+ return _uuid == boost::none;
}
-}
-CollectionCatalog::Range::Range(const OrderedCollectionMap& map,
- const TenantDatabaseName& tenantDbName)
- : _map{map}, _tenantDbName{tenantDbName} {}
-
-CollectionCatalog::iterator CollectionCatalog::Range::begin() const {
- return {_tenantDbName, _map.lower_bound(std::make_pair(_tenantDbName, minUuid)), _map};
+ return _uuid == other._uuid;
}
-CollectionCatalog::iterator CollectionCatalog::Range::end() const {
- return {_tenantDbName, _map.upper_bound(std::make_pair(_tenantDbName, maxUuid)), _map};
+bool CollectionCatalog::iterator::operator!=(const iterator& other) const {
+ return !(*this == other);
}
-bool CollectionCatalog::Range::empty() const {
- return begin() == end();
+bool CollectionCatalog::iterator::_exhausted() {
+ return _mapIter == _catalog->_orderedCollections.end() ||
+ _mapIter->first.first != _tenantDbName;
}
std::shared_ptr<const CollectionCatalog> CollectionCatalog::get(ServiceContext* svcCtx) {
@@ -435,13 +458,6 @@ void CollectionCatalog::write(ServiceContext* svcCtx, CatalogWriteFn job) {
void CollectionCatalog::write(OperationContext* opCtx,
std::function<void(CollectionCatalog&)> job) {
- // Calling the writer must be done with the GlobalLock held. Otherwise we risk having the
- // BatchedCollectionCatalogWriter and this caller concurrently modifying the catalog. This is
- // because normal operations calling this will all be serialized, but
- // BatchedCollectionCatalogWriter skips this mechanism as it knows it is the sole user of the
- // server by holding a Global MODE_X lock.
- invariant(opCtx->lockState()->isNoop() || opCtx->lockState()->isLocked());
-
// If global MODE_X lock are held we can re-use a cloned CollectionCatalog instance when
// 'batchedCatalogWriteInstance' is set. Make sure we are the one holding the write lock.
if (batchedCatalogWriteInstance) {
@@ -453,33 +469,25 @@ void CollectionCatalog::write(OperationContext* opCtx,
write(opCtx->getServiceContext(), std::move(job));
}
-Status CollectionCatalog::createView(OperationContext* opCtx,
- const NamespaceString& viewName,
- const NamespaceString& viewOn,
- const BSONArray& pipeline,
- const BSONObj& collation,
- const ViewsForDatabase::PipelineValidatorFn& pipelineValidator,
- const ViewUpsertMode insertViewMode) const {
- // A view document direct write can occur via the oplog application path, which may only hold a
- // lock on the collection being updated (the database views collection).
- invariant(insertViewMode == ViewUpsertMode::kAlreadyDurableView ||
- opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX));
+Status CollectionCatalog::createView(
+ OperationContext* opCtx,
+ const NamespaceString& viewName,
+ const NamespaceString& viewOn,
+ const BSONArray& pipeline,
+ const BSONObj& collation,
+ const ViewsForDatabase::PipelineValidatorFn& pipelineValidator) const {
+ invariant(opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX));
invariant(opCtx->lockState()->isCollectionLockedForMode(
NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X));
- invariant(_viewsForDatabase.find(viewName.db()));
+ invariant(_viewsForDatabase.contains(viewName.db()));
const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db());
- auto& uncommittedCatalogUpdates = UncommittedCatalogUpdates::get(opCtx);
- if (uncommittedCatalogUpdates.shouldIgnoreExternalViewChanges(viewName.db())) {
- return Status::OK();
- }
-
if (viewName.db() != viewOn.db())
return Status(ErrorCodes::BadValue,
"View must be created on a view or collection in the same database");
- if (viewsForDb.lookup(viewName) || _collections.find(viewName))
+ if (viewsForDb.lookup(viewName) || _collections.contains(viewName))
return Status(ErrorCodes::NamespaceExists, "Namespace already exists");
if (!NamespaceString::validCollectionName(viewOn.coll()))
@@ -500,8 +508,7 @@ Status CollectionCatalog::createView(OperationContext* opCtx,
pipeline,
pipelineValidator,
std::move(collator.getValue()),
- ViewsForDatabase{viewsForDb},
- insertViewMode);
+ ViewsForDatabase{viewsForDb});
}
return result;
@@ -517,7 +524,7 @@ Status CollectionCatalog::modifyView(
invariant(opCtx->lockState()->isCollectionLockedForMode(
NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X));
- invariant(_viewsForDatabase.find(viewName.db()));
+ invariant(_viewsForDatabase.contains(viewName.db()));
const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db());
if (viewName.db() != viewOn.db())
@@ -543,8 +550,7 @@ Status CollectionCatalog::modifyView(
pipeline,
pipelineValidator,
CollatorInterface::cloneCollator(viewPtr->defaultCollator()),
- ViewsForDatabase{viewsForDb},
- ViewUpsertMode::kUpdateView);
+ ViewsForDatabase{viewsForDb});
}
return result;
@@ -555,12 +561,12 @@ Status CollectionCatalog::dropView(OperationContext* opCtx, const NamespaceStrin
invariant(opCtx->lockState()->isCollectionLockedForMode(
NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X));
- invariant(_viewsForDatabase.find(viewName.db()));
+ invariant(_viewsForDatabase.contains(viewName.db()));
const ViewsForDatabase& viewsForDb = *_getViewsForDatabase(opCtx, viewName.db());
viewsForDb.requireValidCatalog();
// Make sure the view exists before proceeding.
- if (!viewsForDb.lookup(viewName)) {
+ if (auto viewPtr = viewsForDb.lookup(viewName); !viewPtr) {
return {ErrorCodes::NamespaceNotFound,
str::stream() << "cannot drop missing view: " << viewName.ns()};
}
@@ -604,10 +610,9 @@ Status CollectionCatalog::reloadViews(OperationContext* opCtx, StringData dbName
// Create a copy of the ViewsForDatabase instance to modify it. Reset the views for this
// database, but preserve the DurableViewCatalog pointer.
- const ViewsForDatabase* viewsForDbPtr = _viewsForDatabase.find(dbName);
- invariant(viewsForDbPtr);
- ViewsForDatabase viewsForDb = *viewsForDbPtr;
-
+ auto it = _viewsForDatabase.find(dbName);
+ invariant(it != _viewsForDatabase.end());
+ ViewsForDatabase viewsForDb{it->second.durable};
viewsForDb.valid = false;
viewsForDb.viewGraphNeedsRefresh = true;
viewsForDb.viewMap.clear();
@@ -672,29 +677,28 @@ void CollectionCatalog::onOpenDatabase(OperationContext* opCtx,
invariant(opCtx->lockState()->isDbLockedForMode(dbName, MODE_IS));
uassert(ErrorCodes::AlreadyInitialized,
str::stream() << "Database " << dbName << " is already initialized",
- !_viewsForDatabase.find(dbName));
+ _viewsForDatabase.find(dbName) == _viewsForDatabase.end());
- _viewsForDatabase = _viewsForDatabase.set(dbName.toString(), std::move(viewsForDb));
+ _viewsForDatabase[dbName] = std::move(viewsForDb);
}
void CollectionCatalog::onCloseDatabase(OperationContext* opCtx, TenantDatabaseName tenantDbName) {
invariant(opCtx->lockState()->isDbLockedForMode(tenantDbName.dbName(), MODE_X));
auto rid = ResourceId(RESOURCE_DATABASE, tenantDbName.dbName());
removeResource(rid, tenantDbName.dbName());
- _viewsForDatabase = _viewsForDatabase.erase(tenantDbName.dbName());
+ _viewsForDatabase.erase(tenantDbName.dbName());
}
-void CollectionCatalog::onCloseCatalog() {
- if (_shadowCatalog) {
- return;
- }
-
+void CollectionCatalog::onCloseCatalog(OperationContext* opCtx) {
+ invariant(opCtx->lockState()->isW());
+ invariant(!_shadowCatalog);
_shadowCatalog.emplace();
for (auto& entry : _catalog)
- _shadowCatalog = _shadowCatalog->insert({entry.first, entry.second->ns()});
+ _shadowCatalog->insert({entry.first, entry.second->ns()});
}
-void CollectionCatalog::onOpenCatalog() {
+void CollectionCatalog::onOpenCatalog(OperationContext* opCtx) {
+ invariant(opCtx->lockState()->isW());
invariant(_shadowCatalog);
_shadowCatalog.reset();
++_epoch;
@@ -704,10 +708,6 @@ uint64_t CollectionCatalog::getEpoch() const {
return _epoch;
}
-CollectionCatalog::Range CollectionCatalog::range(const TenantDatabaseName& tenantDbName) const {
- return {_orderedCollections, tenantDbName};
-}
-
std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByUUIDForRead(
OperationContext* opCtx, const UUID& uuid) const {
auto [found, uncommittedColl, newColl] =
@@ -761,7 +761,6 @@ Collection* CollectionCatalog::lookupCollectionByUUIDForMetadataWrite(OperationC
// on the thread doing the batch write and it would trigger the regular path where we do a
// copy-on-write on the catalog when committing.
if (_isCatalogBatchWriter()) {
- batchedCatalogClonedCollections.emplace(cloned.get());
PublishCatalogUpdates::setCollectionInCatalog(*batchedCatalogWriteInstance,
std::move(cloned));
return ptr;
@@ -795,8 +794,8 @@ bool CollectionCatalog::isCollectionAwaitingVisibility(UUID uuid) const {
}
std::shared_ptr<Collection> CollectionCatalog::_lookupCollectionByUUID(UUID uuid) const {
- const std::shared_ptr<Collection>* coll = _catalog.find(uuid);
- return coll ? *coll : nullptr;
+ auto foundIt = _catalog.find(uuid);
+ return foundIt == _catalog.end() ? nullptr : foundIt->second;
}
std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByNamespaceForRead(
@@ -813,8 +812,8 @@ std::shared_ptr<const Collection> CollectionCatalog::lookupCollectionByNamespace
return nullptr;
}
- const std::shared_ptr<Collection>* collPtr = _collections.find(nss);
- auto coll = collPtr ? *collPtr : nullptr;
+ auto it = _collections.find(nss);
+ auto coll = (it == _collections.end() ? nullptr : it->second);
return (coll && coll->isCommitted()) ? coll : nullptr;
}
@@ -844,8 +843,8 @@ Collection* CollectionCatalog::lookupCollectionByNamespaceForMetadataWrite(
return nullptr;
}
- const std::shared_ptr<Collection>* collPtr = _collections.find(nss);
- auto coll = collPtr ? *collPtr : nullptr;
+ auto it = _collections.find(nss);
+ auto coll = (it == _collections.end() ? nullptr : it->second);
if (!coll || !coll->isCommitted())
return nullptr;
@@ -865,7 +864,6 @@ Collection* CollectionCatalog::lookupCollectionByNamespaceForMetadataWrite(
// on the thread doing the batch write and it would trigger the regular path where we do a
// copy-on-write on the catalog when committing.
if (_isCatalogBatchWriter()) {
- batchedCatalogClonedCollections.emplace(cloned.get());
PublishCatalogUpdates::setCollectionInCatalog(*batchedCatalogWriteInstance,
std::move(cloned));
return ptr;
@@ -892,8 +890,8 @@ CollectionPtr CollectionCatalog::lookupCollectionByNamespace(OperationContext* o
return nullptr;
}
- const std::shared_ptr<Collection>* collPtr = _collections.find(nss);
- auto coll = collPtr ? *collPtr : nullptr;
+ auto it = _collections.find(nss);
+ auto coll = (it == _collections.end() ? nullptr : it->second);
return (coll && coll->isCommitted())
? CollectionPtr(opCtx, coll.get(), LookupCollectionForYieldRestore(coll->ns()))
: nullptr;
@@ -911,21 +909,20 @@ boost::optional<NamespaceString> CollectionCatalog::lookupNSSByUUID(OperationCon
return boost::none;
}
- const std::shared_ptr<Collection>* collPtr = _catalog.find(uuid);
- if (collPtr) {
- auto coll = *collPtr;
- boost::optional<NamespaceString> ns = coll->ns();
- invariant(!ns.value().isEmpty());
- return coll->isCommitted() ? ns : boost::none;
+ auto foundIt = _catalog.find(uuid);
+ if (foundIt != _catalog.end()) {
+ boost::optional<NamespaceString> ns = foundIt->second->ns();
+ invariant(!ns.get().isEmpty());
+ return _collections.find(ns.get())->second->isCommitted() ? ns : boost::none;
}
// Only in the case that the catalog is closed and a UUID is currently unknown, resolve it
// using the pre-close state. This ensures that any tasks reloading the catalog can see their
// own updates.
if (_shadowCatalog) {
- auto* shadowIt = _shadowCatalog->find(uuid);
- if (shadowIt)
- return *shadowIt;
+ auto shadowIt = _shadowCatalog->find(uuid);
+ if (shadowIt != _shadowCatalog->end())
+ return shadowIt->second;
}
return boost::none;
}
@@ -941,11 +938,10 @@ boost::optional<UUID> CollectionCatalog::lookupUUIDByNSS(OperationContext* opCtx
return boost::none;
}
- const std::shared_ptr<Collection>* collPtr = _collections.find(nss);
- if (collPtr) {
- auto coll = *collPtr;
- const boost::optional<UUID>& uuid = coll->uuid();
- return coll->isCommitted() ? uuid : boost::none;
+ auto it = _collections.find(nss);
+ if (it != _collections.end()) {
+ const boost::optional<UUID>& uuid = it->second->uuid();
+ return it->second->isCommitted() ? uuid : boost::none;
}
return boost::none;
}
@@ -1090,34 +1086,23 @@ std::vector<TenantDatabaseName> CollectionCatalog::getAllDbNames() const {
return ret;
}
-void CollectionCatalog::setAllDatabaseProfileFilters(std::shared_ptr<ProfileFilter> filter) {
- auto dbProfileSettingsWriter = _databaseProfileSettings.transient();
- for (const auto& [dbName, settings] : _databaseProfileSettings) {
- ProfileSettings clone = settings;
- clone.filter = filter;
- dbProfileSettingsWriter.set(dbName, std::move(clone));
- }
- _databaseProfileSettings = dbProfileSettingsWriter.persistent();
-}
-
void CollectionCatalog::setDatabaseProfileSettings(
StringData dbName, CollectionCatalog::ProfileSettings newProfileSettings) {
- _databaseProfileSettings =
- _databaseProfileSettings.set(dbName.toString(), std::move(newProfileSettings));
+ _databaseProfileSettings[dbName] = newProfileSettings;
}
CollectionCatalog::ProfileSettings CollectionCatalog::getDatabaseProfileSettings(
StringData dbName) const {
- const ProfileSettings* settings = _databaseProfileSettings.find(dbName);
- if (settings) {
- return *settings;
+ auto it = _databaseProfileSettings.find(dbName);
+ if (it != _databaseProfileSettings.end()) {
+ return it->second;
}
return {serverGlobalParams.defaultProfile, ProfileFilter::getDefault()};
}
void CollectionCatalog::clearDatabaseProfileSettings(StringData dbName) {
- _databaseProfileSettings = _databaseProfileSettings.erase(dbName.toString());
+ _databaseProfileSettings.erase(dbName);
}
CollectionCatalog::Stats CollectionCatalog::getStats() const {
@@ -1145,9 +1130,9 @@ CollectionCatalog::ViewCatalogSet CollectionCatalog::getViewCatalogDbNames(
}
void CollectionCatalog::registerCollection(OperationContext* opCtx,
+ const UUID& uuid,
std::shared_ptr<Collection> coll) {
auto nss = coll->ns();
- auto uuid = coll->uuid();
// TODO SERVER-64608 Use tenantId from nss
auto tenantDbName = TenantDatabaseName(boost::none, nss.db());
_ensureNamespaceDoesNotExist(opCtx, nss, NamespaceType::kAll);
@@ -1162,12 +1147,12 @@ void CollectionCatalog::registerCollection(OperationContext* opCtx,
auto dbIdPair = std::make_pair(tenantDbName, uuid);
// Make sure no entry related to this uuid.
- invariant(!_catalog.find(uuid));
+ invariant(_catalog.find(uuid) == _catalog.end());
invariant(_orderedCollections.find(dbIdPair) == _orderedCollections.end());
- _catalog = _catalog.set(uuid, coll);
- _collections = _collections.set(nss, coll);
- _orderedCollections = _orderedCollections.set(dbIdPair, coll);
+ _catalog[uuid] = coll;
+ _collections[nss] = coll;
+ _orderedCollections[dbIdPair] = coll;
if (!nss.isOnInternalDb() && !nss.isSystem()) {
_stats.userCollections += 1;
@@ -1193,7 +1178,7 @@ void CollectionCatalog::registerCollection(OperationContext* opCtx,
std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(OperationContext* opCtx,
const UUID& uuid) {
- invariant(_catalog.find(uuid));
+ invariant(_catalog.find(uuid) != _catalog.end());
auto coll = std::move(_catalog[uuid]);
auto ns = coll->ns();
@@ -1204,12 +1189,12 @@ std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(OperationCon
LOGV2_DEBUG(20281, 1, "Deregistering collection", logAttrs(ns), "uuid"_attr = uuid);
// Make sure collection object exists.
- invariant(_collections.find(ns));
+ invariant(_collections.find(ns) != _collections.end());
invariant(_orderedCollections.find(dbIdPair) != _orderedCollections.end());
- _orderedCollections = _orderedCollections.erase(dbIdPair);
- _collections = _collections.erase(ns);
- _catalog = _catalog.erase(uuid);
+ _orderedCollections.erase(dbIdPair);
+ _collections.erase(ns);
+ _catalog.erase(uuid);
if (!ns.isOnInternalDb() && !ns.isSystem()) {
_stats.userCollections -= 1;
@@ -1242,18 +1227,18 @@ void CollectionCatalog::registerUncommittedView(OperationContext* opCtx,
// namespaces here.
_ensureNamespaceDoesNotExist(opCtx, nss, NamespaceType::kCollection);
- _uncommittedViews = _uncommittedViews.insert(nss);
+ _uncommittedViews.emplace(nss);
}
void CollectionCatalog::deregisterUncommittedView(const NamespaceString& nss) {
- _uncommittedViews = _uncommittedViews.erase(nss);
+ _uncommittedViews.erase(nss);
}
void CollectionCatalog::_ensureNamespaceDoesNotExist(OperationContext* opCtx,
const NamespaceString& nss,
NamespaceType type) const {
auto existingCollection = _collections.find(nss);
- if (existingCollection) {
+ if (existingCollection != _collections.end()) {
LOGV2(5725001,
"Conflicted registering namespace, already have a collection with the same namespace",
"nss"_attr = nss);
@@ -1261,7 +1246,7 @@ void CollectionCatalog::_ensureNamespaceDoesNotExist(OperationContext* opCtx,
}
if (type == NamespaceType::kAll) {
- if (_uncommittedViews.find(nss)) {
+ if (_uncommittedViews.contains(nss)) {
LOGV2(5725002,
"Conflicted registering namespace, already have a view with the same namespace",
"nss"_attr = nss);
@@ -1289,25 +1274,26 @@ void CollectionCatalog::deregisterAllCollectionsAndViews() {
auto ns = entry.second->ns();
LOGV2_DEBUG(20283, 1, "Deregistering collection", logAttrs(ns), "uuid"_attr = uuid);
+
+ entry.second.reset();
}
- _collections = {};
- _orderedCollections = {};
- _catalog = {};
- _viewsForDatabase = {};
+ _collections.clear();
+ _orderedCollections.clear();
+ _catalog.clear();
+ _viewsForDatabase.clear();
_stats = {};
- _resourceInformation = {};
+ _resourceInformation.clear();
}
void CollectionCatalog::clearViews(OperationContext* opCtx, StringData dbName) const {
invariant(opCtx->lockState()->isCollectionLockedForMode(
NamespaceString(dbName, NamespaceString::kSystemDotViewsCollectionName), MODE_X));
- const ViewsForDatabase* viewsForDbPtr = _viewsForDatabase.find(dbName);
- invariant(viewsForDbPtr);
-
- ViewsForDatabase viewsForDb = *viewsForDbPtr;
+ auto it = _viewsForDatabase.find(dbName);
+ invariant(it != _viewsForDatabase.end());
+ ViewsForDatabase viewsForDb = it->second;
viewsForDb.viewMap.clear();
viewsForDb.viewGraph.clear();
@@ -1318,6 +1304,16 @@ void CollectionCatalog::clearViews(OperationContext* opCtx, StringData dbName) c
catalog._replaceViewsForDatabase(dbName, std::move(viewsForDb));
});
}
+
+CollectionCatalog::iterator CollectionCatalog::begin(OperationContext* opCtx,
+ const TenantDatabaseName& tenantDbName) const {
+ return iterator(opCtx, tenantDbName, *this);
+}
+
+CollectionCatalog::iterator CollectionCatalog::end(OperationContext* opCtx) const {
+ return iterator(opCtx, _orderedCollections.end(), *this);
+}
+
boost::optional<std::string> CollectionCatalog::lookupResourceName(const ResourceId& rid) const {
invariant(rid.getType() == RESOURCE_DATABASE || rid.getType() == RESOURCE_COLLECTION);
@@ -1325,6 +1321,7 @@ boost::optional<std::string> CollectionCatalog::lookupResourceName(const Resourc
if (search == _resourceInformation.end()) {
return boost::none;
}
+
const std::set<std::string>& namespaces = search->second;
// When there are multiple namespaces mapped to the same ResourceId, return boost::none as the
@@ -1344,14 +1341,12 @@ void CollectionCatalog::removeResource(const ResourceId& rid, const std::string&
return;
}
- std::set<std::string> namespaces = search->second;
+ std::set<std::string>& namespaces = search->second;
namespaces.erase(entry);
// Remove the map entry if this is the last namespace in the set for the ResourceId.
if (namespaces.size() == 0) {
- _resourceInformation = _resourceInformation.erase(search, rid);
- } else {
- _resourceInformation = _resourceInformation.set(rid, std::move(namespaces));
+ _resourceInformation.erase(search);
}
}
@@ -1361,17 +1356,16 @@ void CollectionCatalog::addResource(const ResourceId& rid, const std::string& en
auto search = _resourceInformation.find(rid);
if (search == _resourceInformation.end()) {
std::set<std::string> newSet = {entry};
- _resourceInformation = _resourceInformation.set(rid, std::move(newSet));
+ _resourceInformation.insert(std::make_pair(rid, newSet));
return;
}
- if (const auto& namespaces = search->second; namespaces.count(entry) > 0) {
+ std::set<std::string>& namespaces = search->second;
+ if (namespaces.count(entry) > 0) {
return;
}
- std::set<std::string> namespaces = search->second;
namespaces.insert(entry);
- _resourceInformation = _resourceInformation.set(rid, std::move(namespaces));
}
void CollectionCatalog::invariantHasExclusiveAccessToCollection(OperationContext* opCtx,
@@ -1391,15 +1385,15 @@ boost::optional<const ViewsForDatabase&> CollectionCatalog::_getViewsForDatabase
return uncommittedViews;
}
- const ViewsForDatabase* viewsForDb = _viewsForDatabase.find(dbName);
- if (!viewsForDb) {
+ auto it = _viewsForDatabase.find(dbName);
+ if (it == _viewsForDatabase.end()) {
return boost::none;
}
- return *viewsForDb;
+ return it->second;
}
void CollectionCatalog::_replaceViewsForDatabase(StringData dbName, ViewsForDatabase&& views) {
- _viewsForDatabase = _viewsForDatabase.set(dbName.toString(), std::move(views));
+ _viewsForDatabase[dbName] = std::move(views);
}
Status CollectionCatalog::_createOrUpdateView(
@@ -1409,19 +1403,15 @@ Status CollectionCatalog::_createOrUpdateView(
const BSONArray& pipeline,
const ViewsForDatabase::PipelineValidatorFn& pipelineValidator,
std::unique_ptr<CollatorInterface> collator,
- ViewsForDatabase&& viewsForDb,
- ViewUpsertMode insertViewMode) const {
- // A view document direct write can occur via the oplog application path, which may only hold a
- // lock on the collection being updated (the database views collection).
- invariant(insertViewMode == ViewUpsertMode::kAlreadyDurableView ||
- opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX));
+ ViewsForDatabase&& viewsForDb) const {
+ invariant(opCtx->lockState()->isCollectionLockedForMode(viewName, MODE_IX));
invariant(opCtx->lockState()->isCollectionLockedForMode(
NamespaceString(viewName.db(), NamespaceString::kSystemDotViewsCollectionName), MODE_X));
viewsForDb.requireValidCatalog();
- // Build the BSON definition for this view to be saved in the durable view catalog and/or to
- // insert in the viewMap. If the collation is empty, omit it from the definition altogether.
+ // Build the BSON definition for this view to be saved in the durable view catalog. If the
+ // collation is empty, omit it from the definition altogether.
BSONObjBuilder viewDefBuilder;
viewDefBuilder.append("_id", viewName.ns());
viewDefBuilder.append("viewOn", viewOn.coll());
@@ -1430,42 +1420,25 @@ Status CollectionCatalog::_createOrUpdateView(
viewDefBuilder.append("collation", collator->getSpec().toBSON());
}
- BSONObj viewDef = viewDefBuilder.obj();
BSONObj ownedPipeline = pipeline.getOwned();
- ViewDefinition view(
+ auto view = std::make_shared<ViewDefinition>(
viewName.db(), viewName.coll(), viewOn.coll(), ownedPipeline, std::move(collator));
- // If the view is already in the durable view catalog, we don't need to validate the graph. If
- // we need to update the durable view catalog, we need to check that the resulting dependency
- // graph is acyclic and within the maximum depth.
- const bool viewGraphNeedsValidation = insertViewMode != ViewUpsertMode::kAlreadyDurableView;
- Status graphStatus =
- viewsForDb.upsertIntoGraph(opCtx, view, pipelineValidator, viewGraphNeedsValidation);
+ // Check that the resulting dependency graph is acyclic and within the maximum depth.
+ Status graphStatus = viewsForDb.upsertIntoGraph(opCtx, *(view.get()), pipelineValidator);
if (!graphStatus.isOK()) {
return graphStatus;
}
- if (insertViewMode != ViewUpsertMode::kAlreadyDurableView) {
- viewsForDb.durable->upsert(opCtx, viewName, viewDef);
- }
+ viewsForDb.durable->upsert(opCtx, viewName, viewDefBuilder.obj());
+ viewsForDb.viewMap.clear();
viewsForDb.valid = false;
- auto res = [&] {
- switch (insertViewMode) {
- case ViewUpsertMode::kCreateView:
- case ViewUpsertMode::kAlreadyDurableView:
- return viewsForDb.insert(opCtx, viewDef);
- case ViewUpsertMode::kUpdateView:
- viewsForDb.viewMap.clear();
- viewsForDb.viewGraphNeedsRefresh = true;
- viewsForDb.stats = {};
-
- // Reload the view catalog with the changes applied.
- return viewsForDb.reload(opCtx);
- }
- MONGO_UNREACHABLE;
- }();
+ viewsForDb.viewGraphNeedsRefresh = true;
+ viewsForDb.stats = {};
+ // Reload the view catalog with the changes applied.
+ auto res = viewsForDb.reload(opCtx);
if (res.isOK()) {
auto& uncommittedCatalogUpdates = UncommittedCatalogUpdates::get(opCtx);
uncommittedCatalogUpdates.addView(opCtx, viewName);
@@ -1484,9 +1457,16 @@ bool CollectionCatalog::_isCatalogBatchWriter() const {
bool CollectionCatalog::_alreadyClonedForBatchedWriter(
const std::shared_ptr<Collection>& collection) const {
- // We may skip cloning the Collection instance if and only if have already cloned it for write
- // use in this batch writer.
- return _isCatalogBatchWriter() && batchedCatalogClonedCollections.contains(collection.get());
+ // We may skip cloning the Collection instance if and only if we are currently in a batched
+ // catalog write and all references to this Collection is owned by the cloned CollectionCatalog
+ // instance owned by the batch writer. i.e. the Collection is uniquely owned by the batch
+ // writer. When the batch writer initially clones the catalog, all collections will have a
+ // 'use_count' of at least kNumCollectionReferencesStored*2 (because there are at least 2
+ // catalog instances). To check for uniquely owned we need to check that the reference count is
+ // exactly kNumCollectionReferencesStored (owned by a single catalog) while also account for the
+ // instance that is extracted from the catalog and provided as a parameter to this function, we
+ // therefore need to add 1.
+ return _isCatalogBatchWriter() && collection.use_count() == kNumCollectionReferencesStored + 1;
}
CollectionCatalogStasher::CollectionCatalogStasher(OperationContext* opCtx)
@@ -1547,7 +1527,10 @@ const Collection* LookupCollectionForYieldRestore::operator()(OperationContext*
// state. After a query yields its locks, the replication state may have changed, invalidating
// our current choice of ReadSource. Using the same preconditions, change our ReadSource if
// necessary.
- SnapshotHelper::changeReadSourceIfNeeded(opCtx, collection->ns());
+ auto [newReadSource, _] = SnapshotHelper::shouldChangeReadSource(opCtx, collection->ns());
+ if (newReadSource) {
+ opCtx->recoveryUnit()->setTimestampReadSource(*newReadSource);
+ }
return collection.get();
}
@@ -1556,7 +1539,6 @@ BatchedCollectionCatalogWriter::BatchedCollectionCatalogWriter(OperationContext*
: _opCtx(opCtx) {
invariant(_opCtx->lockState()->isW());
invariant(!batchedCatalogWriteInstance);
- invariant(batchedCatalogClonedCollections.empty());
auto& storage = getCatalog(_opCtx->getServiceContext());
// hold onto base so if we need to delete it we can do it outside of the lock
@@ -1579,7 +1561,6 @@ BatchedCollectionCatalogWriter::~BatchedCollectionCatalogWriter() {
// Clear out batched pointer so no more attempts of batching are made
_batchedInstance = nullptr;
batchedCatalogWriteInstance = nullptr;
- batchedCatalogClonedCollections.clear();
}
} // namespace mongo
diff --git a/src/mongo/db/catalog/collection_catalog.h b/src/mongo/db/catalog/collection_catalog.h
index 01727543a4e..f47f94397f9 100644
--- a/src/mongo/db/catalog/collection_catalog.h
+++ b/src/mongo/db/catalog/collection_catalog.h
@@ -40,10 +40,6 @@
#include "mongo/db/tenant_database_name.h"
#include "mongo/db/views/view.h"
#include "mongo/stdx/unordered_map.h"
-#include "mongo/util/functional.h"
-#include "mongo/util/immutable/map.h"
-#include "mongo/util/immutable/unordered_map.h"
-#include "mongo/util/immutable/unordered_set.h"
#include "mongo/util/uuid.h"
namespace mongo {
@@ -53,23 +49,31 @@ class Database;
class CollectionCatalog {
friend class iterator;
- using OrderedCollectionMap =
- immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>;
public:
using CollectionInfoFn = std::function<bool(const CollectionPtr& collection)>;
using ViewIteratorCallback = std::function<bool(const ViewDefinition& view)>;
+ // Number of how many Collection references for a single Collection that is stored in the
+ // catalog. Used to determine whether there are external references (uniquely owned). Needs to
+ // be kept in sync with the data structures below.
+ static constexpr size_t kNumCollectionReferencesStored = 3;
class iterator {
public:
using value_type = CollectionPtr;
- iterator(const TenantDatabaseName& tenantDbName,
- OrderedCollectionMap::iterator it,
- const OrderedCollectionMap& catalog);
+ iterator(OperationContext* opCtx,
+ const TenantDatabaseName& tenantDbName,
+ const CollectionCatalog& catalog);
+ iterator(OperationContext* opCtx,
+ std::map<std::pair<TenantDatabaseName, UUID>,
+ std::shared_ptr<Collection>>::const_iterator mapIter,
+ const CollectionCatalog& catalog);
value_type operator*();
iterator operator++();
+ iterator operator++(int);
+ boost::optional<UUID> uuid();
Collection* getWritableCollection(OperationContext* opCtx);
@@ -81,30 +85,19 @@ public:
bool operator!=(const iterator& other) const;
private:
- void _skipUncommitted();
-
- const OrderedCollectionMap& _map;
- immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::iterator
- _mapIter;
- immutable::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::iterator
- _end;
- };
-
- class Range {
- public:
- Range(const OrderedCollectionMap&, const TenantDatabaseName& tenantDbName);
- iterator begin() const;
- iterator end() const;
- bool empty() const;
+ bool _exhausted();
- private:
- OrderedCollectionMap _map;
+ OperationContext* _opCtx;
TenantDatabaseName _tenantDbName;
+ boost::optional<UUID> _uuid;
+ std::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>::const_iterator
+ _mapIter;
+ const CollectionCatalog* _catalog;
};
struct ProfileSettings {
int level;
- std::shared_ptr<const ProfileFilter> filter; // nullable
+ std::shared_ptr<ProfileFilter> filter; // nullable
ProfileSettings(int level, std::shared_ptr<ProfileFilter> filter)
: level(level), filter(filter) {
@@ -122,19 +115,6 @@ public:
}
};
- enum class ViewUpsertMode {
- // Insert all data for that view into the view map, view graph, and durable view catalog.
- kCreateView,
-
- // Insert into the view map and view graph without reinserting the view into the durable
- // view catalog. Skip view graph validation.
- kAlreadyDurableView,
-
- // Reload the view map, insert into the view graph (flagging it as needing refresh), and
- // update the durable view catalog.
- kUpdateView,
- };
-
static std::shared_ptr<const CollectionCatalog> get(ServiceContext* svcCtx);
static std::shared_ptr<const CollectionCatalog> get(OperationContext* opCtx);
@@ -167,16 +147,14 @@ public:
*
* Must be in WriteUnitOfWork. View creation rolls back if the unit of work aborts.
*
- * Caller must ensure corresponding database exists. Expects db.system.views MODE_X lock and
- * view namespace MODE_IX lock (unless 'insertViewMode' is set to kAlreadyDurableView).
+ * Caller must ensure corresponding database exists.
*/
Status createView(OperationContext* opCtx,
const NamespaceString& viewName,
const NamespaceString& viewOn,
const BSONArray& pipeline,
const BSONObj& collation,
- const ViewsForDatabase::PipelineValidatorFn& pipelineValidator,
- ViewUpsertMode insertViewMode = ViewUpsertMode::kCreateView) const;
+ const ViewsForDatabase::PipelineValidatorFn& pipelineValidator) const;
/**
* Drop the view named 'viewName'.
@@ -257,7 +235,9 @@ public:
/**
* Register the collection with `uuid`.
*/
- void registerCollection(OperationContext* opCtx, std::shared_ptr<Collection> collection);
+ void registerCollection(OperationContext* opCtx,
+ const UUID& uuid,
+ std::shared_ptr<Collection> collection);
/**
* Deregister the collection.
@@ -426,11 +406,6 @@ public:
std::vector<TenantDatabaseName> getAllDbNames() const;
/**
- * Updates the profile filter on all databases with non-default settings.
- */
- void setAllDatabaseProfileFilters(std::shared_ptr<ProfileFilter> filter);
-
- /**
* Sets 'newProfileSettings' as the profiling settings for the database 'dbName'.
*/
void setDatabaseProfileSettings(StringData dbName, ProfileSettings newProfileSettings);
@@ -499,14 +474,14 @@ public:
*
* Must be called with the global lock acquired in exclusive mode.
*/
- void onCloseCatalog();
+ void onCloseCatalog(OperationContext* opCtx);
/**
* Puts the catalog back in open state, removing the pre-close state. See onCloseCatalog.
*
* Must be called with the global lock acquired in exclusive mode.
*/
- void onOpenCatalog();
+ void onOpenCatalog(OperationContext* opCtx);
/**
* The epoch is incremented whenever the catalog is closed and re-opened.
@@ -519,13 +494,8 @@ public:
*/
uint64_t getEpoch() const;
- /**
- * Provides an iterable range for the collections belonging to the specified database.
- *
- * Will not observe any updates made to the catalog after the creation of the 'Range'. The
- * 'Range' object just remain in scope for the duration of the iteration.
- */
- Range range(const TenantDatabaseName& tenantDbName) const;
+ iterator begin(OperationContext* opCtx, const TenantDatabaseName& tenantDbName) const;
+ iterator end(OperationContext* opCtx) const;
/**
* Lookup the name of a resource by its ResourceId. If there are multiple namespaces mapped to
@@ -578,8 +548,7 @@ private:
const BSONArray& pipeline,
const ViewsForDatabase::PipelineValidatorFn& pipelineValidator,
std::unique_ptr<CollatorInterface> collator,
- ViewsForDatabase&& viewsForDb,
- ViewUpsertMode insertViewMode) const;
+ ViewsForDatabase&& viewsForDb) const;
/**
* Returns true if this CollectionCatalog instance is part of an ongoing batched catalog write.
@@ -611,17 +580,15 @@ private:
* When present, indicates that the catalog is in closed state, and contains a map from UUID
* to pre-close NSS. See also onCloseCatalog.
*/
- boost::optional<immutable::unordered_map<UUID, NamespaceString, UUID::Hash>> _shadowCatalog;
+ boost::optional<mongo::stdx::unordered_map<UUID, NamespaceString, UUID::Hash>> _shadowCatalog;
- using CollectionCatalogMap =
- immutable::unordered_map<UUID, std::shared_ptr<Collection>, UUID::Hash>;
+ using CollectionCatalogMap = stdx::unordered_map<UUID, std::shared_ptr<Collection>, UUID::Hash>;
+ using OrderedCollectionMap =
+ std::map<std::pair<TenantDatabaseName, UUID>, std::shared_ptr<Collection>>;
using NamespaceCollectionMap =
- immutable::unordered_map<NamespaceString, std::shared_ptr<Collection>>;
- using UncommittedViewsSet = immutable::unordered_set<NamespaceString>;
- using DatabaseProfileSettingsMap =
- immutable::unordered_map<std::string, ProfileSettings, StringMapHasher, StringMapEq>;
- using ViewsForDatabaseMap =
- immutable::unordered_map<std::string, ViewsForDatabase, StringMapHasher, StringMapEq>;
+ stdx::unordered_map<NamespaceString, std::shared_ptr<Collection>>;
+ using UncommittedViewsSet = stdx::unordered_set<NamespaceString>;
+ using DatabaseProfileSettingsMap = StringMap<ProfileSettings>;
CollectionCatalogMap _catalog;
OrderedCollectionMap _orderedCollections; // Ordered by <tenantDbName, collUUID> pair
@@ -629,7 +596,7 @@ private:
UncommittedViewsSet _uncommittedViews;
// Map of database names to their corresponding views and other associated state.
- ViewsForDatabaseMap _viewsForDatabase;
+ StringMap<ViewsForDatabase> _viewsForDatabase;
// Incremented whenever the CollectionCatalog gets closed and reopened (onCloseCatalog and
// onOpenCatalog).
@@ -644,7 +611,7 @@ private:
uint64_t _epoch = 0;
// Mapping from ResourceId to a set of strings that contains collection and database namespaces.
- immutable::map<ResourceId, std::set<std::string>> _resourceInformation;
+ std::map<ResourceId, std::set<std::string>> _resourceInformation;
/**
* Contains non-default database profile settings. New collections, current collections and
diff --git a/src/mongo/db/catalog/collection_catalog_bm.cpp b/src/mongo/db/catalog/collection_catalog_bm.cpp
index 47b24320009..3ff0e3375bd 100644
--- a/src/mongo/db/catalog/collection_catalog_bm.cpp
+++ b/src/mongo/db/catalog/collection_catalog_bm.cpp
@@ -27,7 +27,6 @@
* it in the license file.
*/
-#include "mongo/db/tenant_database_name.h"
#include <benchmark/benchmark.h>
#include "mongo/db/catalog/collection_catalog.h"
@@ -79,7 +78,7 @@ void createCollections(OperationContext* opCtx, int numCollections) {
for (auto i = 0; i < numCollections; i++) {
const NamespaceString nss("collection_catalog_bm", std::to_string(i));
CollectionCatalog::write(opCtx, [&](CollectionCatalog& catalog) {
- catalog.registerCollection(opCtx, std::make_shared<CollectionMock>(nss));
+ catalog.registerCollection(opCtx, UUID::gen(), std::make_shared<CollectionMock>(nss));
});
}
}
@@ -93,8 +92,6 @@ void BM_CollectionCatalogWrite(benchmark::State& state) {
createCollections(opCtx.get(), state.range(0));
- Lock::GlobalLock lk{opCtx.get(), MODE_IX};
-
for (auto _ : state) {
benchmark::ClobberMemory();
CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) {});
@@ -117,124 +114,7 @@ void BM_CollectionCatalogWriteBatchedWithGlobalExclusiveLock(benchmark::State& s
}
}
-void BM_CollectionCatalogCreateDropCollection(benchmark::State& state) {
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
- Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
-
- createCollections(opCtx.get(), state.range(0));
-
- for (auto _ : state) {
- benchmark::ClobberMemory();
- CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) {
- const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0)));
- const UUID uuid = UUID::gen();
- catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(uuid, nss));
- catalog.deregisterCollection(opCtx.get(), uuid);
- });
- }
-}
-
-void BM_CollectionCatalogCreateNCollectionsBatched(benchmark::State& state) {
- for (auto _ : state) {
- benchmark::ClobberMemory();
-
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
-
- Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- BatchedCollectionCatalogWriter batched(opCtx.get());
-
- auto numCollections = state.range(0);
- for (auto i = 0; i < numCollections; i++) {
- const NamespaceString nss("collection_catalog_bm", std::to_string(i));
- CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) {
- catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(nss));
- });
- }
- }
-}
-
-void BM_CollectionCatalogCreateNCollections(benchmark::State& state) {
- for (auto _ : state) {
- benchmark::ClobberMemory();
-
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
- Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
-
- auto numCollections = state.range(0);
- for (auto i = 0; i < numCollections; i++) {
- const NamespaceString nss("collection_catalog_bm", std::to_string(i));
- CollectionCatalog::write(opCtx.get(), [&](CollectionCatalog& catalog) {
- catalog.registerCollection(opCtx.get(), std::make_shared<CollectionMock>(nss));
- });
- }
- }
-}
-
-void BM_CollectionCatalogLookupCollectionByNamespace(benchmark::State& state) {
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
-
- createCollections(opCtx.get(), state.range(0));
- const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0) / 2));
-
- for (auto _ : state) {
- benchmark::ClobberMemory();
- auto coll =
- CollectionCatalog::get(opCtx.get())->lookupCollectionByNamespace(opCtx.get(), nss);
- invariant(coll);
- }
-}
-
-void BM_CollectionCatalogLookupCollectionByUUID(benchmark::State& state) {
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
-
- createCollections(opCtx.get(), state.range(0));
- const NamespaceString nss("collection_catalog_bm", std::to_string(state.range(0) / 2));
- auto coll = CollectionCatalog::get(opCtx.get())->lookupCollectionByNamespace(opCtx.get(), nss);
- invariant(coll->ns() == nss);
- const UUID uuid = coll->uuid();
-
- for (auto _ : state) {
- benchmark::ClobberMemory();
- auto res = CollectionCatalog::get(opCtx.get())->lookupCollectionByUUID(opCtx.get(), uuid);
- invariant(res == coll);
- }
-}
-
-void BM_CollectionCatalogIterateCollections(benchmark::State& state) {
- auto serviceContext = setupServiceContext();
- ThreadClient threadClient(serviceContext);
- ServiceContext::UniqueOperationContext opCtx = threadClient->makeOperationContext();
-
- createCollections(opCtx.get(), state.range(0));
-
- for (auto _ : state) {
- benchmark::ClobberMemory();
- auto catalog = CollectionCatalog::get(opCtx.get());
- auto count = 0;
- for ([[maybe_unused]] auto&& coll :
- catalog->range(TenantDatabaseName(boost::none, "collection_catalog_bm"))) {
- benchmark::DoNotOptimize(count++);
- }
- }
-}
-
BENCHMARK(BM_CollectionCatalogWrite)->Ranges({{{1}, {100'000}}});
BENCHMARK(BM_CollectionCatalogWriteBatchedWithGlobalExclusiveLock)->Ranges({{{1}, {100'000}}});
-BENCHMARK(BM_CollectionCatalogCreateDropCollection)->Ranges({{{1}, {100'000}}});
-BENCHMARK(BM_CollectionCatalogCreateNCollectionsBatched)->Ranges({{{1}, {100'000}}});
-BENCHMARK(BM_CollectionCatalogCreateNCollections)->Ranges({{{1}, {32'768}}});
-BENCHMARK(BM_CollectionCatalogLookupCollectionByNamespace)->Ranges({{{1}, {100'000}}});
-BENCHMARK(BM_CollectionCatalogLookupCollectionByUUID)->Ranges({{{1}, {100'000}}});
-BENCHMARK(BM_CollectionCatalogIterateCollections)->Ranges({{{1}, {100'000}}});
} // namespace mongo
diff --git a/src/mongo/db/catalog/collection_catalog_helper.cpp b/src/mongo/db/catalog/collection_catalog_helper.cpp
index ed816f2e1cf..f6db8f69764 100644
--- a/src/mongo/db/catalog/collection_catalog_helper.cpp
+++ b/src/mongo/db/catalog/collection_catalog_helper.cpp
@@ -66,10 +66,11 @@ void forEachCollectionFromDb(OperationContext* opCtx,
CollectionCatalog::CollectionInfoFn predicate) {
auto catalogForIteration = CollectionCatalog::get(opCtx);
- size_t collectionCount = 0;
- for (auto&& coll : catalogForIteration->range(tenantDbName)) {
- auto uuid = coll->uuid();
+ for (auto collectionIt = catalogForIteration->begin(opCtx, tenantDbName);
+ collectionIt != catalogForIteration->end(opCtx);) {
+ auto uuid = collectionIt.uuid().get();
if (predicate && !catalogForIteration->checkIfCollectionSatisfiable(uuid, predicate)) {
+ ++collectionIt;
continue;
}
@@ -93,6 +94,10 @@ void forEachCollectionFromDb(OperationContext* opCtx,
clk.reset();
}
+ // Increment iterator before calling callback. This allows for collection deletion inside
+ // this callback even if we are in batched inplace mode.
+ ++collectionIt;
+
// The NamespaceString couldn't be resolved from the uuid, so the collection was dropped.
if (!collection)
continue;
@@ -100,14 +105,7 @@ void forEachCollectionFromDb(OperationContext* opCtx,
if (!callback(collection))
break;
- // This was a rough heuristic that was found that 400 collections would take 100
- // milliseconds with calling checkForInterrupt() (with freeStorage: 1).
- // We made the checkForInterrupt() occur after 200 collections to be conservative.
- if (!(collectionCount % 200)) {
- opCtx->checkForInterrupt();
- }
hangBeforeGettingNextCollection.pauseWhileSet();
- collectionCount += 1;
}
}
diff --git a/src/mongo/db/catalog/collection_catalog_test.cpp b/src/mongo/db/catalog/collection_catalog_test.cpp
index d05913ac7bf..e28bcd4ad28 100644
--- a/src/mongo/db/catalog/collection_catalog_test.cpp
+++ b/src/mongo/db/catalog/collection_catalog_test.cpp
@@ -72,7 +72,12 @@ public:
std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(colUUID, nss);
col = CollectionPtr(collection.get(), CollectionPtr::NoYieldTag{});
// Register dummy collection in catalog.
- catalog.registerCollection(opCtx.get(), collection);
+ catalog.registerCollection(opCtx.get(), colUUID, collection);
+
+ // Validate that kNumCollectionReferencesStored is correct, add one reference for the one we
+ // hold in this function.
+ ASSERT_EQUALS(collection.use_count(),
+ CollectionCatalog::kNumCollectionReferencesStored + 1);
}
protected:
@@ -93,14 +98,17 @@ public:
NamespaceString fooNss("foo", "coll" + std::to_string(counter));
NamespaceString barNss("bar", "coll" + std::to_string(counter));
+ auto fooUuid = UUID::gen();
std::shared_ptr<Collection> fooColl = std::make_shared<CollectionMock>(fooNss);
+
+ auto barUuid = UUID::gen();
std::shared_ptr<Collection> barColl = std::make_shared<CollectionMock>(barNss);
- dbMap["foo"].insert(std::make_pair(fooColl->uuid(), fooColl.get()));
- dbMap["bar"].insert(std::make_pair(barColl->uuid(), barColl.get()));
+ dbMap["foo"].insert(std::make_pair(fooUuid, fooColl.get()));
+ dbMap["bar"].insert(std::make_pair(barUuid, barColl.get()));
- catalog.registerCollection(&opCtx, fooColl);
- catalog.registerCollection(&opCtx, barColl);
+ catalog.registerCollection(&opCtx, fooUuid, fooColl);
+ catalog.registerCollection(&opCtx, barUuid, barColl);
}
}
@@ -126,10 +134,10 @@ public:
void checkCollections(const TenantDatabaseName& tenantDbName) {
unsigned long counter = 0;
- auto orderedIt = collsIterator(tenantDbName.dbName());
- auto catalogRange = catalog.range(tenantDbName);
- auto catalogIt = catalogRange.begin();
- for (; catalogIt != catalogRange.end() &&
+
+ for (auto [orderedIt, catalogIt] = std::tuple{collsIterator(tenantDbName.dbName()),
+ catalog.begin(&opCtx, tenantDbName)};
+ catalogIt != catalog.end(&opCtx) &&
orderedIt != collsIteratorEnd(tenantDbName.dbName());
++catalogIt, ++orderedIt) {
@@ -277,13 +285,17 @@ public:
for (int i = 0; i < 5; i++) {
NamespaceString nss("resourceDb", "coll" + std::to_string(i));
std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(nss);
+ auto uuid = collection->uuid();
- catalog.registerCollection(&opCtx, std::move(collection));
+ catalog.registerCollection(&opCtx, uuid, std::move(collection));
}
int numEntries = 0;
- for (auto&& coll : catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) {
- auto collName = coll->ns().ns();
+ for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb"));
+ it != catalog.end(&opCtx);
+ it++) {
+ auto coll = *it;
+ std::string collName = coll->ns().ns();
ResourceId rid(RESOURCE_COLLECTION, collName);
ASSERT_NE(catalog.lookupResourceName(rid), boost::none);
@@ -294,7 +306,10 @@ public:
void tearDown() {
std::vector<UUID> collectionsToDeregister;
- for (auto&& coll : catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) {
+ for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb"));
+ it != catalog.end(&opCtx);
+ ++it) {
+ auto coll = *it;
auto uuid = coll->uuid();
if (!coll) {
break;
@@ -308,8 +323,9 @@ public:
}
int numEntries = 0;
- for ([[maybe_unused]] auto&& coll :
- catalog.range(TenantDatabaseName(boost::none, "resourceDb"))) {
+ for (auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "resourceDb"));
+ it != catalog.end(&opCtx);
+ it++) {
numEntries++;
}
ASSERT_EQ(0, numEntries);
@@ -386,14 +402,13 @@ TEST_F(CollectionCatalogIterationTest, EndAtEndOfSection) {
}
TEST_F(CollectionCatalogIterationTest, GetUUIDWontRepositionEvenIfEntryIsDropped) {
- auto range = catalog.range(TenantDatabaseName(boost::none, "bar"));
- auto it = range.begin();
+ auto it = catalog.begin(&opCtx, TenantDatabaseName(boost::none, "bar"));
auto collsIt = collsIterator("bar");
auto uuid = collsIt->first;
catalog.deregisterCollection(&opCtx, uuid);
dropColl("bar", uuid);
- ASSERT_EQUALS(uuid, (*it)->uuid());
+ ASSERT_EQUALS(uuid, it.uuid());
}
TEST_F(CollectionCatalogTest, OnCreateCollection) {
@@ -418,13 +433,13 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUID) {
TEST_F(CollectionCatalogTest, InsertAfterLookup) {
auto newUUID = UUID::gen();
NamespaceString newNss(nss.db(), "newcol");
- std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newUUID, newNss);
+ std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss);
auto newCol = newCollShared.get();
// Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs.
ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), newUUID) == nullptr);
ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), newUUID), boost::none);
- catalog.registerCollection(opCtx.get(), std::move(newCollShared));
+ catalog.registerCollection(opCtx.get(), newUUID, std::move(newCollShared));
ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol);
ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss);
}
@@ -467,7 +482,7 @@ TEST_F(CollectionCatalogTest, RenameCollection) {
NamespaceString oldNss(nss.db(), "oldcol");
std::shared_ptr<Collection> collShared = std::make_shared<CollectionMock>(uuid, oldNss);
auto collection = collShared.get();
- catalog.registerCollection(opCtx.get(), std::move(collShared));
+ catalog.registerCollection(opCtx.get(), uuid, std::move(collShared));
auto yieldableColl = catalog.lookupCollectionByUUID(opCtx.get(), uuid);
ASSERT(yieldableColl);
ASSERT_EQUALS(yieldableColl, collection);
@@ -504,7 +519,7 @@ TEST_F(CollectionCatalogTest, RenameCollection) {
TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDropped) {
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onCloseCatalog();
+ catalog.onCloseCatalog(opCtx.get());
}
catalog.deregisterCollection(opCtx.get(), colUUID);
@@ -513,7 +528,7 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDrop
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onOpenCatalog();
+ catalog.onOpenCatalog(opCtx.get());
}
ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), colUUID), boost::none);
@@ -522,25 +537,25 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsOldNSSIfDrop
TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreatedNSS) {
auto newUUID = UUID::gen();
NamespaceString newNss(nss.db(), "newcol");
- std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newUUID, newNss);
+ std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss);
auto newCol = newCollShared.get();
// Ensure that looking up non-existing UUIDs doesn't affect later registration of those UUIDs.
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onCloseCatalog();
+ catalog.onCloseCatalog(opCtx.get());
}
ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), newUUID) == nullptr);
ASSERT_EQUALS(catalog.lookupNSSByUUID(opCtx.get(), newUUID), boost::none);
- catalog.registerCollection(opCtx.get(), std::move(newCollShared));
+ catalog.registerCollection(opCtx.get(), newUUID, std::move(newCollShared));
ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol);
ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss);
// Ensure that collection still exists after opening the catalog again.
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onOpenCatalog();
+ catalog.onOpenCatalog(opCtx.get());
}
ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), newUUID), newCol);
@@ -549,29 +564,25 @@ TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsNewlyCreated
TEST_F(CollectionCatalogTest, LookupNSSByUUIDForClosedCatalogReturnsFreshestNSS) {
NamespaceString newNss(nss.db(), "newcol");
- std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(colUUID, newNss);
+ std::shared_ptr<Collection> newCollShared = std::make_shared<CollectionMock>(newNss);
auto newCol = newCollShared.get();
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onCloseCatalog();
+ catalog.onCloseCatalog(opCtx.get());
}
catalog.deregisterCollection(opCtx.get(), colUUID);
ASSERT(catalog.lookupCollectionByUUID(opCtx.get(), colUUID) == nullptr);
ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), nss);
- {
- Lock::GlobalWrite lk(opCtx.get());
- catalog.registerCollection(opCtx.get(), std::move(newCollShared));
- }
-
+ catalog.registerCollection(opCtx.get(), colUUID, std::move(newCollShared));
ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), colUUID), newCol);
ASSERT_EQUALS(*catalog.lookupNSSByUUID(opCtx.get(), colUUID), newNss);
// Ensure that collection still exists after opening the catalog again.
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onOpenCatalog();
+ catalog.onOpenCatalog(opCtx.get());
}
ASSERT_EQUALS(catalog.lookupCollectionByUUID(opCtx.get(), colUUID), newCol);
@@ -584,8 +595,8 @@ TEST_F(CollectionCatalogTest, CollectionCatalogEpoch) {
{
Lock::GlobalLock globalLk(opCtx.get(), MODE_X);
- catalog.onCloseCatalog();
- catalog.onOpenCatalog();
+ catalog.onCloseCatalog(opCtx.get());
+ catalog.onOpenCatalog(opCtx.get());
}
auto incrementedEpoch = catalog.getEpoch();
@@ -609,7 +620,8 @@ TEST_F(CollectionCatalogTest, GetAllCollectionNamesAndGetAllDbNames) {
std::vector<NamespaceString> nsss = {aColl, b1Coll, b2Coll, cColl, d1Coll, d2Coll, d3Coll};
for (auto& nss : nsss) {
std::shared_ptr<Collection> newColl = std::make_shared<CollectionMock>(nss);
- catalog.registerCollection(opCtx.get(), std::move(newColl));
+ auto uuid = UUID::gen();
+ catalog.registerCollection(opCtx.get(), uuid, std::move(newColl));
}
std::vector<NamespaceString> dCollList = {d1Coll, d2Coll, d3Coll};
@@ -664,7 +676,8 @@ TEST_F(CollectionCatalogTest, GetAllCollectionNamesAndGetAllDbNamesWithUncommitt
std::vector<NamespaceString> nsss = {aColl, b1Coll, b2Coll, cColl, d1Coll, d2Coll, d3Coll};
for (auto& nss : nsss) {
std::shared_ptr<Collection> newColl = std::make_shared<CollectionMock>(nss);
- catalog.registerCollection(opCtx.get(), std::move(newColl));
+ auto uuid = UUID::gen();
+ catalog.registerCollection(opCtx.get(), uuid, std::move(newColl));
}
// One dbName with only an invisible collection does not appear in dbNames. Use const_cast to
diff --git a/src/mongo/db/catalog/collection_compact.cpp b/src/mongo/db/catalog/collection_compact.cpp
index b6cc7cb444d..4fed779eaae 100644
--- a/src/mongo/db/catalog/collection_compact.cpp
+++ b/src/mongo/db/catalog/collection_compact.cpp
@@ -74,10 +74,10 @@ StatusWith<int64_t> compactCollection(OperationContext* opCtx,
Database* database = autoDb.getDb();
uassert(ErrorCodes::NamespaceNotFound, "database does not exist", database);
- // The collection lock will be upgraded to an exclusive lock if the record store does not
- // support online compaction.
+ // The collection lock will be downgraded to an intent lock if the record store supports
+ // online compaction.
boost::optional<Lock::CollectionLock> collLk;
- collLk.emplace(opCtx, collectionNss, MODE_IX);
+ collLk.emplace(opCtx, collectionNss, MODE_X);
CollectionPtr collection = getCollectionForCompact(opCtx, collectionNss);
DisableDocumentValidation validationDisabler(opCtx);
@@ -91,9 +91,10 @@ StatusWith<int64_t> compactCollection(OperationContext* opCtx,
str::stream() << "cannot compact collection with record store: "
<< recordStore->name());
- if (!recordStore->supportsOnlineCompaction()) {
- // Storage engines that disallow online compaction should compact under an exclusive lock.
- collLk.emplace(opCtx, collectionNss, MODE_X);
+ if (recordStore->supportsOnlineCompaction()) {
+ // Storage engines that allow online compaction should do so using an intent lock on the
+ // collection.
+ collLk.emplace(opCtx, collectionNss, MODE_IX);
// Ensure the collection was not dropped during the re-lock.
collection = getCollectionForCompact(opCtx, collectionNss);
diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp
index cdf3730fa1e..b79c8c78914 100644
--- a/src/mongo/db/catalog/collection_impl.cpp
+++ b/src/mongo/db/catalog/collection_impl.cpp
@@ -40,8 +40,6 @@
#include "mongo/bson/simple_bsonobj_comparator.h"
#include "mongo/crypto/fle_crypto.h"
#include "mongo/db/auth/security_token.h"
-#include "mongo/db/catalog/backwards_compatible_collection_options_util.h"
-#include "mongo/db/catalog/catalog_stats.h"
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/collection_options.h"
#include "mongo/db/catalog/document_validation.h"
@@ -49,7 +47,6 @@
#include "mongo/db/catalog/index_consistency.h"
#include "mongo/db/catalog/index_key_validate.h"
#include "mongo/db/catalog/local_oplog_info.h"
-#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h"
#include "mongo/db/catalog/uncommitted_multikey.h"
#include "mongo/db/clientcursor.h"
#include "mongo/db/commands/server_status_metric.h"
@@ -83,7 +80,6 @@
#include "mongo/db/storage/record_store.h"
#include "mongo/db/storage/storage_parameters_gen.h"
#include "mongo/db/timeseries/timeseries_constants.h"
-#include "mongo/db/timeseries/timeseries_extended_range.h"
#include "mongo/db/timeseries/timeseries_index_schema_conversion_functions.h"
#include "mongo/db/transaction_participant.h"
#include "mongo/db/ttl_collection_cache.h"
@@ -354,42 +350,35 @@ bool indexTypeSupportsPathLevelMultikeyTracking(StringData accessMethod) {
return accessMethod == IndexNames::BTREE || accessMethod == IndexNames::GEO_2DSPHERE;
}
-StatusWith<bool> doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) {
+bool doesMinMaxHaveMixedSchemaData(const BSONObj& min, const BSONObj& max) {
auto minIt = min.begin();
auto minEnd = min.end();
auto maxIt = max.begin();
auto maxEnd = max.end();
while (minIt != minEnd && maxIt != maxEnd) {
- // The 'control.min' and 'control.max' fields have the same ordering.
- if (minIt->fieldNameStringData() != maxIt->fieldNameStringData()) {
- return Status{
- ErrorCodes::BadValue,
- "Encountered inconsistent field name ordering in time-series bucket min/max"};
- }
-
- if (minIt->canonicalType() != maxIt->canonicalType()) {
+ bool typeMatch = minIt->canonicalType() == maxIt->canonicalType();
+ if (!typeMatch) {
return true;
} else if (minIt->type() == Object) {
- auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj());
- if (!result.isOK() || result.getValue()) {
- return result;
+ // The 'control.min' and 'control.max' fields have the same ordering.
+ invariant(minIt->fieldNameStringData() == maxIt->fieldNameStringData());
+ if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) {
+ return true;
}
} else if (minIt->type() == Array) {
- auto result = doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj());
- if (!result.isOK() || result.getValue()) {
- return result;
+ if (doesMinMaxHaveMixedSchemaData(minIt->Obj(), maxIt->Obj())) {
+ return true;
}
}
+ invariant(typeMatch);
minIt++;
maxIt++;
}
- if (minIt != minEnd || maxIt != maxEnd) {
- return Status{ErrorCodes::BadValue,
- "Encountered extra field(s) in time-series bucket min/max"};
- }
+ // The 'control.min' and 'control.max' fields have the same cardinality.
+ invariant(minIt == minEnd && maxIt == maxEnd);
return false;
}
@@ -550,11 +539,11 @@ void CollectionImpl::init(OperationContext* opCtx) {
if (opCtx->lockState()->inAWriteUnitOfWork()) {
opCtx->recoveryUnit()->onCommit([svcCtx, uuid](auto ts) {
TTLCollectionCache::get(svcCtx).registerTTLInfo(
- uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
+ uuid, TTLCollectionCache::ClusteredId{});
});
} else {
- TTLCollectionCache::get(svcCtx).registerTTLInfo(
- uuid, TTLCollectionCache::Info{TTLCollectionCache::ClusteredId{}});
+ TTLCollectionCache::get(svcCtx).registerTTLInfo(uuid,
+ TTLCollectionCache::ClusteredId{});
}
}
}
@@ -736,8 +725,6 @@ Collection::Validator CollectionImpl::parseValidator(
auto expCtx = make_intrusive<ExpressionContext>(
opCtx, CollatorInterface::cloneCollator(_shared->_collator.get()), ns());
- expCtx->variables.setDefaultRuntimeConstants(opCtx);
-
// The MatchExpression and contained ExpressionContext created as part of the validator are
// owned by the Collection and will outlive the OperationContext they were created under.
expCtx->opCtx = nullptr;
@@ -824,9 +811,9 @@ Status CollectionImpl::insertDocumentsForOplog(OperationContext* opCtx,
_cappedDeleteAsNeeded(opCtx, records->begin()->id);
- // We do not need to notify capped waiters, as we have not yet updated oplog visibility, so
- // these inserts will not be visible. When visibility updates, it will notify capped
- // waiters.
+ opCtx->recoveryUnit()->onCommit(
+ [this](boost::optional<Timestamp>) { _shared->notifyCappedWaitersIfNeeded(); });
+
return status;
}
@@ -1598,19 +1585,6 @@ bool CollectionImpl::isTemporary() const {
}
boost::optional<bool> CollectionImpl::getTimeseriesBucketsMayHaveMixedSchemaData() const {
- if (!getTimeseriesOptions()) {
- return boost::none;
- }
-
- // If present, reuse storageEngine options to work around the issue described in SERVER-91194
- boost::optional<bool> optBackwardsCompatibleFlag = getFlagFromStorageEngineBson(
- _metadata->options.storageEngine,
- backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData);
- if (optBackwardsCompatibleFlag) {
- return *optBackwardsCompatibleFlag;
- }
-
- // Else, fallback to legacy parameter
return _metadata->timeseriesBucketsMayHaveMixedSchemaData;
}
@@ -1626,21 +1600,11 @@ void CollectionImpl::setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext
"setting"_attr = setting);
_writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) {
- // Reuse storageEngine options to work around the issue described in SERVER-91194
- if (setting.has_value()) {
- md.options.storageEngine = setFlagToStorageEngineBson(
- md.options.storageEngine,
- backwards_compatible_collection_options::kTimeseriesBucketsMayHaveMixedSchemaData,
- *setting);
- }
-
- // Also update legacy parameter for compatibility when downgrading to older sub-versions
- // only relying on this option (best-effort because it may be lost due to SERVER-91194)
md.timeseriesBucketsMayHaveMixedSchemaData = setting;
});
}
-StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData(
+bool CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData(
const BSONObj& bucketsDoc) const {
if (!getTimeseriesOptions()) {
return false;
@@ -1653,29 +1617,6 @@ StatusWith<bool> CollectionImpl::doesTimeseriesBucketsDocContainMixedSchemaData(
return doesMinMaxHaveMixedSchemaData(minObj, maxObj);
}
-bool CollectionImpl::getRequiresTimeseriesExtendedRangeSupport() const {
- return _shared->_requiresTimeseriesExtendedRangeSupport.load();
-}
-
-void CollectionImpl::setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const {
- uassert(6679401, "This is not a time-series collection", _metadata->options.timeseries);
-
- bool expected = false;
- bool set = _shared->_requiresTimeseriesExtendedRangeSupport.compareAndSwap(&expected, true);
- if (set) {
- catalog_stats::requiresTimeseriesExtendedRangeSupport.fetchAndAdd(1);
- if (!timeseries::collectionHasTimeIndex(opCtx, *this)) {
- LOGV2_WARNING(
- 6679402,
- "Time-series collection contains dates outside the standard range. Some query "
- "optimizations may be disabled. Please consider building an index on timeField to "
- "re-enable them.",
- "nss"_attr = ns().getTimeseriesViewNamespace(),
- "timeField"_attr = _metadata->options.timeseries->getTimeField());
- }
- }
-}
-
bool CollectionImpl::isClustered() const {
return getClusteredInfo().is_initialized();
}
@@ -1814,8 +1755,7 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx,
int scale) const {
const IndexCatalog* idxCatalog = getIndexCatalog();
- auto ii = idxCatalog->getIndexIterator(
- opCtx, IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ std::unique_ptr<IndexCatalog::IndexIterator> ii = idxCatalog->getIndexIterator(opCtx, true);
uint64_t totalSize = 0;
@@ -1836,18 +1776,9 @@ uint64_t CollectionImpl::getIndexSize(OperationContext* opCtx,
}
uint64_t CollectionImpl::getIndexFreeStorageBytes(OperationContext* const opCtx) const {
- // Unfinished index builds are excluded to avoid a potential deadlock when trying to collect
- // statistics from the index table while the index build is in the bulk load phase. See
- // SERVER-77018. This should not be too impactful as:
- // - During the collection scan phase, the index table is unused.
- // - During the bulk load phase, getFreeStorageBytes will probably return EBUSY, as the ident is
- // in use by the index builder. (And worst case results in the deadlock).
- // - It might be possible to return meaningful data post bulk-load, but reusable bytes should be
- // low anyways as the collection has been bulk loaded. Additionally, this would be a inaccurate
- // anyways as the build is in progress.
- // - Once the index build is finished, this will be eventually accounted for.
const auto idxCatalog = getIndexCatalog();
- auto indexIt = idxCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady);
+ const bool includeUnfinished = true;
+ auto indexIt = idxCatalog->getIndexIterator(opCtx, includeUnfinished);
uint64_t totalSize = 0;
while (indexIt->more()) {
@@ -1871,7 +1802,8 @@ Status CollectionImpl::truncate(OperationContext* opCtx) {
// 1) store index specs
std::vector<BSONObj> indexSpecs;
{
- auto ii = _indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady);
+ std::unique_ptr<IndexCatalog::IndexIterator> ii =
+ _indexCatalog->getIndexIterator(opCtx, false);
while (ii->more()) {
const IndexDescriptor* idx = ii->next()->descriptor();
indexSpecs.push_back(idx->infoObj().getOwned());
@@ -2206,22 +2138,20 @@ void CollectionImpl::updatePrepareUniqueSetting(OperationContext* opCtx,
});
}
-std::vector<std::string> CollectionImpl::repairInvalidIndexOptions(OperationContext* opCtx) {
+std::vector<std::string> CollectionImpl::removeInvalidIndexOptions(OperationContext* opCtx) {
std::vector<std::string> indexesWithInvalidOptions;
_writeMetadata(opCtx, [&](BSONCollectionCatalogEntry::MetaData& md) {
for (auto& index : md.indexes) {
- if (index.isPresent()) {
- BSONObj oldSpec = index.spec;
-
- Status status = index_key_validate::validateIndexSpec(opCtx, oldSpec).getStatus();
- if (status.isOK()) {
- continue;
- }
+ BSONObj oldSpec = index.spec;
- indexesWithInvalidOptions.push_back(std::string(index.nameStringData()));
- index.spec = index_key_validate::repairIndexSpec(NamespaceString(md.ns), oldSpec);
+ Status status = index_key_validate::validateIndexSpecFieldNames(oldSpec);
+ if (status.isOK()) {
+ continue;
}
+
+ indexesWithInvalidOptions.push_back(std::string(index.nameStringData()));
+ index.spec = index_key_validate::removeUnknownFields(NamespaceString(md.ns), oldSpec);
}
});
@@ -2298,65 +2228,44 @@ bool CollectionImpl::isIndexMultikey(OperationContext* opCtx,
StringData indexName,
MultikeyPaths* multikeyPaths,
int indexOffset) const {
- int offset = indexOffset;
- if (offset < 0) {
- offset = _metadata->findIndexOffset(indexName);
- invariant(offset >= 0,
- str::stream() << "cannot get multikey for index " << indexName << " @ "
- << getCatalogId() << " : " << _metadata->toBSON());
- } else {
- invariant(offset < int(_metadata->indexes.size()),
- str::stream() << "out of bounds index offset for multikey info " << indexName
- << " @ " << getCatalogId() << " : " << _metadata->toBSON()
- << "; offset : " << offset
- << " ; actual : " << _metadata->findIndexOffset(indexName));
- invariant(indexName == _metadata->indexes[offset].nameStringData(),
- str::stream() << "invalid index offset for multikey info " << indexName << " @ "
- << getCatalogId() << " : " << _metadata->toBSON()
- << "; offset : " << offset
- << " ; actual : " << _metadata->findIndexOffset(indexName));
- }
-
- // If we have uncommitted multikey writes we need to check here to read our own writes
+ auto isMultikey = [this, multikeyPaths, indexName, indexOffset](
+ const BSONCollectionCatalogEntry::MetaData& metadata) {
+ int offset = indexOffset;
+ if (offset < 0) {
+ offset = metadata.findIndexOffset(indexName);
+ invariant(offset >= 0,
+ str::stream() << "cannot get multikey for index " << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON());
+ } else {
+ invariant(offset < int(metadata.indexes.size()),
+ str::stream()
+ << "out of bounds index offset for multikey info " << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset
+ << " ; actual : " << metadata.findIndexOffset(indexName));
+ invariant(indexName == metadata.indexes[offset].nameStringData(),
+ str::stream()
+ << "invalid index offset for multikey info " << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset
+ << " ; actual : " << metadata.findIndexOffset(indexName));
+ }
+
+ const auto& index = metadata.indexes[offset];
+ stdx::lock_guard lock(index.multikeyMutex);
+ if (multikeyPaths && !index.multikeyPaths.empty()) {
+ *multikeyPaths = index.multikeyPaths;
+ }
+
+ return index.multikey;
+ };
+
const auto& uncommittedMultikeys = UncommittedMultikey::get(opCtx).resources();
if (uncommittedMultikeys) {
if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) {
- const auto& index = it->second.indexes[offset];
- if (multikeyPaths && !index.multikeyPaths.empty()) {
- *multikeyPaths = index.multikeyPaths;
- }
- return index.multikey;
- }
- }
-
- // Otherwise read from the metadata cache if there are no concurrent multikey writers
- {
- const auto& index = _metadata->indexes[offset];
- // Check for concurrent writers, this can race with writers where it can be set immediately
- // after checking. This is fine we know that the reader in that case opened its snapshot
- // before the writer and we do not need to observe its result.
- if (index.concurrentWriters.load() == 0) {
- stdx::lock_guard lock(index.multikeyMutex);
- if (multikeyPaths && !index.multikeyPaths.empty()) {
- *multikeyPaths = index.multikeyPaths;
- }
- return index.multikey;
+ return isMultikey(it->second);
}
}
- // We need to read from the durable catalog if there are concurrent multikey writers to avoid
- // reading between the multikey write committing in the storage engine but before its onCommit
- // handler made the write visible for readers.
- auto snapshotMetadata = DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId());
- int snapshotOffset = snapshotMetadata->findIndexOffset(indexName);
- invariant(snapshotOffset >= 0,
- str::stream() << "cannot get multikey for index " << indexName << " @ "
- << getCatalogId() << " : " << _metadata->toBSON());
- const auto& index = snapshotMetadata->indexes[snapshotOffset];
- if (multikeyPaths && !index.multikeyPaths.empty()) {
- *multikeyPaths = index.multikeyPaths;
- }
- return index.multikey;
+ return isMultikey(*_metadata);
}
bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx,
@@ -2364,31 +2273,31 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx,
const MultikeyPaths& multikeyPaths,
int indexOffset) const {
- int offset = indexOffset;
- if (offset < 0) {
- offset = _metadata->findIndexOffset(indexName);
- invariant(offset >= 0,
- str::stream() << "cannot set multikey for index " << indexName << " @ "
- << getCatalogId() << " : " << _metadata->toBSON());
- } else {
- invariant(offset < int(_metadata->indexes.size()),
- str::stream() << "out of bounds index offset for multikey update" << indexName
- << " @ " << getCatalogId() << " : " << _metadata->toBSON()
- << "; offset : " << offset
- << " ; actual : " << _metadata->findIndexOffset(indexName));
- invariant(indexName == _metadata->indexes[offset].nameStringData(),
- str::stream() << "invalid index offset for multikey update " << indexName << " @ "
- << getCatalogId() << " : " << _metadata->toBSON()
- << "; offset : " << offset
- << " ; actual : " << _metadata->findIndexOffset(indexName));
- }
-
- auto setMultikey = [offset,
- multikeyPaths](const BSONCollectionCatalogEntry::MetaData& metadata) {
+ auto setMultikey = [this, indexName, multikeyPaths, indexOffset](
+ const BSONCollectionCatalogEntry::MetaData& metadata) {
+ int offset = indexOffset;
+ if (offset < 0) {
+ offset = metadata.findIndexOffset(indexName);
+ invariant(offset >= 0,
+ str::stream() << "cannot set multikey for index " << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON());
+ } else {
+ invariant(offset < int(metadata.indexes.size()),
+ str::stream()
+ << "out of bounds index offset for multikey update" << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset
+ << " ; actual : " << metadata.findIndexOffset(indexName));
+ invariant(indexName == metadata.indexes[offset].nameStringData(),
+ str::stream()
+ << "invalid index offset for multikey update " << indexName << " @ "
+ << getCatalogId() << " : " << metadata.toBSON() << "; offset : " << offset
+ << " ; actual : " << metadata.findIndexOffset(indexName));
+ }
+
auto* index = &metadata.indexes[offset];
stdx::lock_guard lock(index->multikeyMutex);
- auto tracksPathLevelMultikeyInfo = !index->multikeyPaths.empty();
+ auto tracksPathLevelMultikeyInfo = !metadata.indexes[offset].multikeyPaths.empty();
if (!tracksPathLevelMultikeyInfo) {
invariant(multikeyPaths.empty());
@@ -2404,7 +2313,7 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx,
// We are tracking path-level multikey information for this index.
invariant(!multikeyPaths.empty());
- invariant(multikeyPaths.size() == index->multikeyPaths.size());
+ invariant(multikeyPaths.size() == metadata.indexes[offset].multikeyPaths.size());
index->multikey = true;
@@ -2443,31 +2352,11 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx,
}
BSONCollectionCatalogEntry::MetaData* metadata = nullptr;
bool hasSetMultikey = false;
-
if (auto it = uncommittedMultikeys->find(this); it != uncommittedMultikeys->end()) {
metadata = &it->second;
hasSetMultikey = setMultikey(*metadata);
} else {
- // First time this OperationContext needs to change multikey information for this
- // collection. We cannot use the cached metadata in this collection as we may have just
- // committed a multikey change concurrently to the storage engine without being able to
- // observe it if its onCommit handlers haven't run yet.
- auto metadataLocal = *DurableCatalog::get(opCtx)->getMetaData(opCtx, getCatalogId());
- // When reading from the durable catalog the index offsets are different because when
- // removing indexes in-memory just zeros out the slot instead of actually removing it. We
- // must adjust the entries so they match how they are stored in _metadata so we can rely on
- // the index offsets being stable. The order of valid indexes are the same, so we can
- // iterate from the end and move them into the right positions.
- int localIdx = metadataLocal.indexes.size() - 1;
- metadataLocal.indexes.resize(_metadata->indexes.size());
- for (int i = _metadata->indexes.size() - 1; i >= 0 && localIdx != i; --i) {
- if (_metadata->indexes[i].isPresent()) {
- metadataLocal.indexes[i] = std::move(metadataLocal.indexes[localIdx]);
- metadataLocal.indexes[localIdx] = {};
- --localIdx;
- }
- }
-
+ BSONCollectionCatalogEntry::MetaData metadataLocal(*_metadata);
hasSetMultikey = setMultikey(metadataLocal);
if (hasSetMultikey) {
metadata = &uncommittedMultikeys->emplace(this, std::move(metadataLocal)).first->second;
@@ -2482,44 +2371,8 @@ bool CollectionImpl::setIndexIsMultikey(OperationContext* opCtx,
DurableCatalog::get(opCtx)->putMetaData(opCtx, getCatalogId(), *metadata);
- // RAII Helper object to ensure we decrement the concurrent counter if and only if we
- // incremented it in a preCommit handler.
- class ConcurrentMultikeyWriteTracker {
- public:
- ConcurrentMultikeyWriteTracker(
- std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> meta, int indexOffset)
- : metadata(std::move(meta)), offset(indexOffset) {}
-
- ~ConcurrentMultikeyWriteTracker() {
- if (hasIncremented) {
- metadata->indexes[offset].concurrentWriters.fetchAndSubtract(1);
- }
- }
-
- void preCommit() {
- metadata->indexes[offset].concurrentWriters.fetchAndAdd(1);
- hasIncremented = true;
- }
-
- private:
- std::shared_ptr<const BSONCollectionCatalogEntry::MetaData> metadata;
- int offset;
- bool hasIncremented = false;
- };
-
- auto concurrentWriteTracker =
- std::make_shared<ConcurrentMultikeyWriteTracker>(_metadata, offset);
-
- // Mark this index that there is an ongoing multikey write. This forces readers to read from the
- // durable catalog to determine if the index is multikey or not.
- opCtx->recoveryUnit()->registerPreCommitHook(
- [concurrentWriteTracker](OperationContext*) { concurrentWriteTracker->preCommit(); });
-
- // Capture a reference to 'concurrentWriteTracker' to extend the lifetime of this object until
- // commiting/rolling back the transaction is fully complete.
opCtx->recoveryUnit()->onCommit(
- [this, uncommittedMultikeys, setMultikey = std::move(setMultikey), concurrentWriteTracker](
- auto ts) {
+ [this, uncommittedMultikeys, setMultikey = std::move(setMultikey)](auto ts) {
// Merge in changes to this index, other indexes may have been updated since we made our
// copy. Don't check for result as another thread could be setting multikey at the same
// time
diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h
index 58662679df1..d3379568d22 100644
--- a/src/mongo/db/catalog/collection_impl.h
+++ b/src/mongo/db/catalog/collection_impl.h
@@ -329,11 +329,7 @@ public:
void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx,
boost::optional<bool> setting) final;
- StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData(
- const BSONObj& bucketsDoc) const final;
-
- bool getRequiresTimeseriesExtendedRangeSupport() const final;
- void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const final;
+ bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const final;
/**
* isClustered() relies on the object returned from getClusteredInfo(). If
@@ -450,7 +446,7 @@ public:
StringData idxName,
bool prepareUnique) final;
- std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) final;
+ std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) final;
void setIsTemp(OperationContext* opCtx, bool isTemp) final;
@@ -583,18 +579,6 @@ private:
AtomicWord<bool> _committed{true};
- // Time-series collections are allowed to contain measurements with arbitrary dates;
- // however, many of our query optimizations only work properly with dates that can be stored
- // as an offset in seconds from the Unix epoch within 31 bits (roughly 1970-2038). When this
- // flag is set to true, these optimizations will be disabled. It must be set to true if the
- // collection contains any measurements with dates outside this normal range.
- //
- // This is set from the write path where we only hold an IX lock, so we want to be able to
- // set it from a const method on the Collection. In order to do this, we need to make it
- // mutable. Given that the value may only transition from false to true, but never back
- // again, and that we store and retrieve it atomically, this should be safe.
- mutable AtomicWord<bool> _requiresTimeseriesExtendedRangeSupport{false};
-
// Capped information.
const bool _isCapped;
diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h
index 820ed47cdf4..00d21ec21dd 100644
--- a/src/mongo/db/catalog/collection_mock.h
+++ b/src/mongo/db/catalog/collection_mock.h
@@ -253,19 +253,10 @@ public:
std::abort();
}
- StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData(
- const BSONObj& bucketsDoc) const {
+ bool doesTimeseriesBucketsDocContainMixedSchemaData(const BSONObj& bucketsDoc) const {
std::abort();
}
- bool getRequiresTimeseriesExtendedRangeSupport() const {
- MONGO_UNREACHABLE;
- }
-
- void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const {
- MONGO_UNREACHABLE;
- }
-
bool isClustered() const {
return false;
}
@@ -431,7 +422,7 @@ public:
std::abort();
}
- std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) {
+ std::vector<std::string> removeInvalidIndexOptions(OperationContext* opCtx) {
std::abort();
}
diff --git a/src/mongo/db/catalog/collection_operation_source.cpp b/src/mongo/db/catalog/collection_operation_source.cpp
index 198e929c7f4..84b291f19ce 100644
--- a/src/mongo/db/catalog/collection_operation_source.cpp
+++ b/src/mongo/db/catalog/collection_operation_source.cpp
@@ -38,8 +38,6 @@ StringData toString(OperationSource source) {
static constexpr StringData kTimeseriesInsertString = "time-series insert"_sd;
static constexpr StringData kTimeseriesUpdateString = "time-series update"_sd;
static constexpr StringData kTimeseriesDeleteString = "time-series delete"_sd;
- static constexpr StringData kTimeseriesBucketCompressionString =
- "time-series bucket compression"_sd;
switch (source) {
case OperationSource::kStandard:
@@ -52,8 +50,6 @@ StringData toString(OperationSource source) {
return kTimeseriesUpdateString;
case OperationSource::kTimeseriesDelete:
return kTimeseriesDeleteString;
- case OperationSource::kTimeseriesBucketCompression:
- return kTimeseriesBucketCompressionString;
}
MONGO_UNREACHABLE;
diff --git a/src/mongo/db/catalog/collection_operation_source.h b/src/mongo/db/catalog/collection_operation_source.h
index 7546d681850..6cfff61882f 100644
--- a/src/mongo/db/catalog/collection_operation_source.h
+++ b/src/mongo/db/catalog/collection_operation_source.h
@@ -43,7 +43,6 @@ enum class OperationSource {
kTimeseriesInsert,
kTimeseriesUpdate,
kTimeseriesDelete,
- kTimeseriesBucketCompression
};
StringData toString(OperationSource source);
diff --git a/src/mongo/db/catalog/collection_test.cpp b/src/mongo/db/catalog/collection_test.cpp
index 00c131756ac..fd73c68bad8 100644
--- a/src/mongo/db/catalog/collection_test.cpp
+++ b/src/mongo/db/catalog/collection_test.cpp
@@ -403,9 +403,7 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) {
"max" : { "x" : [ 2, 3 ] } } })")};
for (const auto& controlDoc : mixedSchemaControlDocs) {
- auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc);
- ASSERT_OK(mixedSchema) << controlDoc;
- ASSERT_TRUE(mixedSchema.getValue()) << controlDoc;
+ ASSERT_TRUE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc));
}
std::vector<BSONObj> nonMixedSchemaControlDocs = {
@@ -462,27 +460,7 @@ TEST_F(CollectionTest, CheckTimeseriesBucketDocsForMixedSchemaData) {
for (const auto& controlDoc : nonMixedSchemaControlDocs) {
- auto mixedSchema = coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc);
- ASSERT_OK(mixedSchema) << controlDoc;
- ASSERT_FALSE(mixedSchema.getValue()) << controlDoc;
- }
-
- std::vector<BSONObj> malformedControlDocs = {
- // Inconsistent field name ordering
- ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 },
- "max" : { "y" : 2, "x" : 2 } } })"),
-
- // Extra field in min
- ::mongo::fromjson(R"({ "control" : { "min" : { "x" : 1, "y" : 1 },
- "max" : { "x" : 2 } } })"),
-
- // Extra field in max
- ::mongo::fromjson(R"({ "control" : { "min" : { "y" : 1 },
- "max" : { "y" : 2, "x" : 2 } } })")};
-
- for (const auto& controlDoc : malformedControlDocs) {
- ASSERT_NOT_OK(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc))
- << controlDoc;
+ ASSERT_FALSE(coll->doesTimeseriesBucketsDocContainMixedSchemaData(controlDoc));
}
}
diff --git a/src/mongo/db/catalog/collection_validation.cpp b/src/mongo/db/catalog/collection_validation.cpp
index 9d0c487b41a..42c5ac18dc7 100644
--- a/src/mongo/db/catalog/collection_validation.cpp
+++ b/src/mongo/db/catalog/collection_validation.cpp
@@ -78,7 +78,8 @@ void _validateIndexesInternalStructure(OperationContext* opCtx,
// Need to use the IndexCatalog here because the 'validateState->indexes' object hasn't been
// constructed yet. It must be initialized to ensure we're validating all indexes.
const IndexCatalog* indexCatalog = validateState->getCollection()->getIndexCatalog();
- const auto it = indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady);
+ const std::unique_ptr<IndexCatalog::IndexIterator> it =
+ indexCatalog->getIndexIterator(opCtx, false);
// Validate Indexes Internal Structure, checking if index files have been compromised or
// corrupted.
@@ -97,11 +98,14 @@ void _validateIndexesInternalStructure(OperationContext* opCtx,
auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()];
- iam->validate(opCtx, nullptr, &curIndexResults);
+ int64_t numValidated;
+ iam->validate(opCtx, &numValidated, &curIndexResults);
if (!curIndexResults.valid) {
results->valid = false;
}
+
+ curIndexResults.keysTraversedFromFullValidate = numValidated;
}
}
@@ -133,6 +137,33 @@ void _validateIndexes(OperationContext* opCtx,
auto& curIndexResults = (results->indexResultsMap)[descriptor->indexName()];
curIndexResults.keysTraversed = numTraversedKeys;
+ // If we are performing a full index validation, we have information on the number of index
+ // keys validated in _validateIndexesInternalStructure (when we validated the internal
+ // structure of the index). Check if this is consistent with 'numTraversedKeys' from
+ // traverseIndex above.
+ if (validateState->isFullIndexValidation()) {
+ invariant(opCtx->lockState()->isCollectionLockedForMode(validateState->nss(), MODE_X));
+
+ // The number of keys counted in _validateIndexesInternalStructure, when checking the
+ // internal structure of the index.
+ const int64_t numIndexKeys = curIndexResults.keysTraversedFromFullValidate;
+
+ // Check if currIndexResults is valid to ensure that this index is not corrupted or
+ // comprised (which was set in _validateIndexesInternalStructure). If the index is
+ // corrupted, there is no use in checking if the traversal yielded the same key count.
+ if (curIndexResults.valid) {
+ if (numIndexKeys != numTraversedKeys) {
+ curIndexResults.valid = false;
+ string msg = str::stream()
+ << "number of traversed index entries (" << numTraversedKeys
+ << ") does not match the number of expected index entries (" << numIndexKeys
+ << ")";
+ results->errors.push_back(msg);
+ results->valid = false;
+ }
+ }
+ }
+
if (!curIndexResults.valid) {
results->valid = false;
}
@@ -162,8 +193,7 @@ void _gatherIndexEntryErrors(OperationContext* opCtx,
ValidateResults tempValidateResults;
BSONObjBuilder tempBuilder;
- indexValidator->traverseRecordStore(
- opCtx, &tempValidateResults, &tempBuilder, validateState->validationVersion());
+ indexValidator->traverseRecordStore(opCtx, &tempValidateResults, &tempBuilder);
}
LOGV2_OPTIONS(
@@ -200,7 +230,7 @@ void _gatherIndexEntryErrors(OperationContext* opCtx,
LOGV2_OPTIONS(20301, {LogComponent::kIndex}, "Finished traversing through all the indexes");
- indexConsistency->addIndexEntryErrors(opCtx, result);
+ indexConsistency->addIndexEntryErrors(result);
}
void _validateIndexKeyCount(OperationContext* opCtx,
@@ -217,94 +247,6 @@ void _validateIndexKeyCount(OperationContext* opCtx,
}
}
-void _printIndexSpec(const ValidateState* validateState, StringData indexName) {
- auto& indexes = validateState->getIndexes();
- auto indexEntry =
- std::find_if(indexes.begin(),
- indexes.end(),
- [&](const std::shared_ptr<const IndexCatalogEntry> indexEntry) -> bool {
- return indexEntry->descriptor()->indexName() == indexName;
- });
- if (indexEntry != indexes.end()) {
- auto indexSpec = (*indexEntry)->descriptor()->infoObj();
- LOGV2_ERROR(7463100, "Index failed validation", "spec"_attr = indexSpec);
- }
-}
-
-/**
- * Logs oplog entries related to corrupted records/indexes in validation results.
- */
-void _logOplogEntriesForInvalidResults(OperationContext* opCtx, ValidateResults* results) {
- if (results->recordTimestamps.empty()) {
- return;
- }
-
- LOGV2(
- 7464200,
- "Validation failed: oplog timestamps referenced by corrupted collection and index entries",
- "numTimestamps"_attr = results->recordTimestamps.size());
-
- // Set up read on oplog collection.
- try {
- AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
- const auto& oplogCollection = oplogRead.getCollection();
-
- if (!oplogCollection) {
- for (auto it = results->recordTimestamps.rbegin();
- it != results->recordTimestamps.rend();
- it++) {
- const auto& timestamp = *it;
- LOGV2(8080900,
- " Validation failed: Oplog entry timestamp for corrupted collection and "
- "index entry",
- "timestamp"_attr = timestamp);
- }
- return;
- }
-
- // Log oplog entries in reverse from most recent timestamp to oldest.
- // Due to oplog truncation, if we fail to find any oplog entry for a particular timestamp,
- // we can stop searching for oplog entries with earlier timestamps.
- auto recordStore = oplogCollection->getRecordStore();
- uassert(ErrorCodes::InternalError,
- "Validation failed: Unable to get oplog record store for corrupted collection and "
- "index entries",
- recordStore);
-
- auto cursor = recordStore->getCursor(opCtx, /*forward=*/false);
- uassert(ErrorCodes::CursorNotFound,
- "Validation failed: Unable to get cursor to oplog collection.",
- cursor);
-
- for (auto it = results->recordTimestamps.rbegin(); it != results->recordTimestamps.rend();
- it++) {
- const auto& timestamp = *it;
-
- // A record id in the oplog collection is equivalent to the document's timestamp field.
- RecordId recordId(timestamp.asULL());
- auto record = cursor->seekExact(recordId);
- if (!record) {
- LOGV2(7464201,
- " Validation failed: Stopping oplog entry search for corrupted collection "
- "and index entries.",
- "timestamp"_attr = timestamp);
- break;
- }
-
- LOGV2(
- 7464202,
- " Validation failed: Oplog entry found for corrupted collection and index entry",
- "timestamp"_attr = timestamp,
- "oplogEntryDoc"_attr = redact(record->data.toBson()));
- }
- } catch (DBException& ex) {
- LOGV2_ERROR(7464203,
- "Validation failed: Unable to fetch entries from oplog collection for "
- "corrupted collection and index entries",
- "ex"_attr = ex);
- }
-}
-
void _reportValidationResults(OperationContext* opCtx,
ValidateState* validateState,
ValidateResults* results,
@@ -321,19 +263,17 @@ void _reportValidationResults(OperationContext* opCtx,
// Report detailed index validation results gathered when using {full: true} for validated
// indexes.
- int nIndexes = results->indexResultsMap.size();
- for (const auto& [indexName, vr] : results->indexResultsMap) {
- if (!vr.valid) {
- results->valid = false;
- _printIndexSpec(validateState, indexName);
+ for (const auto& index : validateState->getIndexes()) {
+ const std::string indexName = index->descriptor()->indexName();
+ auto& indexResultsMap = results->indexResultsMap;
+ if (indexResultsMap.find(indexName) == indexResultsMap.end()) {
+ continue;
}
- if (validateState->getSkippedIndexes().contains(indexName)) {
- // Index internal state was checked and cleared, so it was reported in indexResultsMap,
- // but we did not verify the index contents against the collection, so we should exclude
- // it from this report.
- --nIndexes;
- continue;
+ auto& vr = indexResultsMap.at(indexName);
+
+ if (!vr.valid) {
+ results->valid = false;
}
BSONObjBuilder bob(indexDetails.subobjStart(indexName));
@@ -354,7 +294,7 @@ void _reportValidationResults(OperationContext* opCtx,
results->errors.insert(results->errors.end(), vr.errors.begin(), vr.errors.end());
}
- output->append("nIndexes", nIndexes);
+ output->append("nIndexes", static_cast<int>(validateState->getIndexes().size()));
output->append("keysPerIndex", keysPerIndex.done());
output->append("indexDetails", indexDetails.done());
}
@@ -364,7 +304,6 @@ void _reportInvalidResults(OperationContext* opCtx,
ValidateResults* results,
BSONObjBuilder* output) {
_reportValidationResults(opCtx, validateState, results, output);
- _logOplogEntriesForInvalidResults(opCtx, results);
LOGV2_OPTIONS(20302,
{LogComponent::kIndex},
"Validation complete -- Corruption found",
@@ -402,6 +341,35 @@ void addErrorIfUnequal(boost::optional<ValidationActionEnum> stored,
results);
}
+std::string multikeyPathsToString(MultikeyPaths paths) {
+ str::stream builder;
+ builder << "[";
+ auto pathIt = paths.begin();
+ while (true) {
+ builder << "{";
+
+ auto pathSet = *pathIt;
+ auto setIt = pathSet.begin();
+ while (true) {
+ builder << *setIt++;
+ if (setIt == pathSet.end()) {
+ break;
+ } else {
+ builder << ",";
+ }
+ }
+ builder << "}";
+
+ if (++pathIt == paths.end()) {
+ break;
+ } else {
+ builder << ",";
+ }
+ }
+ builder << "]";
+ return builder;
+}
+
void _validateCatalogEntry(OperationContext* opCtx,
ValidateState* validateState,
ValidateResults* results) {
@@ -442,23 +410,20 @@ void _validateCatalogEntry(OperationContext* opCtx,
}
const auto& indexCatalog = collection->getIndexCatalog();
- auto indexIt = indexCatalog->getIndexIterator(opCtx,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ auto indexIt = indexCatalog->getIndexIterator(opCtx, /*includeUnfinishedIndexes=*/true);
while (indexIt->more()) {
const IndexCatalogEntry* indexEntry = indexIt->next();
const std::string indexName = indexEntry->descriptor()->indexName();
Status status =
- index_key_validate::validateIndexSpec(opCtx, indexEntry->descriptor()->infoObj())
- .getStatus();
+ index_key_validate::validateIndexSpecFieldNames(indexEntry->descriptor()->infoObj());
if (!status.isOK()) {
- results->warnings.push_back(
- fmt::format("The index specification for index '{}' contains invalid fields. {}. "
- "Run the 'collMod' command on the collection without any arguments "
- "to fix the invalid index options",
+ results->valid = false;
+ results->errors.push_back(
+ fmt::format("The index specification for index '{}' contains invalid field names. "
+ "{}. Run the 'collMod' command on the collection without any arguments "
+ "to remove the invalid index options",
indexName,
status.reason()));
}
@@ -594,15 +559,14 @@ Status validate(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
- const AdditionalOptions& additionalOptions,
ValidateResults* results,
BSONObjBuilder* output,
- bool logDiagnostics) {
+ bool turnOnExtraLoggingForTest) {
invariant(!opCtx->lockState()->isLocked() || storageGlobalParams.repair);
// This is deliberately outside of the try-catch block, so that any errors thrown in the
// constructor fail the cmd, as opposed to returning OK with valid:false.
- ValidateState validateState(opCtx, nss, mode, repairMode, additionalOptions, logDiagnostics);
+ ValidateState validateState(opCtx, nss, mode, repairMode, turnOnExtraLoggingForTest);
const auto replCoord = repl::ReplicationCoordinator::get(opCtx);
// Check whether we are allowed to read from this node after acquiring our locks. If we are
@@ -621,14 +585,6 @@ Status validate(OperationContext* opCtx,
opCtx->recoveryUnit()->abandonSnapshot();
opCtx->recoveryUnit()->setPrepareConflictBehavior(oldPrepareConflictBehavior);
});
-
- // Relax corruption detection so that we log and continue scanning instead of failing early.
- auto oldDataCorruptionMode = opCtx->recoveryUnit()->getDataCorruptionDetectionMode();
- opCtx->recoveryUnit()->setDataCorruptionDetectionMode(
- DataCorruptionDetectionMode::kLogAndContinue);
- ON_BLOCK_EXIT(
- [&] { opCtx->recoveryUnit()->setDataCorruptionDetectionMode(oldDataCorruptionMode); });
-
if (validateState.fixErrors()) {
// Note: cannot set PrepareConflictBehavior here, since the validate command with repair
// needs kIngnoreConflictsAllowWrites, but validate repair at startup cannot set that here
@@ -705,8 +661,7 @@ Status validate(OperationContext* opCtx,
// the collection. For clustered collections, the validator also verifies that the
// record key (RecordId) matches the cluster key field in the record value (document's
// cluster key).
- indexValidator.traverseRecordStore(
- opCtx, results, output, additionalOptions.validationVersion);
+ indexValidator.traverseRecordStore(opCtx, results, output);
// Pause collection validation while a lock is held and between collection and index data
// validation.
@@ -780,7 +735,8 @@ Status validate(OperationContext* opCtx,
return e.toStatus();
}
string err = str::stream() << "exception during collection validation: " << e.toString();
- results->warnings.push_back(err);
+ results->errors.push_back(err);
+ results->valid = false;
LOGV2_OPTIONS(5160302,
{LogComponent::kIndex},
"Validation failed due to exception",
diff --git a/src/mongo/db/catalog/collection_validation.h b/src/mongo/db/catalog/collection_validation.h
index f68258fee14..378244f39b7 100644
--- a/src/mongo/db/catalog/collection_validation.h
+++ b/src/mongo/db/catalog/collection_validation.h
@@ -29,9 +29,6 @@
#pragma once
-#include "mongo/base/status.h"
-#include "mongo/bson/bson_validate.h"
-#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/db/catalog/validate_results.h"
#include "mongo/db/namespace_string.h"
@@ -90,13 +87,6 @@ enum class RepairMode {
};
/**
- * Additional validation options that can run in any mode.
- */
-struct AdditionalOptions {
- ValidationVersion validationVersion = currentValidationVersion;
-};
-
-/**
* Expects the caller to hold no locks.
*
* Background validation does not support any type of full validation above.
@@ -111,10 +101,9 @@ Status validate(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
- const AdditionalOptions& additionalOptions,
ValidateResults* results,
BSONObjBuilder* output,
- bool logDiagnostics);
+ bool turnOnExtraLoggingForTest = false);
/**
* Checks whether a failpoint has been hit in the above validate() code..
diff --git a/src/mongo/db/catalog/collection_validation_test.cpp b/src/mongo/db/catalog/collection_validation_test.cpp
index 870c0408565..4ffff4f653b 100644
--- a/src/mongo/db/catalog/collection_validation_test.cpp
+++ b/src/mongo/db/catalog/collection_validation_test.cpp
@@ -47,9 +47,6 @@ namespace {
const NamespaceString kNss = NamespaceString("test.t");
class CollectionValidationTest : public CatalogTestFixture {
-protected:
- CollectionValidationTest(Options options = {}) : CatalogTestFixture(std::move(options)) {}
-
private:
void setUp() override {
CatalogTestFixture::setUp();
@@ -61,11 +58,6 @@ private:
};
};
-class CollectionValidationDiskTest : public CollectionValidationTest {
-protected:
- CollectionValidationDiskTest() : CollectionValidationTest(Options{}.ephemeral(false)) {}
-};
-
/**
* Calls validate on collection kNss with both kValidateFull and kValidateNormal validation levels
* and verifies the results.
@@ -86,14 +78,8 @@ std::vector<std::pair<BSONObj, ValidateResults>> foregroundValidate(
for (auto mode : modes) {
ValidateResults validateResults;
BSONObjBuilder output;
- ASSERT_OK(CollectionValidation::validate(opCtx,
- kNss,
- mode,
- repairMode,
- /*additionalOptions=*/{},
- &validateResults,
- &output,
- /*logDiagnostics=*/false));
+ ASSERT_OK(CollectionValidation::validate(
+ opCtx, kNss, mode, repairMode, &validateResults, &output));
BSONObj obj = output.obj();
BSONObjBuilder validateResultsBuilder;
validateResults.appendToResultObj(&validateResultsBuilder, true /* debugging */);
@@ -159,10 +145,8 @@ void backgroundValidate(OperationContext* opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
&validateResults,
- &output,
- /*logDiagnostics=*/false));
+ &output));
BSONObj obj = output.obj();
ASSERT_EQ(validateResults.valid, valid);
@@ -286,19 +270,6 @@ TEST_F(CollectionValidationTest, ValidateEnforceFastCount) {
{CollectionValidation::ValidateMode::kForegroundFullEnforceFastCount});
}
-TEST_F(CollectionValidationDiskTest, ValidateIndexDetailResultsSurfaceVerifyErrors) {
- FailPointEnableBlock fp{"WTValidateIndexStructuralDamage"};
- auto opCtx = operationContext();
- insertDataRange(opCtx, 0, 5); // initialize collection
- foregroundValidate(
- opCtx,
- /*valid*/ false,
- /*numRecords*/ std::numeric_limits<int32_t>::min(), // uninitialized
- /*numInvalidDocuments*/ std::numeric_limits<int32_t>::min(), // uninitialized
- /*numErrors*/ 1,
- {CollectionValidation::ValidateMode::kForegroundFull});
-}
-
/**
* Waits for a parallel running collection validation operation to start and then hang at a
* failpoint.
diff --git a/src/mongo/db/catalog/collection_writer_test.cpp b/src/mongo/db/catalog/collection_writer_test.cpp
index 7b8719a190f..1f828f38800 100644
--- a/src/mongo/db/catalog/collection_writer_test.cpp
+++ b/src/mongo/db/catalog/collection_writer_test.cpp
@@ -58,7 +58,7 @@ protected:
std::shared_ptr<Collection> collection = std::make_shared<CollectionMock>(kNss);
CollectionCatalog::write(getServiceContext(), [&](CollectionCatalog& catalog) {
- catalog.registerCollection(operationContext(), std::move(collection));
+ catalog.registerCollection(operationContext(), UUID::gen(), std::move(collection));
});
}
@@ -253,6 +253,7 @@ public:
CollectionCatalog::write(getServiceContext(), [&](CollectionCatalog& catalog) {
for (size_t i = 0; i < NumCollections; ++i) {
catalog.registerCollection(operationContext(),
+ UUID::gen(),
std::make_shared<CollectionMock>(
NamespaceString("many", fmt::format("coll{}", i))));
}
diff --git a/src/mongo/db/catalog/create_collection.cpp b/src/mongo/db/catalog/create_collection.cpp
index c64decf02a3..74be2bd73e4 100644
--- a/src/mongo/db/catalog/create_collection.cpp
+++ b/src/mongo/db/catalog/create_collection.cpp
@@ -44,7 +44,7 @@
#include "mongo/db/catalog/index_key_validate.h"
#include "mongo/db/commands.h"
#include "mongo/db/commands/create_gen.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -117,9 +117,7 @@ Status validateClusteredIndexSpec(OperationContext* opCtx,
if (expireAfterSeconds) {
// Not included in the indexSpec itself.
- auto status = index_key_validate::validateExpireAfterSeconds(
- *expireAfterSeconds,
- index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex);
+ auto status = index_key_validate::validateExpireAfterSeconds(*expireAfterSeconds);
if (!status.isOK()) {
return status;
}
@@ -208,117 +206,6 @@ Status _createView(OperationContext* opCtx,
});
}
-BSONObj _generateTimeseriesValidator(StringData timeField) {
- // '$jsonSchema' : {
- // bsonType: 'object',
- // required: ['_id', 'control', 'data'],
- // properties: {
- // _id: {bsonType: 'objectId'},
- // control: {
- // bsonType: 'object',
- // required: ['version', 'min', 'max'],
- // properties: {
- // version: {bsonType: 'number'},
- // min: {
- // bsonType: 'object',
- // required: ['%s'],
- // properties: {'%s': {bsonType: 'date'}}
- // },
- // max: {
- // bsonType: 'object',
- // required: ['%s'],
- // properties: {'%s': {bsonType: 'date'}}
- // },
- // closed: {bsonType: 'bool'},
- // count: {bsonType: 'number', minimum: 1},
- // },
- // additionalProperties: false,
- // },
- // data: {bsonType: 'object'},
- // meta: {}
- // },
- // additionalProperties: false
- // }
- BSONObjBuilder validator;
- BSONObjBuilder schema(validator.subobjStart("$jsonSchema"));
- schema.append("bsonType", "object");
- schema.append("required",
- BSON_ARRAY("_id"
- << "control"
- << "data"));
- {
- BSONObjBuilder properties(schema.subobjStart("properties"));
- {
- BSONObjBuilder _id(properties.subobjStart("_id"));
- _id.append("bsonType", "objectId");
- _id.done();
- }
- {
- BSONObjBuilder control(properties.subobjStart("control"));
- control.append("bsonType", "object");
- control.append("required",
- BSON_ARRAY("version"
- << "min"
- << "max"));
- {
- BSONObjBuilder innerProperties(control.subobjStart("properties"));
- {
- BSONObjBuilder version(innerProperties.subobjStart("version"));
- version.append("bsonType", "number");
- version.done();
- }
- {
- BSONObjBuilder min(innerProperties.subobjStart("min"));
- min.append("bsonType", "object");
- min.append("required", BSON_ARRAY(timeField));
- BSONObjBuilder minProperties(min.subobjStart("properties"));
- BSONObjBuilder timeFieldObj(minProperties.subobjStart(timeField));
- timeFieldObj.append("bsonType", "date");
- timeFieldObj.done();
- minProperties.done();
- min.done();
- }
-
- {
- BSONObjBuilder max(innerProperties.subobjStart("max"));
- max.append("bsonType", "object");
- max.append("required", BSON_ARRAY(timeField));
- BSONObjBuilder maxProperties(max.subobjStart("properties"));
- BSONObjBuilder timeFieldObj(maxProperties.subobjStart(timeField));
- timeFieldObj.append("bsonType", "date");
- timeFieldObj.done();
- maxProperties.done();
- max.done();
- }
- {
- BSONObjBuilder closed(innerProperties.subobjStart("closed"));
- closed.append("bsonType", "bool");
- closed.done();
- }
- {
- BSONObjBuilder count(innerProperties.subobjStart("count"));
- count.append("bsonType", "number");
- count.append("minimum", 1);
- count.done();
- }
- innerProperties.done();
- }
- control.append("additionalProperties", false);
- control.done();
- }
- {
- BSONObjBuilder data(properties.subobjStart("data"));
- data.append("bsonType", "object");
- data.done();
- }
- properties.append("meta", BSONObj{});
- properties.done();
- }
- schema.append("additionalProperties", false);
- schema.done();
- return validator.obj();
-}
-
Status _createTimeseries(OperationContext* opCtx,
const NamespaceString& ns,
const CollectionOptions& optionsArg) {
@@ -343,13 +230,46 @@ Status _createTimeseries(OperationContext* opCtx,
maxSpanSeconds == options.timeseries->getBucketMaxSpanSeconds());
options.timeseries->setBucketMaxSpanSeconds(maxSpanSeconds);
-
// Set the validator option to a JSON schema enforcing constraints on bucket documents.
// This validation is only structural to prevent accidental corruption by users and
// cannot cover all constraints. Leave the validationLevel and validationAction to their
// strict/error defaults.
auto timeField = options.timeseries->getTimeField();
- auto validatorObj = _generateTimeseriesValidator(timeField);
+ auto validatorObj = fromjson(fmt::sprintf(R"(
+{
+'$jsonSchema' : {
+ bsonType: 'object',
+ required: ['_id', 'control', 'data'],
+ properties: {
+ _id: {bsonType: 'objectId'},
+ control: {
+ bsonType: 'object',
+ required: ['version', 'min', 'max'],
+ properties: {
+ version: {bsonType: 'number'},
+ min: {
+ bsonType: 'object',
+ required: ['%s'],
+ properties: {'%s': {bsonType: 'date'}}
+ },
+ max: {
+ bsonType: 'object',
+ required: ['%s'],
+ properties: {'%s': {bsonType: 'date'}}
+ },
+ closed: {bsonType: 'bool'}
+ }
+ },
+ data: {bsonType: 'object'},
+ meta: {}
+ },
+ additionalProperties: false
+}
+})",
+ timeField,
+ timeField,
+ timeField,
+ timeField));
bool existingBucketCollectionIsCompatible = false;
@@ -401,9 +321,8 @@ Status _createTimeseries(OperationContext* opCtx,
// Cluster time-series buckets collections by _id.
auto expireAfterSeconds = options.expireAfterSeconds;
if (expireAfterSeconds) {
- uassertStatusOK(index_key_validate::validateExpireAfterSeconds(
- *expireAfterSeconds,
- index_key_validate::ValidateExpireAfterSecondsMode::kClusteredTTLIndex));
+ uassertStatusOK(
+ index_key_validate::validateExpireAfterSeconds(*expireAfterSeconds));
bucketsOptions.expireAfterSeconds = expireAfterSeconds;
}
diff --git a/src/mongo/db/catalog/create_collection_test.cpp b/src/mongo/db/catalog/create_collection_test.cpp
index b5cce6e6686..3141d18905c 100644
--- a/src/mongo/db/catalog/create_collection_test.cpp
+++ b/src/mongo/db/catalog/create_collection_test.cpp
@@ -34,7 +34,7 @@
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/create_collection.h"
#include "mongo/db/catalog/database_holder.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/jsobj.h"
#include "mongo/db/repl/replication_coordinator.h"
diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h
index f3a5a1393be..2640abefbcb 100644
--- a/src/mongo/db/catalog/database.h
+++ b/src/mongo/db/catalog/database.h
@@ -110,18 +110,13 @@ public:
* If we are applying a 'drop' oplog entry on a secondary, 'dropOpTime' will contain the optime
* of the oplog entry.
*
- * When fromMigrate is set, the related oplog entry will be marked with a 'fromMigrate' field to
- * reduce its visibility (e.g. in change streams).
- *
* The caller should hold a DB X lock and ensure there are no index builds in progress on the
* collection.
* N.B. Namespace argument is passed by value as it may otherwise disappear or change.
*/
virtual Status dropCollection(OperationContext* opCtx,
NamespaceString nss,
- repl::OpTime dropOpTime = {},
- bool markFromMigrate = false) const = 0;
-
+ repl::OpTime dropOpTime = {}) const = 0;
virtual Status dropCollectionEvenIfSystem(OperationContext* opCtx,
NamespaceString nss,
repl::OpTime dropOpTime = {},
diff --git a/src/mongo/db/catalog/database_holder_impl.cpp b/src/mongo/db/catalog/database_holder_impl.cpp
index f28a44bc0dd..28e44082c2a 100644
--- a/src/mongo/db/catalog/database_holder_impl.cpp
+++ b/src/mongo/db/catalog/database_holder_impl.cpp
@@ -37,7 +37,7 @@
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/collection_impl.h"
#include "mongo/db/catalog/database_impl.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/index_builds_coordinator.h"
#include "mongo/db/op_observer.h"
#include "mongo/db/operation_context.h"
@@ -220,7 +220,8 @@ void DatabaseHolderImpl::dropDb(OperationContext* opCtx, Database* db) {
invariant(opCtx->lockState()->isDbLockedForMode(name.dbName(), MODE_X));
auto catalog = CollectionCatalog::get(opCtx);
- for (auto&& coll : catalog->range(name)) {
+ for (auto collIt = catalog->begin(opCtx, name); collIt != catalog->end(opCtx); ++collIt) {
+ auto coll = *collIt;
if (!coll) {
break;
}
@@ -235,7 +236,8 @@ void DatabaseHolderImpl::dropDb(OperationContext* opCtx, Database* db) {
auto const serviceContext = opCtx->getServiceContext();
- for (auto&& coll : catalog->range(name)) {
+ for (auto collIt = catalog->begin(opCtx, name); collIt != catalog->end(opCtx); ++collIt) {
+ auto coll = *collIt;
if (!coll) {
break;
}
diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp
index 62f789ad003..077fc8d2313 100644
--- a/src/mongo/db/catalog/database_impl.cpp
+++ b/src/mongo/db/catalog/database_impl.cpp
@@ -299,7 +299,7 @@ void DatabaseImpl::clearTmpCollections(OperationContext* opCtx) const {
CollectionCatalog::CollectionInfoFn callback = [&](const CollectionPtr& collection) {
try {
WriteUnitOfWork wuow(opCtx);
- Status status = dropCollection(opCtx, collection->ns(), {}, false);
+ Status status = dropCollection(opCtx, collection->ns(), {});
if (!status.isOK()) {
LOGV2_WARNING(20327,
"could not drop temp collection '{namespace}': {error}",
@@ -438,8 +438,7 @@ Status DatabaseImpl::dropView(OperationContext* opCtx, NamespaceString viewName)
Status DatabaseImpl::dropCollection(OperationContext* opCtx,
NamespaceString nss,
- repl::OpTime dropOpTime,
- bool markFromMigrate) const {
+ repl::OpTime dropOpTime) const {
// Cannot drop uncommitted collections.
invariant(!UncommittedCatalogUpdates::isCreatedCollection(opCtx, nss));
@@ -475,7 +474,7 @@ Status DatabaseImpl::dropCollection(OperationContext* opCtx,
}
}
- return dropCollectionEvenIfSystem(opCtx, nss, dropOpTime, markFromMigrate);
+ return dropCollectionEvenIfSystem(opCtx, nss, dropOpTime);
}
Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx,
@@ -745,7 +744,7 @@ void DatabaseImpl::_checkCanCreateCollection(OperationContext* opCtx,
const CollectionOptions& options) const {
if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, nss)) {
if (options.isView()) {
- uasserted(ErrorCodes::NamespaceExists,
+ uasserted(17399,
str::stream()
<< "Cannot create collection " << nss << " - collection already exists.");
} else {
@@ -1090,10 +1089,6 @@ Status DatabaseImpl::userCreateNS(OperationContext* opCtx,
ExtensionsCallbackNoop(),
allowedFeatures);
- // Increment counters to track the usage of schema validators.
- validatorCounters.incrementCounters(
- "create", collectionOptions.validator, statusWithMatcher.isOK());
-
// We check the status of the parse to see if there are any banned features, but we don't
// actually need the result for now.
if (!statusWithMatcher.isOK()) {
diff --git a/src/mongo/db/catalog/database_impl.h b/src/mongo/db/catalog/database_impl.h
index c87f811c7d8..fa09177bc1b 100644
--- a/src/mongo/db/catalog/database_impl.h
+++ b/src/mongo/db/catalog/database_impl.h
@@ -62,20 +62,16 @@ public:
* If we are applying a 'drop' oplog entry on a secondary, 'dropOpTime' will contain the optime
* of the oplog entry.
*
- * When fromMigrate is set, the related oplog entry will be marked with a 'fromMigrate' field to
- * reduce its visibility (e.g. in change streams).
- *
* The caller should hold a DB X lock and ensure there are no index builds in progress on the
* collection.
*/
Status dropCollection(OperationContext* opCtx,
NamespaceString nss,
- repl::OpTime dropOpTime,
- bool markFromMigrate) const final;
+ repl::OpTime dropOpTime) const final;
Status dropCollectionEvenIfSystem(OperationContext* opCtx,
NamespaceString nss,
repl::OpTime dropOpTime,
- bool markFromMigrate) const final;
+ bool markFromMigrate = false) const final;
Status dropView(OperationContext* opCtx, NamespaceString viewName) const final;
diff --git a/src/mongo/db/catalog/database_test.cpp b/src/mongo/db/catalog/database_test.cpp
index c4669cd85b2..dd72c38588c 100644
--- a/src/mongo/db/catalog/database_test.cpp
+++ b/src/mongo/db/catalog/database_test.cpp
@@ -39,7 +39,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/client.h"
#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/jsobj.h"
@@ -261,7 +261,7 @@ void _testDropCollectionThrowsExceptionIfThereAreIndexesInProgress(OperationCont
collection->ns(), indexInfoObj, IndexBuildMethod::kHybrid, UUID::gen());
{
WriteUnitOfWork wuow(opCtx);
- ASSERT_OK(indexBuildBlock->init(opCtx, collection, /*forRecovery=*/false));
+ ASSERT_OK(indexBuildBlock->init(opCtx, collection));
wuow.commit();
}
ON_BLOCK_EXIT([&indexBuildBlock, opCtx, collection] {
diff --git a/src/mongo/db/catalog/drop_collection.cpp b/src/mongo/db/catalog/drop_collection.cpp
index 12e78022b2b..f04a8fa2b79 100644
--- a/src/mongo/db/catalog/drop_collection.cpp
+++ b/src/mongo/db/catalog/drop_collection.cpp
@@ -36,10 +36,9 @@
#include "mongo/db/audit.h"
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/collection_uuid_mismatch.h"
-#include "mongo/db/catalog/collection_uuid_mismatch_info.h"
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index_builds_coordinator.h"
@@ -57,16 +56,7 @@ namespace {
MONGO_FAIL_POINT_DEFINE(hangDropCollectionBeforeLockAcquisition);
MONGO_FAIL_POINT_DEFINE(hangDuringDropCollection);
-Status _checkNssAndReplState(OperationContext* opCtx,
- const CollectionPtr& coll,
- const NamespaceString& nss,
- const boost::optional<UUID>& expectedUUID = boost::none) {
- try {
- checkCollectionUUIDMismatch(opCtx, nss, coll, expectedUUID);
- } catch (const DBException& ex) {
- return ex.toStatus();
- }
-
+Status _checkNssAndReplState(OperationContext* opCtx, const CollectionPtr& coll) {
if (!coll) {
return Status(ErrorCodes::NamespaceNotFound, "ns not found");
}
@@ -205,13 +195,19 @@ Status _abortIndexBuildsAndDrop(OperationContext* opCtx,
CollectionPtr coll =
CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, startingNss);
- Status status = _checkNssAndReplState(opCtx, coll, startingNss, expectedUUID);
+ Status status = _checkNssAndReplState(opCtx, coll);
if (!status.isOK()) {
return status;
}
warnEncryptedCollectionsIfNeeded(opCtx, coll);
+ try {
+ checkCollectionUUIDMismatch(opCtx, startingNss, coll, expectedUUID);
+ } catch (const DBException& ex) {
+ return ex.toStatus();
+ }
+
if (MONGO_unlikely(hangDuringDropCollection.shouldFail())) {
LOGV2(518090,
"hangDuringDropCollection fail point enabled. Blocking until fail point is "
@@ -260,7 +256,7 @@ Status _abortIndexBuildsAndDrop(OperationContext* opCtx,
opCtx->recoveryUnit()->abandonSnapshot();
coll = CollectionCatalog::get(opCtx)->lookupCollectionByUUID(opCtx, collectionUUID);
- status = _checkNssAndReplState(opCtx, coll, startingNss, expectedUUID);
+ status = _checkNssAndReplState(opCtx, coll);
if (!status.isOK()) {
return status;
}
@@ -308,7 +304,7 @@ Status _dropCollectionForApplyOps(OperationContext* opCtx,
Lock::CollectionLock collLock(opCtx, collectionName, MODE_X);
const CollectionPtr& coll =
CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, collectionName);
- Status status = _checkNssAndReplState(opCtx, coll, collectionName);
+ Status status = _checkNssAndReplState(opCtx, coll);
if (!status.isOK()) {
return status;
}
@@ -352,7 +348,6 @@ Status _dropCollection(OperationContext* opCtx,
const boost::optional<UUID>& expectedUUID,
DropReply* reply,
DropCollectionSystemCollectionMode systemCollectionMode,
- bool fromMigrate,
boost::optional<UUID> dropIfUUIDNotMatching = boost::none) {
try {
@@ -360,13 +355,7 @@ Status _dropCollection(OperationContext* opCtx,
AutoGetDb autoDb(opCtx, collectionName.db(), MODE_IX);
auto db = autoDb.getDb();
if (!db) {
- return expectedUUID
- ? Status{CollectionUUIDMismatchInfo(collectionName.db().toString(),
- *expectedUUID,
- collectionName.coll().toString(),
- boost::none),
- "Database does not exist"}
- : Status(ErrorCodes::NamespaceNotFound, "ns not found");
+ return Status(ErrorCodes::NamespaceNotFound, "ns not found");
}
if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, collectionName)) {
@@ -375,14 +364,13 @@ Status _dropCollection(OperationContext* opCtx,
std::move(autoDb),
collectionName,
expectedUUID,
- [opCtx, systemCollectionMode, fromMigrate](Database* db,
- const NamespaceString& resolvedNs) {
+ [opCtx, systemCollectionMode](Database* db, const NamespaceString& resolvedNs) {
WriteUnitOfWork wuow(opCtx);
auto status = systemCollectionMode ==
DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops
- ? db->dropCollection(opCtx, resolvedNs, {}, fromMigrate)
- : db->dropCollectionEvenIfSystem(opCtx, resolvedNs, {}, fromMigrate);
+ ? db->dropCollection(opCtx, resolvedNs)
+ : db->dropCollectionEvenIfSystem(opCtx, resolvedNs);
if (!status.isOK()) {
return status;
}
@@ -395,18 +383,14 @@ Status _dropCollection(OperationContext* opCtx,
dropIfUUIDNotMatching);
}
- auto dropTimeseries = [opCtx,
- &expectedUUID,
- &autoDb,
- &collectionName,
- &reply,
- fromMigrate](const NamespaceString& bucketNs, bool dropView) {
+ auto dropTimeseries = [opCtx, &expectedUUID, &autoDb, &collectionName, &reply](
+ const NamespaceString& bucketNs, bool dropView) {
return _abortIndexBuildsAndDrop(
opCtx,
std::move(autoDb),
bucketNs,
expectedUUID,
- [opCtx, dropView, &expectedUUID, &collectionName, &reply, fromMigrate](
+ [opCtx, dropView, &expectedUUID, &collectionName, &reply](
Database* db, const NamespaceString& bucketsNs) {
// Disallow checking the expectedUUID when dropping time-series collections.
uassert(ErrorCodes::InvalidOptions,
@@ -430,13 +414,11 @@ Status _dropCollection(OperationContext* opCtx,
// Drop the buckets collection in its own writeConflictRetry so that if
// it throws a WCE, only the buckets collection drop is retried.
- writeConflictRetry(
- opCtx, "drop", bucketsNs.ns(), [opCtx, db, &bucketsNs, fromMigrate] {
- WriteUnitOfWork wuow(opCtx);
- db->dropCollectionEvenIfSystem(opCtx, bucketsNs, {}, fromMigrate)
- .ignore();
- wuow.commit();
- });
+ writeConflictRetry(opCtx, "drop", bucketsNs.ns(), [opCtx, db, &bucketsNs] {
+ WriteUnitOfWork wuow(opCtx);
+ db->dropCollectionEvenIfSystem(opCtx, bucketsNs).ignore();
+ wuow.commit();
+ });
return Status::OK();
},
@@ -486,8 +468,7 @@ Status dropCollection(OperationContext* opCtx,
const NamespaceString& nss,
const boost::optional<UUID>& expectedUUID,
DropReply* reply,
- DropCollectionSystemCollectionMode systemCollectionMode,
- bool fromMigrate) {
+ DropCollectionSystemCollectionMode systemCollectionMode) {
if (!serverGlobalParams.quiet.load()) {
LOGV2(518070, "CMD: drop", logAttrs(nss));
}
@@ -502,16 +483,14 @@ Status dropCollection(OperationContext* opCtx,
const auto collectionName =
nss.isTimeseriesBucketsCollection() ? nss.getTimeseriesViewNamespace() : nss;
- return _dropCollection(
- opCtx, collectionName, expectedUUID, reply, systemCollectionMode, fromMigrate);
+ return _dropCollection(opCtx, collectionName, expectedUUID, reply, systemCollectionMode);
}
Status dropCollection(OperationContext* opCtx,
const NamespaceString& nss,
DropReply* reply,
- DropCollectionSystemCollectionMode systemCollectionMode,
- bool fromMigrate) {
- return dropCollection(opCtx, nss, boost::none, reply, systemCollectionMode, fromMigrate);
+ DropCollectionSystemCollectionMode systemCollectionMode) {
+ return dropCollection(opCtx, nss, boost::none, reply, systemCollectionMode);
}
Status dropCollectionIfUUIDNotMatching(OperationContext* opCtx,
@@ -533,7 +512,6 @@ Status dropCollectionIfUUIDNotMatching(OperationContext* opCtx,
boost::none,
&repl,
DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops,
- false /*fromMigrate*/,
expectedUUID);
}
diff --git a/src/mongo/db/catalog/drop_collection.h b/src/mongo/db/catalog/drop_collection.h
index 016bc5e3d1a..7f857d88547 100644
--- a/src/mongo/db/catalog/drop_collection.h
+++ b/src/mongo/db/catalog/drop_collection.h
@@ -50,21 +50,17 @@ enum class DropCollectionSystemCollectionMode {
* Drops the collection "collectionName" and populates "reply" with statistics about what
* was removed. Aborts in-progress index builds on the collection if two phase index builds are
* supported. Throws if the expectedUUID does not match the UUID of the collection being dropped.
- * When fromMigrate is set, the related oplog entry will be marked accordingly using the
- * 'fromMigrate' field to reduce its visibility (e.g. in change streams).
*/
Status dropCollection(OperationContext* opCtx,
const NamespaceString& collectionName,
const boost::optional<UUID>& expectedUUID,
DropReply* reply,
- DropCollectionSystemCollectionMode systemCollectionMode,
- bool fromMigrate = false);
+ DropCollectionSystemCollectionMode systemCollectionMode);
Status dropCollection(OperationContext* opCtx,
const NamespaceString& collectionName,
DropReply* reply,
- DropCollectionSystemCollectionMode systemCollectionMode,
- bool fromMigrate = false);
+ DropCollectionSystemCollectionMode systemCollectionMode);
/**
* Drops the collection "collectionName" only if its uuid is not matching "expectedUUID".
diff --git a/src/mongo/db/catalog/drop_database.cpp b/src/mongo/db/catalog/drop_database.cpp
index 5e11a71dbfc..85e2087a321 100644
--- a/src/mongo/db/catalog/drop_database.cpp
+++ b/src/mongo/db/catalog/drop_database.cpp
@@ -38,7 +38,7 @@
#include "mongo/db/catalog/database_holder.h"
#include "mongo/db/catalog_raii.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/index_builds_coordinator.h"
#include "mongo/db/op_observer.h"
@@ -99,19 +99,17 @@ void _finishDropDatabase(OperationContext* opCtx,
IndexBuildsCoordinator::get(opCtx)->assertNoBgOpInProgForDb(dbName);
}
- // Testing depends on this failpoint stopping execution before the dropDatabase oplog entry is
- // written, as well as before the in-memory state is cleared.
- if (MONGO_unlikely(dropDatabaseHangBeforeInMemoryDrop.shouldFail())) {
- LOGV2(20334, "dropDatabase - fail point dropDatabaseHangBeforeInMemoryDrop enabled");
- dropDatabaseHangBeforeInMemoryDrop.pauseWhileSet();
- }
-
writeConflictRetry(opCtx, "dropDatabase_database", dbName, [&] {
WriteUnitOfWork wunit(opCtx);
opCtx->getServiceContext()->getOpObserver()->onDropDatabase(opCtx, dbName);
wunit.commit();
});
+ if (MONGO_unlikely(dropDatabaseHangBeforeInMemoryDrop.shouldFail())) {
+ LOGV2(20334, "dropDatabase - fail point dropDatabaseHangBeforeInMemoryDrop enabled");
+ dropDatabaseHangBeforeInMemoryDrop.pauseWhileSet();
+ }
+
auto databaseHolder = DatabaseHolder::get(opCtx);
databaseHolder->dropDb(opCtx, db);
dropPendingGuard.dismiss();
@@ -229,7 +227,9 @@ Status _dropDatabase(OperationContext* opCtx, const std::string& dbName, bool ab
std::vector<NamespaceString> collectionsToDrop;
auto catalog = CollectionCatalog::get(opCtx);
- for (auto&& collection : catalog->range(db->name())) {
+ for (auto collIt = catalog->begin(opCtx, db->name()); collIt != catalog->end(opCtx);
+ ++collIt) {
+ auto collection = *collIt;
if (!collection) {
break;
}
diff --git a/src/mongo/db/catalog/drop_database_test.cpp b/src/mongo/db/catalog/drop_database_test.cpp
index e14cf74b809..0e46980f33c 100644
--- a/src/mongo/db/catalog/drop_database_test.cpp
+++ b/src/mongo/db/catalog/drop_database_test.cpp
@@ -36,7 +36,7 @@
#include "mongo/db/catalog/database_holder.h"
#include "mongo/db/catalog/drop_database.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/jsobj.h"
#include "mongo/db/namespace_string.h"
diff --git a/src/mongo/db/catalog/drop_indexes.cpp b/src/mongo/db/catalog/drop_indexes.cpp
index 4720a7ee83c..2c10eb005c5 100644
--- a/src/mongo/db/catalog/drop_indexes.cpp
+++ b/src/mongo/db/catalog/drop_indexes.cpp
@@ -39,7 +39,7 @@
#include "mongo/db/catalog/collection_uuid_mismatch.h"
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -50,9 +50,7 @@
#include "mongo/db/s/collection_sharding_state.h"
#include "mongo/db/s/database_sharding_state.h"
#include "mongo/db/s/shard_key_index_util.h"
-#include "mongo/db/server_feature_flags_gen.h"
#include "mongo/db/service_context.h"
-#include "mongo/db/storage/storage_parameters_gen.h"
#include "mongo/logv2/log.h"
#include "mongo/util/visit_helper.h"
@@ -109,13 +107,9 @@ Status checkReplState(OperationContext* opCtx,
StatusWith<const IndexDescriptor*> getDescriptorByKeyPattern(OperationContext* opCtx,
const IndexCatalog* indexCatalog,
const BSONObj& keyPattern) {
+ const bool includeUnfinished = true;
std::vector<const IndexDescriptor*> indexes;
- indexCatalog->findIndexesByKeyPattern(opCtx,
- keyPattern,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen,
- &indexes);
+ indexCatalog->findIndexesByKeyPattern(opCtx, keyPattern, includeUnfinished, &indexes);
if (indexes.empty()) {
return Status(ErrorCodes::IndexNotFound,
str::stream() << "can't find index with key: " << keyPattern);
@@ -309,24 +303,12 @@ void dropReadyIndexes(OperationContext* opCtx,
if (desc->isIdIndex()) {
return false;
}
- // For any index that is compatible with the shard key, if
- // gFeatureFlagShardKeyIndexOptionalHashedSharding is enabled and
- // the shard key is hashed, allow users to drop the hashed index. Note
- // skipDroppingHashedShardKeyIndex is used in some tests to prevent dropIndexes
- // from dropping the hashed shard key index so we can continue to test chunk
- // migration with hashed sharding. Otherwise, dropIndexes with '*' would drop
- // the index and prevent chunk migration from running.
- const auto& shardKey = collDescription.getShardKeyPattern();
- const bool skipDropIndex = skipDroppingHashedShardKeyIndex ||
- !(gFeatureFlagShardKeyIndexOptionalHashedSharding.isEnabled(
- serverGlobalParams.featureCompatibility) &&
- shardKey.isHashedPattern());
+
if (isCompatibleWithShardKey(opCtx,
CollectionPtr(collection),
desc->getEntry(),
- shardKey.toBSON(),
- false /* requiresSingleKey */) &&
- skipDropIndex) {
+ collDescription.getKeyPattern(),
+ false /* requiresSingleKey */)) {
return false;
}
@@ -356,20 +338,17 @@ void dropReadyIndexes(OperationContext* opCtx,
return;
}
+ bool includeUnfinished = true;
for (const auto& indexName : indexNames) {
if (collDescription.isSharded()) {
uassert(
ErrorCodes::CannotDropShardKeyIndex,
"Cannot drop the only compatible index for this collection's shard key",
- !isLastNonHiddenRangedShardKeyIndex(
+ !isLastShardKeyIndex(
opCtx, collection, indexCatalog, indexName, collDescription.getKeyPattern()));
}
- auto desc = indexCatalog->findIndexByName(opCtx,
- indexName,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ auto desc = indexCatalog->findIndexByName(opCtx, indexName, includeUnfinished);
if (!desc) {
uasserted(ErrorCodes::IndexNotFound,
str::stream() << "index not found with name [" << indexName << "]");
@@ -528,6 +507,7 @@ DropIndexesReply dropIndexes(OperationContext* opCtx,
// the index catalog. This would indicate that while we yielded our locks during the
// abort phase, a new identical index was created.
auto indexCatalog = collection->getWritableCollection(opCtx)->getIndexCatalog();
+ const bool includeUnfinished = false;
for (const auto& indexName : indexNames) {
auto collDescription =
CollectionShardingState::get(opCtx, nss)->getCollectionDescription(opCtx);
@@ -535,19 +515,14 @@ DropIndexesReply dropIndexes(OperationContext* opCtx,
if (collDescription.isSharded()) {
uassert(ErrorCodes::CannotDropShardKeyIndex,
"Cannot drop the only compatible index for this collection's shard key",
- !isLastNonHiddenRangedShardKeyIndex(opCtx,
- collection->getCollection(),
- indexCatalog,
- indexName,
- collDescription.getKeyPattern()));
+ !isLastShardKeyIndex(opCtx,
+ collection->getCollection(),
+ indexCatalog,
+ indexName,
+ collDescription.getKeyPattern()));
}
- auto desc =
- indexCatalog->findIndexByName(opCtx,
- indexName,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ auto desc = indexCatalog->findIndexByName(opCtx, indexName, includeUnfinished);
if (!desc) {
// A similar index wasn't created while we yielded the locks during abort.
continue;
diff --git a/src/mongo/db/catalog/health_log.cpp b/src/mongo/db/catalog/health_log.cpp
index 9626d456111..aa49302f169 100644
--- a/src/mongo/db/catalog/health_log.cpp
+++ b/src/mongo/db/catalog/health_log.cpp
@@ -27,14 +27,19 @@
* it in the license file.
*/
+#include "mongo/platform/basic.h"
+
#include "mongo/db/catalog/health_log.h"
#include "mongo/db/catalog/health_log_gen.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
-#include "mongo/db/namespace_string.h"
namespace mongo {
namespace {
+const ServiceContext::Decoration<HealthLog> getHealthLog =
+ ServiceContext::declareDecoration<HealthLog>();
+
const int64_t kDefaultHealthlogSize = 100'000'000;
CollectionOptions getOptions(void) {
@@ -46,17 +51,24 @@ CollectionOptions getOptions(void) {
}
} // namespace
-HealthLog::HealthLog()
- : _writer(NamespaceString::kLocalHealthLogNamespace, getOptions(), kMaxBufferSize) {}
+HealthLog::HealthLog() : _writer(nss, getOptions(), kMaxBufferSize) {}
-void HealthLog::startup() {
+void HealthLog::startup(void) {
_writer.startup(std::string("healthlog writer"));
}
-void HealthLog::shutdown() {
+void HealthLog::shutdown(void) {
_writer.shutdown();
}
+HealthLog& HealthLog::get(ServiceContext* svcCtx) {
+ return getHealthLog(svcCtx);
+}
+
+HealthLog& HealthLog::get(OperationContext* opCtx) {
+ return getHealthLog(opCtx->getServiceContext());
+}
+
bool HealthLog::log(const HealthLogEntry& entry) {
BSONObjBuilder builder;
OID oid;
@@ -65,4 +77,6 @@ bool HealthLog::log(const HealthLogEntry& entry) {
entry.serialize(&builder);
return _writer.insertDocument(builder.obj());
}
+
+const NamespaceString HealthLog::nss("local", "system.healthlog");
} // namespace mongo
diff --git a/src/mongo/db/catalog/health_log.h b/src/mongo/db/catalog/health_log.h
index f9fe3c5b4c3..ba2bcbf440a 100644
--- a/src/mongo/db/catalog/health_log.h
+++ b/src/mongo/db/catalog/health_log.h
@@ -29,14 +29,21 @@
#pragma once
-#include "mongo/db/catalog/health_log_interface.h"
#include "mongo/db/concurrency/deferred_writer.h"
+#include "mongo/db/service_context.h"
namespace mongo {
class HealthLogEntry;
-class HealthLog : public HealthLogInterface {
+/**
+ * The interface to the local healthlog.
+ *
+ * This class contains facilities for creating and asynchronously writing to the local healthlog
+ * collection. There should only be one instance of this class, initialized on startup and cleaned
+ * up on shutdown.
+ */
+class HealthLog {
HealthLog(const HealthLog&) = delete;
HealthLog& operator=(const HealthLog&) = delete;
@@ -48,11 +55,38 @@ public:
*/
HealthLog();
- void startup() override;
+ /**
+ * The maximum size of the in-memory buffer of health-log entries, in bytes.
+ */
+ static const int64_t kMaxBufferSize = 25'000'000;
+
+ /**
+ * Start the worker thread writing the buffer to the collection.
+ */
+ void startup(void);
- void shutdown() override;
+ /**
+ * Stop the worker thread.
+ */
+ void shutdown(void);
- bool log(const HealthLogEntry& entry) override;
+ /**
+ * The name of the collection.
+ */
+ static const NamespaceString nss;
+
+ /**
+ * Get the current context's HealthLog.
+ */
+ static HealthLog& get(ServiceContext* ctx);
+ static HealthLog& get(OperationContext* ctx);
+
+ /**
+ * Asynchronously insert the given entry.
+ *
+ * Return `false` iff there is no more space in the buffer.
+ */
+ bool log(const HealthLogEntry& entry);
private:
DeferredWriter _writer;
diff --git a/src/mongo/db/catalog/health_log_interface.cpp b/src/mongo/db/catalog/health_log_interface.cpp
deleted file mode 100644
index 69e8f03020f..00000000000
--- a/src/mongo/db/catalog/health_log_interface.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/catalog/health_log_interface.h"
-#include "mongo/db/operation_context.h"
-
-namespace mongo {
-
-namespace {
-const auto getHealthLog = ServiceContext::declareDecoration<std::unique_ptr<HealthLogInterface>>();
-} // namespace
-
-void HealthLogInterface::set(ServiceContext* serviceContext,
- std::unique_ptr<HealthLogInterface> newHealthLog) {
- auto& healthLog = getHealthLog(serviceContext);
- invariant(!healthLog);
-
- healthLog = std::move(newHealthLog);
-}
-
-HealthLogInterface* HealthLogInterface::get(ServiceContext* svcCtx) {
- return getHealthLog(svcCtx).get();
-}
-
-HealthLogInterface* HealthLogInterface::get(OperationContext* opCtx) {
- return getHealthLog(opCtx->getServiceContext()).get();
-}
-} // namespace mongo
diff --git a/src/mongo/db/catalog/health_log_interface.h b/src/mongo/db/catalog/health_log_interface.h
deleted file mode 100644
index 885015fb6be..00000000000
--- a/src/mongo/db/catalog/health_log_interface.h
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Copyright (C) 2023-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/catalog/health_log_gen.h"
-#include "mongo/db/service_context.h"
-
-namespace mongo {
-
-/**
- * The interface to the local healthlog.
- *
- * This class contains facilities for creating and asynchronously writing to the local healthlog
- * collection. There should only be one instance of this class, initialized on startup and cleaned
- * up on shutdown.
- */
-class HealthLogInterface {
- HealthLogInterface(const HealthLogInterface&) = delete;
- HealthLogInterface& operator=(const HealthLogInterface&) = delete;
-
-public:
- /**
- * The maximum size of the in-memory buffer of health-log entries, in bytes.
- */
- static const int64_t kMaxBufferSize = 25'000'000;
-
- /**
- * Stores a health log on the specified service context. May only be called once for the
- * lifetime of the service context.
- */
- static void set(ServiceContext* serviceContext,
- std::unique_ptr<HealthLogInterface> newHealthLog);
-
- /**
- * Get the current context's HealthLog. set() above must be called before any get() calls.
- */
- static HealthLogInterface* get(ServiceContext* ctx);
- static HealthLogInterface* get(OperationContext* ctx);
-
- /**
- * Required to use HealthLogInterface as a ServiceContext decorator.
- *
- * Should not be used anywhere else.
- */
- HealthLogInterface() = default;
- virtual ~HealthLogInterface() = default;
-
- /**
- * Start the worker thread writing the buffer to the collection.
- */
- virtual void startup() = 0;
-
- /**
- * Stop the worker thread.
- */
- virtual void shutdown() = 0;
-
- /**
- * Asynchronously insert the given entry.
- *
- * Return `false` iff there is no more space in the buffer.
- */
- virtual bool log(const HealthLogEntry& entry) = 0;
-};
-} // namespace mongo
diff --git a/src/mongo/db/catalog/index_build_block.cpp b/src/mongo/db/catalog/index_build_block.cpp
index 34dbf0c49ad..6174e891b64 100644
--- a/src/mongo/db/catalog/index_build_block.cpp
+++ b/src/mongo/db/catalog/index_build_block.cpp
@@ -44,6 +44,7 @@
#include "mongo/db/query/collection_index_usage_tracker_decoration.h"
#include "mongo/db/query/collection_query_info.h"
#include "mongo/db/storage/durable_catalog.h"
+#include "mongo/db/storage/storage_parameters_gen.h"
#include "mongo/db/ttl_collection_cache.h"
#include "mongo/db/vector_clock.h"
#include "mongo/logv2/log.h"
@@ -57,7 +58,14 @@ IndexBuildBlock::IndexBuildBlock(const NamespaceString& nss,
const BSONObj& spec,
IndexBuildMethod method,
boost::optional<UUID> indexBuildUUID)
- : _nss(nss), _spec(spec.getOwned()), _method(method), _buildUUID(indexBuildUUID) {}
+ : _nss(nss),
+ _spec(spec.getOwned()),
+ _method(method),
+ _buildUUID(indexBuildUUID),
+ _pooledBuilder(
+ gOperationMemoryPoolBlockInitialSizeKB.loadRelaxed() * static_cast<size_t>(1024),
+ SharedBufferFragmentBuilder::DoubleGrowStrategy(
+ gOperationMemoryPoolBlockMaxSizeKB.loadRelaxed() * static_cast<size_t>(1024))) {}
void IndexBuildBlock::keepTemporaryTables() {
if (_indexBuildInterceptor) {
@@ -75,11 +83,6 @@ void IndexBuildBlock::_completeInit(OperationContext* opCtx, Collection* collect
.registerIndex(desc->indexName(),
desc->keyPattern(),
IndexFeatures::make(desc, collection->ns().isOnInternalDb()));
- opCtx->recoveryUnit()->onRollback(
- [collectionDecorations = collection->getSharedDecorations(), indexName = _indexName] {
- CollectionIndexUsageTrackerDecoration::get(collectionDecorations)
- .unregisterIndex(indexName);
- });
}
Status IndexBuildBlock::initForResume(OperationContext* opCtx,
@@ -89,9 +92,7 @@ Status IndexBuildBlock::initForResume(OperationContext* opCtx,
_indexName = _spec.getStringField("name").toString();
auto descriptor = collection->getIndexCatalog()->findIndexByName(
- opCtx,
- _indexName,
- IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ opCtx, _indexName, true /* includeUnfinishedIndexes */);
auto indexCatalogEntry = descriptor->getEntry();
@@ -127,7 +128,7 @@ Status IndexBuildBlock::initForResume(OperationContext* opCtx,
return Status::OK();
}
-Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection, bool forRecovery) {
+Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection) {
// Being in a WUOW means all timestamping responsibility can be pushed up to the caller.
invariant(opCtx->lockState()->inAWriteUnitOfWork());
@@ -155,25 +156,14 @@ Status IndexBuildBlock::init(OperationContext* opCtx, Collection* collection, bo
!replCoord->getMemberState().primary() && isBackgroundIndex;
}
- if (!forRecovery) {
- // Setup on-disk structures. We skip this during startup recovery for unfinished indexes as
- // everything is already in-place.
- Status status = collection->prepareForIndexBuild(
- opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild);
- if (!status.isOK())
- return status;
- }
+ // Setup on-disk structures.
+ Status status = collection->prepareForIndexBuild(
+ opCtx, descriptor.get(), _buildUUID, isBackgroundSecondaryBuild);
+ if (!status.isOK())
+ return status;
- auto indexCatalog = collection->getIndexCatalog();
- IndexCatalogEntry* indexCatalogEntry = nullptr;
- if (forRecovery) {
- auto desc = indexCatalog->findIndexByName(
- opCtx, _indexName, IndexCatalog::InclusionPolicy::kUnfinished);
- indexCatalogEntry = desc->getEntry();
- } else {
- indexCatalogEntry = indexCatalog->createIndexEntry(
- opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone);
- }
+ auto indexCatalogEntry = collection->getIndexCatalog()->createIndexEntry(
+ opCtx, collection, std::move(descriptor), CreateIndexEntryFlags::kNone);
if (_method == IndexBuildMethod::kHybrid) {
_indexBuildInterceptor = std::make_unique<IndexBuildInterceptor>(opCtx, indexCatalogEntry);
@@ -284,10 +274,7 @@ void IndexBuildBlock::success(OperationContext* opCtx, Collection* collection) {
// Note that TTL deletion is supported on capped clustered collections via bounded
// collection scan, which does not use an index.
if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) && !coll->isCapped()) {
- TTLCollectionCache::get(svcCtx).registerTTLInfo(
- coll->uuid(),
- TTLCollectionCache::Info{
- indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()});
+ TTLCollectionCache::get(svcCtx).registerTTLInfo(coll->uuid(), indexName);
}
});
}
@@ -295,18 +282,14 @@ void IndexBuildBlock::success(OperationContext* opCtx, Collection* collection) {
const IndexCatalogEntry* IndexBuildBlock::getEntry(OperationContext* opCtx,
const CollectionPtr& collection) const {
auto descriptor = collection->getIndexCatalog()->findIndexByName(
- opCtx,
- _indexName,
- IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ opCtx, _indexName, true /* includeUnfinishedIndexes */);
return descriptor->getEntry();
}
IndexCatalogEntry* IndexBuildBlock::getEntry(OperationContext* opCtx, Collection* collection) {
auto descriptor = collection->getIndexCatalog()->findIndexByName(
- opCtx,
- _indexName,
- IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished);
+ opCtx, _indexName, true /* includeUnfinishedIndexes */);
return descriptor->getEntry();
}
diff --git a/src/mongo/db/catalog/index_build_block.h b/src/mongo/db/catalog/index_build_block.h
index 2b94849b46c..48d0f6e49af 100644
--- a/src/mongo/db/catalog/index_build_block.h
+++ b/src/mongo/db/catalog/index_build_block.h
@@ -62,7 +62,7 @@ public:
*
* Must be called from within a `WriteUnitOfWork`
*/
- Status init(OperationContext* opCtx, Collection* collection, bool forRecovery);
+ Status init(OperationContext* opCtx, Collection* collection);
/**
* Makes sure that an entry for the index was created at startup in the IndexCatalog. Returns
@@ -111,6 +111,13 @@ public:
return _spec;
}
+ /**
+ * Returns a memory pool for creating temporary objects for this index build.
+ */
+ SharedBufferFragmentBuilder& getPooledBuilder() {
+ return _pooledBuilder;
+ }
+
private:
void _completeInit(OperationContext* opCtx, Collection* collection);
@@ -124,5 +131,7 @@ private:
std::string _indexNamespace;
std::unique_ptr<IndexBuildInterceptor> _indexBuildInterceptor;
+
+ SharedBufferFragmentBuilder _pooledBuilder;
};
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_build_entry_test.cpp b/src/mongo/db/catalog/index_build_entry_test.cpp
index d53c7e0046c..ffa6a870874 100644
--- a/src/mongo/db/catalog/index_build_entry_test.cpp
+++ b/src/mongo/db/catalog/index_build_entry_test.cpp
@@ -32,7 +32,6 @@
#include <string>
#include <vector>
-#include "mongo/bson/bson_validate.h"
#include "mongo/bson/bsonobj.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/bson/bsontypes.h"
@@ -125,7 +124,7 @@ TEST(IndexBuildEntryTest, SerializeAndDeserialize) {
entry.setCommitReadyMembers(generateCommitReadyMembers(3));
BSONObj obj = entry.toBSON();
- ASSERT_TRUE(validateBSON(obj).isOK());
+ ASSERT_TRUE(obj.valid());
IDLParserErrorContext ctx("IndexBuildsEntry Parser");
IndexBuildEntry rebuiltEntry = IndexBuildEntry::parse(ctx, obj);
diff --git a/src/mongo/db/catalog/index_builds_manager.cpp b/src/mongo/db/catalog/index_builds_manager.cpp
index 0c44c44de48..82483f9cf21 100644
--- a/src/mongo/db/catalog/index_builds_manager.cpp
+++ b/src/mongo/db/catalog/index_builds_manager.cpp
@@ -38,7 +38,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_repair.h"
#include "mongo/db/catalog_raii.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/operation_context.h"
#include "mongo/db/storage/storage_repair_observer.h"
@@ -113,8 +113,7 @@ Status IndexBuildsManager::setUpIndexBuild(OperationContext* opCtx,
std::vector<BSONObj> indexes;
try {
indexes = writeConflictRetry(opCtx, "IndexBuildsManager::setUpIndexBuild", nss.ns(), [&]() {
- return uassertStatusOK(
- builder->init(opCtx, collection, specs, onInit, options.forRecovery, resumeInfo));
+ return uassertStatusOK(builder->init(opCtx, collection, specs, onInit, resumeInfo));
});
} catch (const DBException& ex) {
return ex.toStatus();
diff --git a/src/mongo/db/catalog/index_builds_manager.h b/src/mongo/db/catalog/index_builds_manager.h
index 37614751eb5..cddb46c3d16 100644
--- a/src/mongo/db/catalog/index_builds_manager.h
+++ b/src/mongo/db/catalog/index_builds_manager.h
@@ -73,7 +73,6 @@ public:
IndexConstraints indexConstraints = IndexConstraints::kEnforce;
IndexBuildProtocol protocol = IndexBuildProtocol::kSinglePhase;
IndexBuildMethod method = IndexBuildMethod::kHybrid;
- bool forRecovery = false;
};
IndexBuildsManager() = default;
diff --git a/src/mongo/db/catalog/index_catalog.h b/src/mongo/db/catalog/index_catalog.h
index 6f4fa0c8ae3..2754ac3e22f 100644
--- a/src/mongo/db/catalog/index_catalog.h
+++ b/src/mongo/db/catalog/index_catalog.h
@@ -197,12 +197,6 @@ public:
std::unique_ptr<std::vector<IndexCatalogEntry*>> _ownedContainer;
};
- enum class InclusionPolicy {
- kReady = 1 << 0,
- kUnfinished = 1 << 1,
- kFrozen = 1 << 2,
- };
-
IndexCatalog() = default;
virtual ~IndexCatalog() = default;
@@ -243,10 +237,9 @@ public:
*
* @return null if cannot find
*/
- virtual const IndexDescriptor* findIndexByName(
- OperationContext* opCtx,
- StringData name,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0;
+ virtual const IndexDescriptor* findIndexByName(OperationContext* opCtx,
+ StringData name,
+ bool includeUnfinishedIndexes = false) const = 0;
/**
* Find index by matching key pattern and options. The key pattern, collation spec, and partial
@@ -258,7 +251,7 @@ public:
OperationContext* opCtx,
const BSONObj& key,
const BSONObj& indexSpec,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0;
+ bool includeUnfinishedIndexes = false) const = 0;
/**
* Find indexes with a matching key pattern, putting them into the vector 'matches'. The key
@@ -268,13 +261,12 @@ public:
*/
virtual void findIndexesByKeyPattern(OperationContext* opCtx,
const BSONObj& key,
- InclusionPolicy inclusionPolicy,
+ bool includeUnfinishedIndexes,
std::vector<const IndexDescriptor*>* matches) const = 0;
- virtual void findIndexByType(
- OperationContext* opCtx,
- const std::string& type,
- std::vector<const IndexDescriptor*>& matches,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const = 0;
+ virtual void findIndexByType(OperationContext* opCtx,
+ const std::string& type,
+ std::vector<const IndexDescriptor*>& matches,
+ bool includeUnfinishedIndexes = false) const = 0;
/**
* Reload the index definition for 'oldDesc' from the CollectionCatalogEntry. 'oldDesc'
@@ -316,7 +308,7 @@ public:
* Returns an iterator for the index descriptors in this IndexCatalog.
*/
virtual std::unique_ptr<IndexIterator> getIndexIterator(
- OperationContext* opCtx, InclusionPolicy inclusionPolicy) const = 0;
+ OperationContext* opCtx, bool includeUnfinishedIndexes) const = 0;
// ---- index set modifiers ------
@@ -419,16 +411,6 @@ public:
const IndexDescriptor* desc) = 0;
/**
- * Resets the index given its descriptor.
- *
- * This can only be called during startup recovery as it involves recreating the index table to
- * allow bulk cursors to be used again.
- */
- virtual Status resetUnfinishedIndexForRecovery(OperationContext* opCtx,
- Collection* collection,
- const IndexDescriptor* desc) = 0;
-
- /**
* Drops an unfinished index given its descriptor.
*
* The caller must hold the collection X lock.
@@ -539,16 +521,4 @@ public:
Collection* coll,
IndexCatalogEntry* index) = 0;
};
-
-inline IndexCatalog::InclusionPolicy operator|(IndexCatalog::InclusionPolicy lhs,
- IndexCatalog::InclusionPolicy rhs) {
- return static_cast<IndexCatalog::InclusionPolicy>(
- static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(lhs) |
- static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(rhs));
-}
-
-inline bool operator&(IndexCatalog::InclusionPolicy lhs, IndexCatalog::InclusionPolicy rhs) {
- return static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(lhs) &
- static_cast<std::underlying_type_t<IndexCatalog::InclusionPolicy>>(rhs);
-}
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h
index 2cf80bb8d3f..fa77370bb96 100644
--- a/src/mongo/db/catalog/index_catalog_entry.h
+++ b/src/mongo/db/catalog/index_catalog_entry.h
@@ -64,6 +64,8 @@ public:
inline IndexCatalogEntry(IndexCatalogEntry&&) = delete;
inline IndexCatalogEntry& operator=(IndexCatalogEntry&&) = delete;
+ virtual void init(std::unique_ptr<IndexAccessMethod> accessMethod) = 0;
+
virtual const std::string& getIdent() const = 0;
virtual std::shared_ptr<Ident> getSharedIdent() const = 0;
@@ -73,8 +75,6 @@ public:
virtual IndexAccessMethod* accessMethod() const = 0;
- virtual void setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) = 0;
-
virtual bool isHybridBuilding() const = 0;
virtual IndexBuildInterceptor* indexBuildInterceptor() const = 0;
@@ -95,7 +95,6 @@ public:
/// ---------------------
virtual void setIsReady(bool newIsReady) = 0;
- virtual void setIsFrozen(bool newIsFrozen) = 0;
virtual void setDropped() = 0;
virtual bool isDropped() const = 0;
diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.cpp b/src/mongo/db/catalog/index_catalog_entry_impl.cpp
index e104623ef2d..daf74714c84 100644
--- a/src/mongo/db/catalog/index_catalog_entry_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_entry_impl.cpp
@@ -39,7 +39,7 @@
#include "mongo/base/init.h"
#include "mongo/db/catalog/collection.h"
#include "mongo/db/concurrency/d_concurrency.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/index/index_access_method.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/matcher/expression.h"
@@ -122,7 +122,7 @@ IndexCatalogEntryImpl::IndexCatalogEntryImpl(OperationContext* const opCtx,
}
}
-void IndexCatalogEntryImpl::setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) {
+void IndexCatalogEntryImpl::init(std::unique_ptr<IndexAccessMethod> accessMethod) {
invariant(!_accessMethod);
_accessMethod = std::move(accessMethod);
}
@@ -179,10 +179,6 @@ void IndexCatalogEntryImpl::setIsReady(bool newIsReady) {
_isReady = newIsReady;
}
-void IndexCatalogEntryImpl::setIsFrozen(bool newIsFrozen) {
- _isFrozen = newIsFrozen;
-}
-
void IndexCatalogEntryImpl::setMultikey(OperationContext* opCtx,
const CollectionPtr& collection,
const KeyStringSet& multikeyMetadataKeys,
@@ -374,8 +370,7 @@ Status IndexCatalogEntryImpl::_setMultikeyInMultiDocumentTransaction(
}
std::shared_ptr<Ident> IndexCatalogEntryImpl::getSharedIdent() const {
- return _accessMethod ? std::shared_ptr<Ident>{shared_from_this(), _accessMethod->getIdentPtr()}
- : nullptr;
+ return {shared_from_this(), _accessMethod->getIdentPtr()}; // aliasing constructor
}
// ----
diff --git a/src/mongo/db/catalog/index_catalog_entry_impl.h b/src/mongo/db/catalog/index_catalog_entry_impl.h
index 0760989b99a..821176164e1 100644
--- a/src/mongo/db/catalog/index_catalog_entry_impl.h
+++ b/src/mongo/db/catalog/index_catalog_entry_impl.h
@@ -61,6 +61,8 @@ public:
std::unique_ptr<IndexDescriptor> descriptor, // ownership passes to me
bool isFrozen);
+ void init(std::unique_ptr<IndexAccessMethod> accessMethod) final;
+
const std::string& getIdent() const final {
return _ident;
}
@@ -78,8 +80,6 @@ public:
return _accessMethod.get();
}
- void setAccessMethod(std::unique_ptr<IndexAccessMethod> accessMethod) final;
-
bool isHybridBuilding() const final {
return _indexBuildInterceptor != nullptr;
}
@@ -110,8 +110,6 @@ public:
void setIsReady(bool newIsReady) final;
- void setIsFrozen(bool newIsFrozen) final;
-
void setDropped() final {
_isDropped.store(true);
}
diff --git a/src/mongo/db/catalog/index_catalog_impl.cpp b/src/mongo/db/catalog/index_catalog_impl.cpp
index 2c3086a54cf..6733dac0ba1 100644
--- a/src/mongo/db/catalog/index_catalog_impl.cpp
+++ b/src/mongo/db/catalog/index_catalog_impl.cpp
@@ -47,6 +47,7 @@
#include "mongo/db/catalog/uncommitted_catalog_updates.h"
#include "mongo/db/client.h"
#include "mongo/db/clientcursor.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/field_ref.h"
#include "mongo/db/fts/fts_spec.h"
@@ -204,30 +205,13 @@ Status IndexCatalogImpl::init(OperationContext* opCtx, Collection* collection) {
auto descriptor = std::make_unique<IndexDescriptor>(_getAccessMethodName(keyPattern), spec);
- // TTL indexes with NaN 'expireAfterSeconds' cause problems in multiversion settings.
- if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName)) {
- if (spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()) {
- LOGV2_OPTIONS(6852200,
- {logv2::LogTag::kStartupWarnings},
- "Found an existing TTL index with NaN 'expireAfterSeconds' in the "
- "catalog.",
- "ns"_attr = collection->ns(),
- "uuid"_attr = collection->uuid(),
- "index"_attr = indexName,
- "spec"_attr = spec);
- }
- }
-
// TTL indexes are not compatible with capped collections.
// Note that TTL deletion is supported on capped clustered collections via bounded
// collection scan, which does not use an index.
if (spec.hasField(IndexDescriptor::kExpireAfterSecondsFieldName) &&
!collection->isCapped()) {
TTLCollectionCache::get(opCtx->getServiceContext())
- .registerTTLInfo(
- collection->uuid(),
- TTLCollectionCache::Info{
- indexName, spec[IndexDescriptor::kExpireAfterSecondsFieldName].isNaN()});
+ .registerTTLInfo(collection->uuid(), indexName);
}
bool ready = collection->isIndexReady(indexName);
@@ -273,8 +257,8 @@ Status IndexCatalogImpl::init(OperationContext* opCtx, Collection* collection) {
}
std::unique_ptr<IndexCatalog::IndexIterator> IndexCatalogImpl::getIndexIterator(
- OperationContext* const opCtx, InclusionPolicy inclusionPolicy) const {
- if (inclusionPolicy == InclusionPolicy::kReady) {
+ OperationContext* const opCtx, const bool includeUnfinishedIndexes) const {
+ if (!includeUnfinishedIndexes) {
// If the caller only wants the ready indexes, we return an iterator over the catalog's
// ready indexes vector. When the user advances this iterator, it will filter out any
// indexes that were not ready at the OperationContext's read timestamp.
@@ -282,28 +266,17 @@ std::unique_ptr<IndexCatalog::IndexIterator> IndexCatalogImpl::getIndexIterator(
opCtx, _readyIndexes.begin(), _readyIndexes.end());
}
- // If the caller doesn't only want the ready indexes, for simplicity of implementation, we copy
- // the pointers to a new vector. The vector's ownership is passed to the iterator. The query
- // code path from an external client is not expected to hit this case so the cost isn't paid by
- // the important code path.
+ // If the caller wants all indexes, for simplicity of implementation, we copy the pointers to
+ // a new vector. The vector's ownership is passed to the iterator. The query code path from an
+ // external client is not expected to hit this case so the cost isn't paid by the important
+ // code path.
auto allIndexes = std::make_unique<std::vector<IndexCatalogEntry*>>();
-
- if (inclusionPolicy & InclusionPolicy::kReady) {
- for (auto it = _readyIndexes.begin(); it != _readyIndexes.end(); ++it) {
- allIndexes->push_back(it->get());
- }
+ for (auto it = _readyIndexes.begin(); it != _readyIndexes.end(); ++it) {
+ allIndexes->push_back(it->get());
}
- if (inclusionPolicy & InclusionPolicy::kUnfinished) {
- for (auto it = _buildingIndexes.begin(); it != _buildingIndexes.end(); ++it) {
- allIndexes->push_back(it->get());
- }
- }
-
- if (inclusionPolicy & InclusionPolicy::kFrozen) {
- for (auto it = _frozenIndexes.begin(); it != _frozenIndexes.end(); ++it) {
- allIndexes->push_back(it->get());
- }
+ for (auto it = _buildingIndexes.begin(); it != _buildingIndexes.end(); ++it) {
+ allIndexes->push_back(it->get());
}
return std::make_unique<AllIndexesIterator>(opCtx, std::move(allIndexes));
@@ -377,7 +350,6 @@ void IndexCatalogImpl::_logInternalState(OperationContext* opCtx,
"numIndexesInCollectionCatalogEntry"_attr = numIndexesInCollectionCatalogEntry,
"numReadyIndexes"_attr = _readyIndexes.size(),
"numBuildingIndexes"_attr = _buildingIndexes.size(),
- "numFrozenIndexes"_attr = _frozenIndexes.size(),
"indexNamesToDrop"_attr = indexNamesToDrop);
// Report the ready indexes.
@@ -453,8 +425,7 @@ StatusWith<BSONObj> IndexCatalogImpl::prepareSpecForCreate(
}
// First check against only the ready indexes for conflicts.
- status =
- _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, InclusionPolicy::kReady);
+ status = _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, false);
if (!status.isOK()) {
return status;
}
@@ -470,12 +441,7 @@ StatusWith<BSONObj> IndexCatalogImpl::prepareSpecForCreate(
// The index catalog cannot currently iterate over only in-progress indexes. So by previously
// checking against only ready indexes without error, we know that any errors encountered
// checking against all indexes occurred due to an in-progress index.
- status = _doesSpecConflictWithExisting(opCtx,
- collection,
- validatedSpec,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ status = _doesSpecConflictWithExisting(opCtx, collection, validatedSpec, true);
if (!status.isOK()) {
if (ErrorCodes::IndexAlreadyExists == status.code()) {
// Callers need to be able to distinguish conflicts against ready indexes versus
@@ -504,11 +470,8 @@ std::vector<BSONObj> IndexCatalogImpl::removeExistingIndexesNoChecks(
// _doesSpecConflictWithExisting currently does more work than we require here: we are only
// interested in the index already exists error.
if (ErrorCodes::IndexAlreadyExists ==
- _doesSpecConflictWithExisting(opCtx,
- collection,
- spec,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished)) {
+ _doesSpecConflictWithExisting(
+ opCtx, collection, spec, true /*includeUnfinishedIndexes*/)) {
continue;
}
@@ -571,20 +534,19 @@ IndexCatalogEntry* IndexCatalogImpl::createIndexEntry(OperationContext* opCtx,
engine->getEngine()->alterIdentMetadata(opCtx, ident, desc, isForceUpdateMetadata);
}
- if (!frozen) {
- const auto& collOptions = collection->getCollectionOptions();
- std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface(
- opCtx, collection->ns(), collOptions, ident, desc);
- std::unique_ptr<IndexAccessMethod> accessMethod =
- IndexAccessMethod::make(entry.get(), std::move(sdi));
- entry->setAccessMethod(std::move(accessMethod));
- }
+ const auto& collOptions = collection->getCollectionOptions();
+ std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface(
+ opCtx, collection->ns(), collOptions, ident, desc);
+
+ std::unique_ptr<IndexAccessMethod> accessMethod =
+ IndexAccessMethodFactory::get(opCtx)->make(entry.get(), std::move(sdi));
+
+ entry->init(std::move(accessMethod));
+
IndexCatalogEntry* save = entry.get();
if (isReadyIndex) {
_readyIndexes.add(std::move(entry));
- } else if (frozen) {
- _frozenIndexes.add(std::move(entry));
} else {
_buildingIndexes.add(std::move(entry));
}
@@ -623,7 +585,7 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont
boost::optional<UUID> buildUUID = boost::none;
IndexBuildBlock indexBuildBlock(
collection->ns(), spec, IndexBuildMethod::kForeground, buildUUID);
- status = indexBuildBlock.init(opCtx, collection, /*forRecovery=*/false);
+ status = indexBuildBlock.init(opCtx, collection);
if (!status.isOK())
return status;
@@ -911,11 +873,10 @@ Status IndexCatalogImpl::_isSpecOk(OperationContext* opCtx,
}
const std::unique_ptr<MatchExpression> filterExpr = std::move(statusWithMatcher.getValue());
- Status status = _checkValidFilterExpressions(
- filterExpr.get(),
- !serverGlobalParams.featureCompatibility.isVersionInitialized() ||
- feature_flags::gTimeseriesMetricIndexes.isEnabled(
- serverGlobalParams.featureCompatibility));
+ Status status =
+ _checkValidFilterExpressions(filterExpr.get(),
+ feature_flags::gTimeseriesMetricIndexes.isEnabled(
+ serverGlobalParams.featureCompatibility));
if (!status.isOK()) {
return status;
}
@@ -990,7 +951,7 @@ Status IndexCatalogImpl::_isSpecOk(OperationContext* opCtx,
Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx,
const CollectionPtr& collection,
const BSONObj& spec,
- InclusionPolicy inclusionPolicy) const {
+ const bool includeUnfinishedIndexes) const {
StringData name = spec.getStringField(IndexDescriptor::kIndexNameFieldName);
invariant(name[0]);
@@ -1004,7 +965,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx,
{
// Check whether an index with the specified candidate name already exists in the catalog.
- const IndexDescriptor* desc = findIndexByName(opCtx, name, inclusionPolicy);
+ const IndexDescriptor* desc = findIndexByName(opCtx, name, includeUnfinishedIndexes);
if (desc) {
// Index already exists with same name. Check whether the options are the same as well.
@@ -1057,7 +1018,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx,
{
// No index with the candidate name exists. Check for an index with conflicting options.
const IndexDescriptor* desc =
- findIndexByKeyPatternAndOptions(opCtx, key, spec, inclusionPolicy);
+ findIndexByKeyPatternAndOptions(opCtx, key, spec, includeUnfinishedIndexes);
if (desc) {
LOGV2_DEBUG(20353,
@@ -1108,7 +1069,7 @@ Status IndexCatalogImpl::_doesSpecConflictWithExisting(OperationContext* opCtx,
string pluginName = IndexNames::findPluginName(key);
if (pluginName == IndexNames::TEXT) {
vector<const IndexDescriptor*> textIndexes;
- findIndexByType(opCtx, IndexNames::TEXT, textIndexes, inclusionPolicy);
+ findIndexByType(opCtx, IndexNames::TEXT, textIndexes, includeUnfinishedIndexes);
if (textIndexes.size() > 0) {
return Status(ErrorCodes::CannotCreateIndex,
str::stream() << "only one text index per collection allowed, "
@@ -1149,10 +1110,7 @@ void IndexCatalogImpl::dropIndexes(OperationContext* opCtx,
vector<string> indexNamesToDrop;
{
int seen = 0;
- auto ii = getIndexIterator(opCtx,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, true);
while (ii->more()) {
seen++;
const IndexDescriptor* desc = ii->next()->descriptor();
@@ -1167,11 +1125,7 @@ void IndexCatalogImpl::dropIndexes(OperationContext* opCtx,
for (size_t i = 0; i < indexNamesToDrop.size(); i++) {
string indexName = indexNamesToDrop[i];
- const IndexDescriptor* desc = findIndexByName(
- opCtx,
- indexName,
- IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ const IndexDescriptor* desc = findIndexByName(opCtx, indexName, true);
invariant(desc);
LOGV2_DEBUG(20355, 1, "\t dropAllIndexes dropping: {desc}", "desc"_attr = *desc);
IndexCatalogEntry* entry = desc->getEntry();
@@ -1230,73 +1184,6 @@ Status IndexCatalogImpl::dropIndex(OperationContext* opCtx,
return dropIndexEntry(opCtx, collection, entry);
}
-Status IndexCatalogImpl::resetUnfinishedIndexForRecovery(OperationContext* opCtx,
- Collection* collection,
- const IndexDescriptor* desc) {
- invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X));
- invariant(opCtx->lockState()->inAWriteUnitOfWork());
-
- IndexCatalogEntry* entry = desc->getEntry();
- const std::string indexName = entry->descriptor()->indexName();
-
- // Only indexes that aren't ready can be reset.
- invariant(!collection->isIndexReady(indexName));
-
- auto released = [&] {
- if (auto released = _readyIndexes.release(entry->descriptor())) {
- invariant(!released, "Cannot reset a ready index");
- }
- if (auto released = _buildingIndexes.release(entry->descriptor())) {
- return released;
- }
- if (auto released = _frozenIndexes.release(entry->descriptor())) {
- return released;
- }
- MONGO_UNREACHABLE;
- }();
-
- LOGV2(6987700,
- "Resetting unfinished index",
- logAttrs(collection->ns()),
- "index"_attr = indexName,
- "ident"_attr = released->getIdent());
-
- invariant(released.get() == entry);
-
- // Drop the ident if it exists. The storage engine will return OK if the ident is not found.
- auto engine = opCtx->getServiceContext()->getStorageEngine();
- const std::string ident = released->getIdent();
- Status status = engine->getEngine()->dropIdent(opCtx->recoveryUnit(), ident);
- if (!status.isOK()) {
- return status;
- }
-
- // Recreate the ident on-disk. DurableCatalog::createIndex() will lookup the ident internally
- // using the catalogId and index name.
- status = DurableCatalog::get(opCtx)->createIndex(opCtx,
- collection->getCatalogId(),
- collection->ns(),
- collection->getCollectionOptions(),
- released->descriptor());
- if (!status.isOK()) {
- return status;
- }
-
- // Update the index entry state in preparation to rebuild the index.
- if (!released->accessMethod()) {
- std::unique_ptr<SortedDataInterface> sdi = engine->getEngine()->getSortedDataInterface(
- opCtx, collection->ns(), collection->getCollectionOptions(), ident, desc);
- std::unique_ptr<IndexAccessMethod> accessMethod =
- IndexAccessMethod::make(released.get(), std::move(sdi));
- released->setAccessMethod(std::move(accessMethod));
- }
-
- released->setIsFrozen(false);
- _buildingIndexes.add(std::move(released));
-
- return Status::OK();
-}
-
Status IndexCatalogImpl::dropUnfinishedIndex(OperationContext* opCtx,
Collection* collection,
const IndexDescriptor* desc) {
@@ -1364,26 +1251,25 @@ Status IndexCatalogImpl::dropIndexEntry(OperationContext* opCtx,
audit::logDropIndex(opCtx->getClient(), indexName, collection->ns());
- auto released = [&] {
- if (auto released = _readyIndexes.release(entry->descriptor())) {
- return released;
- }
- if (auto released = _buildingIndexes.release(entry->descriptor())) {
- return released;
- }
- if (auto released = _frozenIndexes.release(entry->descriptor())) {
- return released;
- }
- MONGO_UNREACHABLE;
- }();
-
- invariant(released.get() == entry);
- opCtx->recoveryUnit()->registerChange(
- std::make_unique<IndexRemoveChange>(opCtx,
- collection->ns(),
- collection->uuid(),
- std::move(released),
- collection->getSharedDecorations()));
+ auto released = _readyIndexes.release(entry->descriptor());
+ if (released) {
+ invariant(released.get() == entry);
+ opCtx->recoveryUnit()->registerChange(
+ std::make_unique<IndexRemoveChange>(opCtx,
+ collection->ns(),
+ collection->uuid(),
+ std::move(released),
+ collection->getSharedDecorations()));
+ } else {
+ released = _buildingIndexes.release(entry->descriptor());
+ invariant(released.get() == entry);
+ opCtx->recoveryUnit()->registerChange(
+ std::make_unique<IndexRemoveChange>(opCtx,
+ collection->ns(),
+ collection->uuid(),
+ std::move(released),
+ collection->getSharedDecorations()));
+ }
CollectionQueryInfo::get(collection).rebuildIndexData(opCtx, collection);
CollectionIndexUsageTrackerDecoration::get(collection->getSharedDecorations())
@@ -1403,11 +1289,7 @@ void IndexCatalogImpl::_deleteIndexFromDisk(OperationContext* opCtx,
Collection* collection,
const string& indexName,
std::shared_ptr<Ident> ident) {
- invariant(!findIndexByName(opCtx,
- indexName,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen));
+ invariant(!findIndexByName(opCtx, indexName, true /* includeUnfinishedIndexes*/));
catalog::removeIndex(opCtx, indexName, collection, std::move(ident));
}
@@ -1437,7 +1319,7 @@ int IndexCatalogImpl::numIndexesTotal(OperationContext* opCtx) const {
int IndexCatalogImpl::numIndexesReady(OperationContext* opCtx) const {
std::vector<const IndexDescriptor*> itIndexes;
- auto ii = getIndexIterator(opCtx, InclusionPolicy::kReady);
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, /*includeUnfinished*/ false);
while (ii->more()) {
itIndexes.push_back(ii->next()->descriptor());
}
@@ -1449,7 +1331,7 @@ bool IndexCatalogImpl::haveIdIndex(OperationContext* opCtx) const {
}
const IndexDescriptor* IndexCatalogImpl::findIdIndex(OperationContext* opCtx) const {
- auto ii = getIndexIterator(opCtx, InclusionPolicy::kReady);
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, false);
while (ii->more()) {
const IndexDescriptor* desc = ii->next()->descriptor();
if (desc->isIdIndex())
@@ -1460,8 +1342,8 @@ const IndexDescriptor* IndexCatalogImpl::findIdIndex(OperationContext* opCtx) co
const IndexDescriptor* IndexCatalogImpl::findIndexByName(OperationContext* opCtx,
StringData name,
- InclusionPolicy inclusionPolicy) const {
- auto ii = getIndexIterator(opCtx, inclusionPolicy);
+ bool includeUnfinishedIndexes) const {
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes);
while (ii->more()) {
const IndexDescriptor* desc = ii->next()->descriptor();
if (desc->indexName() == name)
@@ -1474,8 +1356,8 @@ const IndexDescriptor* IndexCatalogImpl::findIndexByKeyPatternAndOptions(
OperationContext* opCtx,
const BSONObj& key,
const BSONObj& indexSpec,
- InclusionPolicy inclusionPolicy) const {
- auto ii = getIndexIterator(opCtx, inclusionPolicy);
+ bool includeUnfinishedIndexes) const {
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes);
IndexDescriptor needle(_getAccessMethodName(key), indexSpec);
while (ii->more()) {
const auto* entry = ii->next();
@@ -1489,10 +1371,10 @@ const IndexDescriptor* IndexCatalogImpl::findIndexByKeyPatternAndOptions(
void IndexCatalogImpl::findIndexesByKeyPattern(OperationContext* opCtx,
const BSONObj& key,
- InclusionPolicy inclusionPolicy,
+ bool includeUnfinishedIndexes,
std::vector<const IndexDescriptor*>* matches) const {
invariant(matches);
- auto ii = getIndexIterator(opCtx, inclusionPolicy);
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes);
while (ii->more()) {
const IndexDescriptor* desc = ii->next()->descriptor();
if (SimpleBSONObjComparator::kInstance.evaluate(desc->keyPattern() == key)) {
@@ -1504,8 +1386,8 @@ void IndexCatalogImpl::findIndexesByKeyPattern(OperationContext* opCtx,
void IndexCatalogImpl::findIndexByType(OperationContext* opCtx,
const string& type,
vector<const IndexDescriptor*>& matches,
- InclusionPolicy inclusionPolicy) const {
- auto ii = getIndexIterator(opCtx, inclusionPolicy);
+ bool includeUnfinishedIndexes) const {
+ std::unique_ptr<IndexIterator> ii = getIndexIterator(opCtx, includeUnfinishedIndexes);
while (ii->more()) {
const IndexDescriptor* desc = ii->next()->descriptor();
if (IndexNames::findPluginName(desc->keyPattern()) == type) {
@@ -1746,10 +1628,7 @@ Status IndexCatalogImpl::indexRecords(OperationContext* opCtx,
for (const MultikeyPathInfo& newPath : newPaths) {
invariant(newPath.nss == coll->ns());
- auto idx = findIndexByName(opCtx,
- newPath.indexName,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished);
+ auto idx = findIndexByName(opCtx, newPath.indexName, /*includeUnfinishedIndexes=*/true);
if (!idx) {
return Status(ErrorCodes::IndexNotFound,
str::stream()
@@ -1852,10 +1731,7 @@ Status IndexCatalogImpl::compactIndexes(OperationContext* opCtx) const {
}
std::string::size_type IndexCatalogImpl::getLongestIndexNameLength(OperationContext* opCtx) const {
- auto it = getIndexIterator(opCtx,
- IndexCatalog::InclusionPolicy::kReady |
- IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ std::unique_ptr<IndexIterator> it = getIndexIterator(opCtx, true);
std::string::size_type longestIndexNameLength = 0;
while (it->more()) {
auto thisLength = it->next()->descriptor()->indexName().length();
@@ -1903,11 +1779,7 @@ void IndexCatalogImpl::indexBuildSuccess(OperationContext* opCtx,
invariant(releasedEntry.get() == index);
_readyIndexes.add(std::move(releasedEntry));
- // Wait to unset the interceptor until the index actually commits. If a write conflict is
- // encountered and the index commit process is restated, the multikey information from the
- // interceptor may still be needed.
- opCtx->recoveryUnit()->onCommit(
- [index](boost::optional<Timestamp>) { index->setIndexBuildInterceptor(nullptr); });
+ index->setIndexBuildInterceptor(nullptr);
index->setIsReady(true);
}
diff --git a/src/mongo/db/catalog/index_catalog_impl.h b/src/mongo/db/catalog/index_catalog_impl.h
index 223399e5bf0..259732da61c 100644
--- a/src/mongo/db/catalog/index_catalog_impl.h
+++ b/src/mongo/db/catalog/index_catalog_impl.h
@@ -96,10 +96,9 @@ public:
*
* @return null if cannot find
*/
- const IndexDescriptor* findIndexByName(
- OperationContext* opCtx,
- StringData name,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override;
+ const IndexDescriptor* findIndexByName(OperationContext* opCtx,
+ StringData name,
+ bool includeUnfinishedIndexes = false) const override;
/**
* Find index by matching key pattern and options. The key pattern, collation spec, and partial
@@ -111,7 +110,7 @@ public:
OperationContext* opCtx,
const BSONObj& key,
const BSONObj& indexSpec,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override;
+ bool includeUnfinishedIndexes = false) const override;
/**
* Find indexes with a matching key pattern, putting them into the vector 'matches'. The key
@@ -121,12 +120,12 @@ public:
*/
void findIndexesByKeyPattern(OperationContext* opCtx,
const BSONObj& key,
- InclusionPolicy inclusionPolicy,
+ bool includeUnfinishedIndexes,
std::vector<const IndexDescriptor*>* matches) const override;
void findIndexByType(OperationContext* opCtx,
const std::string& type,
std::vector<const IndexDescriptor*>& matches,
- InclusionPolicy inclusionPolicy = InclusionPolicy::kReady) const override;
+ bool includeUnfinishedIndexes = false) const override;
/**
@@ -154,7 +153,7 @@ public:
using IndexIterator = IndexCatalog::IndexIterator;
std::unique_ptr<IndexIterator> getIndexIterator(OperationContext* opCtx,
- InclusionPolicy inclusionPolicy) const override;
+ bool includeUnfinishedIndexes) const override;
// ---- index set modifiers ------
@@ -200,9 +199,6 @@ public:
Status dropIndex(OperationContext* opCtx,
Collection* collection,
const IndexDescriptor* desc) override;
- Status resetUnfinishedIndexForRecovery(OperationContext* opCtx,
- Collection* collection,
- const IndexDescriptor* desc) override;
Status dropUnfinishedIndex(OperationContext* opCtx,
Collection* collection,
const IndexDescriptor* desc) override;
@@ -381,7 +377,7 @@ private:
Status _doesSpecConflictWithExisting(OperationContext* opCtx,
const CollectionPtr& collection,
const BSONObj& spec,
- InclusionPolicy inclusionPolicy) const;
+ bool includeUnfinishedIndexes) const;
/**
* Returns true if the replica set member's config has {buildIndexes:false} set, which means
@@ -397,6 +393,5 @@ private:
IndexCatalogEntryContainer _readyIndexes;
IndexCatalogEntryContainer _buildingIndexes;
- IndexCatalogEntryContainer _frozenIndexes;
};
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_consistency.cpp b/src/mongo/db/catalog/index_consistency.cpp
index 64b2dfe40ba..55e35b744f8 100644
--- a/src/mongo/db/catalog/index_consistency.cpp
+++ b/src/mongo/db/catalog/index_consistency.cpp
@@ -38,7 +38,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_repair.h"
#include "mongo/db/catalog/validate_gen.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_access_method.h"
#include "mongo/db/index/index_descriptor.h"
@@ -106,7 +106,7 @@ IndexConsistency::IndexConsistency(OperationContext* opCtx,
void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, IndexInfo* indexInfo) {
auto hash = _hashKeyString(ks, indexInfo->indexNameHash);
- if (MONGO_unlikely(_validateState->logDiagnostics())) {
+ if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
LOGV2(6208500,
"[validate](multikeyMetadataPath) Adding with the hash",
"hash"_attr = hash,
@@ -118,7 +118,7 @@ void IndexConsistency::addMultikeyMetadataPath(const KeyString::Value& ks, Index
void IndexConsistency::removeMultikeyMetadataPath(const KeyString::Value& ks,
IndexInfo* indexInfo) {
auto hash = _hashKeyString(ks, indexInfo->indexNameHash);
- if (MONGO_unlikely(_validateState->logDiagnostics())) {
+ if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
LOGV2(6208501,
"[validate](multikeyMetadataPath) Removing with the hash",
"hash"_attr = hash,
@@ -132,26 +132,9 @@ size_t IndexConsistency::getMultikeyMetadataPathCount(IndexInfo* indexInfo) {
}
bool IndexConsistency::haveEntryMismatch() const {
- bool haveMismatch =
- std::any_of(_indexKeyBuckets.begin(),
- _indexKeyBuckets.end(),
- [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; });
-
- if (haveMismatch && _validateState->logDiagnostics()) {
- for (size_t i = 0; i < _indexKeyBuckets.size(); i++) {
- if (_indexKeyBuckets[i].indexKeyCount == 0) {
- continue;
- }
-
- LOGV2(7404500,
- "[validate](bucket entry mismatch)",
- "hash"_attr = i,
- "indexKeyCount"_attr = _indexKeyBuckets[i].indexKeyCount,
- "bucketBytesSize"_attr = _indexKeyBuckets[i].bucketSizeBytes);
- }
- }
-
- return haveMismatch;
+ return std::any_of(_indexKeyBuckets.begin(),
+ _indexKeyBuckets.end(),
+ [](const IndexKeyBucket& bucket) -> bool { return bucket.indexKeyCount; });
}
void IndexConsistency::setSecondPhase() {
@@ -209,7 +192,7 @@ void IndexConsistency::repairMissingIndexEntries(OperationContext* opCtx,
}
}
-void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResults* results) {
+void IndexConsistency::addIndexEntryErrors(ValidateResults* results) {
invariant(!_firstPhase);
// We'll report up to 1MB for extra index entry errors and missing index entry errors.
@@ -223,26 +206,11 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
numExtraIndexEntryErrors += item.second.size();
}
- // Sort missing index entries by size so we can process in order of increasing size and return
- // as many as possible within memory limits.
- using MissingIt = decltype(_missingIndexEntries)::const_iterator;
- std::vector<MissingIt> missingIndexEntriesBySize;
- missingIndexEntriesBySize.reserve(_missingIndexEntries.size());
- for (auto it = _missingIndexEntries.begin(); it != _missingIndexEntries.end(); ++it) {
- missingIndexEntriesBySize.push_back(it);
- }
- std::sort(missingIndexEntriesBySize.begin(),
- missingIndexEntriesBySize.end(),
- [](const MissingIt& a, const MissingIt& b) {
- return a->second.keyString.getSize() < b->second.keyString.getSize();
- });
-
// Inform which indexes have inconsistencies and add the BSON objects of the inconsistent index
// entries to the results vector.
bool missingIndexEntrySizeLimitWarning = false;
- bool first = true;
- for (const auto& missingIndexEntry : missingIndexEntriesBySize) {
- const IndexEntryInfo& entryInfo = missingIndexEntry->second;
+ for (const auto& missingIndexEntry : _missingIndexEntries) {
+ const IndexEntryInfo& entryInfo = missingIndexEntry.second;
KeyString::Value ks = entryInfo.keyString;
auto indexKey =
KeyString::toBsonSafe(ks.getBuffer(), ks.getSize(), entryInfo.ord, ks.getTypeBits());
@@ -253,9 +221,8 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
entryInfo.idKey);
numMissingIndexEntriesSizeBytes += entry.objsize();
- if (first || numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) {
+ if (numMissingIndexEntriesSizeBytes <= kErrorSizeBytes) {
results->missingIndexEntries.push_back(entry);
- first = false;
} else if (!missingIndexEntrySizeLimitWarning) {
StringBuilder ss;
ss << "Not all missing index entry inconsistencies are listed due to size limitations.";
@@ -264,8 +231,6 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
missingIndexEntrySizeLimitWarning = true;
}
- _printMetadata(opCtx, results, entryInfo);
-
std::string indexName = entry["indexName"].String();
if (!results->indexResultsMap.at(indexName).valid) {
continue;
@@ -278,58 +243,33 @@ void IndexConsistency::addIndexEntryErrors(OperationContext* opCtx, ValidateResu
results->indexResultsMap.at(indexName).valid = false;
}
- // Sort extra index entries by size so we can process in order of increasing size and return as
- // many as possible within memory limits.
- using ExtraIt = SimpleBSONObjSet::const_iterator;
- std::vector<ExtraIt> extraIndexEntriesBySize;
- // Since the extra entries are stored in a map of sets, we have to iterate the entries in the
- // map and sum the size of the sets in order to get the total number. Given that we can have at
- // most 64 indexes per collection, and the total number of entries could potentially be in the
- // millions, we expect that iterating the map will be much less costly than the additional
- // allocations and copies that could result from not calling 'reserve' on the vector.
- size_t totalExtraIndexEntriesCount =
- std::accumulate(_extraIndexEntries.begin(),
- _extraIndexEntries.end(),
- 0,
- [](size_t total, const std::pair<IndexKey, SimpleBSONObjSet>& set) {
- return total + set.second.size();
- });
- extraIndexEntriesBySize.reserve(totalExtraIndexEntriesCount);
+ bool extraIndexEntrySizeLimitWarning = false;
for (const auto& extraIndexEntry : _extraIndexEntries) {
const SimpleBSONObjSet& entries = extraIndexEntry.second;
- for (auto it = entries.begin(); it != entries.end(); ++it) {
- extraIndexEntriesBySize.push_back(it);
- }
- }
- std::sort(extraIndexEntriesBySize.begin(),
- extraIndexEntriesBySize.end(),
- [](const ExtraIt& a, const ExtraIt& b) { return a->objsize() < b->objsize(); });
+ for (const auto& entry : entries) {
+ numExtraIndexEntriesSizeBytes += entry.objsize();
+ if (numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) {
+ results->extraIndexEntries.push_back(entry);
+ } else if (!extraIndexEntrySizeLimitWarning) {
+ StringBuilder ss;
+ ss << "Not all extra index entry inconsistencies are listed due to size "
+ "limitations.";
+ results->errors.push_back(ss.str());
+
+ extraIndexEntrySizeLimitWarning = true;
+ }
+
+ std::string indexName = entry["indexName"].String();
+ if (!results->indexResultsMap.at(indexName).valid) {
+ continue;
+ }
- bool extraIndexEntrySizeLimitWarning = false;
- for (const auto& entry : extraIndexEntriesBySize) {
- numExtraIndexEntriesSizeBytes += entry->objsize();
- if (first || numExtraIndexEntriesSizeBytes <= kErrorSizeBytes) {
- results->extraIndexEntries.push_back(*entry);
- first = false;
- } else if (!extraIndexEntrySizeLimitWarning) {
StringBuilder ss;
- ss << "Not all extra index entry inconsistencies are listed due to size "
- "limitations.";
+ ss << "Index with name '" << indexName << "' has inconsistencies.";
results->errors.push_back(ss.str());
- extraIndexEntrySizeLimitWarning = true;
+ results->indexResultsMap.at(indexName).valid = false;
}
-
- std::string indexName = (*entry)["indexName"].String();
- if (!results->indexResultsMap.at(indexName).valid) {
- continue;
- }
-
- StringBuilder ss;
- ss << "Index with name '" << indexName << "' has inconsistencies.";
- results->errors.push_back(ss.str());
-
- results->indexResultsMap.at(indexName).valid = false;
}
// Inform how many inconsistencies were detected.
@@ -362,8 +302,7 @@ void IndexConsistency::addDocumentMultikeyPaths(IndexInfo* indexInfo,
void IndexConsistency::addDocKey(OperationContext* opCtx,
const KeyString::Value& ks,
IndexInfo* indexInfo,
- RecordId recordId,
- ValidateResults* results) {
+ RecordId recordId) {
auto rawHash = ks.hash(indexInfo->indexNameHash);
auto hashLower = rawHash % kNumHashBuckets;
auto hashUpper = (rawHash / kNumHashBuckets) % kNumHashBuckets;
@@ -379,7 +318,7 @@ void IndexConsistency::addDocKey(OperationContext* opCtx,
upper.bucketSizeBytes += ks.getSize();
indexInfo->numRecords++;
- if (MONGO_unlikely(_validateState->logDiagnostics())) {
+ if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
LOGV2(4666602,
"[validate](record) Adding with hashes",
"hashUpper"_attr = hashUpper,
@@ -409,6 +348,9 @@ void IndexConsistency::addDocKey(OperationContext* opCtx,
invariant(_missingIndexEntries.count(key) == 0);
_missingIndexEntries.insert(
std::make_pair(key, IndexEntryInfo(*indexInfo, recordId, idKeyBuilder.obj(), ks)));
+
+ // Prints the collection document's metadata.
+ _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx, recordId);
}
}
@@ -432,7 +374,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
upper.bucketSizeBytes += ks.getSize();
indexInfo->numKeys++;
- if (MONGO_unlikely(_validateState->logDiagnostics())) {
+ if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
LOGV2(4666603,
"[validate](index) Adding with hashes",
"hashUpper"_attr = hashUpper,
@@ -481,12 +423,9 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
SimpleBSONObjSet infoSet = {info};
_extraIndexEntries.insert(std::make_pair(key, infoSet));
- // Prints the collection document's and index entry's metadata.
- _validateState->getCollection()->getRecordStore()->printRecordMetadata(
- opCtx, recordId, &(results->recordTimestamps));
- indexInfo->accessMethod->asSortedData()
- ->getSortedDataInterface()
- ->printIndexEntryMetadata(opCtx, ks);
+ // Prints the collection document's metadata.
+ _validateState->getCollection()->getRecordStore()->printRecordMetadata(opCtx,
+ recordId);
return;
}
search->second.insert(info);
@@ -499,8 +438,7 @@ void IndexConsistency::addIndexKey(OperationContext* opCtx,
bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) {
invariant(!_firstPhase);
- const uint64_t maxMemoryUsageBytes =
- static_cast<uint64_t>(maxValidateMemoryUsageMB.load()) * 1024 * 1024;
+ const uint32_t maxMemoryUsageBytes = maxValidateMemoryUsageMB.load() * 1024 * 1024;
const uint64_t totalMemoryNeededBytes =
std::accumulate(_indexKeyBuckets.begin(),
_indexKeyBuckets.end(),
@@ -515,57 +453,48 @@ bool IndexConsistency::limitMemoryUsageForSecondPhase(ValidateResults* result) {
return true;
}
- // At this point we know we'll exceed the memory limit, and will pare back some of the buckets.
- // First we'll see what the smallest bucket is, and if that's over the limit by itself, then
- // we can zero out all the other buckets. Otherwise we'll keep as many buckets as we can.
+ bool hasNonZeroBucket = false;
+ uint64_t memoryUsedSoFarBytes = 0;
+ uint32_t smallestBucketBytes = std::numeric_limits<uint32_t>::max();
+ // Zero out any nonzero buckets that would put us over maxMemoryUsageBytes.
+ std::for_each(_indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
+ if (bucket.indexKeyCount == 0) {
+ return;
+ }
- auto smallestBucketWithAnInconsistency = std::min_element(
- _indexKeyBuckets.begin(),
- _indexKeyBuckets.end(),
- [](const IndexKeyBucket& lhs, const IndexKeyBucket& rhs) {
- if (lhs.indexKeyCount != 0) {
- return rhs.indexKeyCount == 0 || lhs.bucketSizeBytes < rhs.bucketSizeBytes;
- }
- return false;
- });
- invariant(smallestBucketWithAnInconsistency->indexKeyCount != 0);
-
- if (smallestBucketWithAnInconsistency->bucketSizeBytes > maxMemoryUsageBytes) {
- // We're going to just keep the smallest bucket, and zero everything else.
- std::for_each(
- _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
- if (&bucket == &(*smallestBucketWithAnInconsistency)) {
- // We keep the smallest bucket.
- return;
- }
+ smallestBucketBytes = std::min(smallestBucketBytes, bucket.bucketSizeBytes);
+ if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) {
+ // Including this bucket would put us over the memory limit, so zero this bucket. We
+ // don't want to keep any entry that will exceed the memory limit in the second phase so
+ // we don't double the 'maxMemoryUsageBytes' here.
+ bucket.indexKeyCount = 0;
+ return;
+ }
+ memoryUsedSoFarBytes += bucket.bucketSizeBytes;
+ hasNonZeroBucket = true;
+ });
- bucket.indexKeyCount = 0;
- });
- } else {
- // We're going to scan through the buckets and keep as many as we can.
- std::uint32_t memoryUsedSoFarBytes = 0;
- std::for_each(
- _indexKeyBuckets.begin(), _indexKeyBuckets.end(), [&](IndexKeyBucket& bucket) {
- if (bucket.indexKeyCount == 0) {
- return;
- }
+ StringBuilder memoryLimitMessage;
+ memoryLimitMessage << "Memory limit for validation is currently set to "
+ << maxValidateMemoryUsageMB.load()
+ << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter.";
- if (bucket.bucketSizeBytes + memoryUsedSoFarBytes > maxMemoryUsageBytes) {
- // Including this bucket would put us over the memory limit, so zero this
- // bucket. We don't want to keep any entry that will exceed the memory limit in
- // the second phase so we don't double the 'maxMemoryUsageBytes' here.
- bucket.indexKeyCount = 0;
- return;
- }
- memoryUsedSoFarBytes += bucket.bucketSizeBytes;
- });
+ if (!hasNonZeroBucket) {
+ const uint32_t minMemoryNeededMB = (smallestBucketBytes / (1024 * 1024)) + 1;
+ StringBuilder ss;
+ ss << "Unable to report index entry inconsistencies due to memory limitations. Need at "
+ "least "
+ << minMemoryNeededMB << "MB to report at least one index entry inconsistency. "
+ << memoryLimitMessage.str();
+ result->errors.push_back(ss.str());
+ result->valid = false;
+
+ return false;
}
StringBuilder ss;
- ss << "Not all index entry inconsistencies are reported due to memory limitations. Memory "
- "limit for validation is currently set to "
- << maxValidateMemoryUsageMB.load()
- << "MB and can be configured via the 'maxValidateMemoryUsageMB' parameter.";
+ ss << "Not all index entry inconsistencies are reported due to memory limitations. "
+ << memoryLimitMessage.str();
result->errors.push_back(ss.str());
result->valid = false;
@@ -611,16 +540,4 @@ uint32_t IndexConsistency::_hashKeyString(const KeyString::Value& ks,
uint32_t indexNameHash) const {
return ks.hash(indexNameHash);
}
-
-void IndexConsistency::_printMetadata(OperationContext* opCtx,
- ValidateResults* results,
- const IndexEntryInfo& entryInfo) {
- _validateState->getCollection()->getRecordStore()->printRecordMetadata(
- opCtx, entryInfo.recordId, &(results->recordTimestamps));
- getIndexInfo(entryInfo.indexName)
- .accessMethod->asSortedData()
- ->getSortedDataInterface()
- ->printIndexEntryMetadata(opCtx, entryInfo.keyString);
-}
-
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_consistency.h b/src/mongo/db/catalog/index_consistency.h
index f4eab27660b..dab1d2d8d97 100644
--- a/src/mongo/db/catalog/index_consistency.h
+++ b/src/mongo/db/catalog/index_consistency.h
@@ -108,8 +108,7 @@ public:
void addDocKey(OperationContext* opCtx,
const KeyString::Value& ks,
IndexInfo* indexInfo,
- RecordId recordId,
- ValidateResults* results);
+ RecordId recordId);
/**
* During the first phase of validation, given the index entry's KeyString, decrement the
@@ -169,7 +168,7 @@ public:
* Records the errors gathered from the second phase of index validation into the provided
* ValidateResultsMap and ValidateResults.
*/
- void addIndexEntryErrors(OperationContext* opCtx, ValidateResults* results);
+ void addIndexEntryErrors(ValidateResults* results);
/**
* Sets up this IndexConsistency object to limit memory usage in the second phase of index
@@ -180,8 +179,8 @@ public:
private:
struct IndexKeyBucket {
- uint32_t indexKeyCount = 0;
- uint32_t bucketSizeBytes = 0;
+ uint32_t indexKeyCount;
+ uint32_t bucketSizeBytes;
};
IndexConsistency() = delete;
@@ -242,11 +241,5 @@ private:
*/
uint32_t _hashKeyString(const KeyString::Value& ks, uint32_t indexNameHash) const;
- /**
- * Prints the collection document's and index entry's metadata.
- */
- void _printMetadata(OperationContext* opCtx,
- ValidateResults* results,
- const IndexEntryInfo& info);
}; // IndexConsistency
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_key_validate.cpp b/src/mongo/db/catalog/index_key_validate.cpp
index 20dd9f37fc0..d6f70219594 100644
--- a/src/mongo/db/catalog/index_key_validate.cpp
+++ b/src/mongo/db/catalog/index_key_validate.cpp
@@ -58,7 +58,7 @@
namespace mongo {
namespace index_key_validate {
-std::function<void(std::map<StringData, std::set<IndexType>>&)> filterAllowedIndexFieldNames;
+std::function<void(std::set<StringData>&)> filterAllowedIndexFieldNames;
using IndexVersion = IndexDescriptor::IndexVersion;
@@ -68,10 +68,6 @@ namespace {
// specification.
MONGO_FAIL_POINT_DEFINE(skipIndexCreateFieldNameValidation);
-// When the skipTTLIndexNaNExpireAfterSecondsValidation failpoint is enabled, validation for
-// TTL index 'expireAfterSeconds' will be disabled.
-MONGO_FAIL_POINT_DEFINE(skipTTLIndexNaNExpireAfterSecondsValidation);
-
static const std::set<StringData> allowedIdIndexFieldNames = {
IndexDescriptor::kCollationFieldName,
IndexDescriptor::kIndexNameFieldName,
@@ -108,16 +104,12 @@ Status isIndexVersionAllowedForCreation(IndexVersion indexVersion, const BSONObj
BSONObj buildRepairedIndexSpec(
const NamespaceString& ns,
const BSONObj& indexSpec,
- const std::map<StringData, std::set<IndexType>>& allowedFieldNames,
+ const std::set<StringData>& allowedFieldNames,
std::function<void(const BSONElement&, BSONObjBuilder*)> indexSpecHandleFn) {
- const auto key = indexSpec.getObjectField(IndexDescriptor::kKeyPatternFieldName);
- const auto indexName = IndexNames::nameToType(IndexNames::findPluginName(key));
BSONObjBuilder builder;
for (const auto& indexSpecElem : indexSpec) {
StringData fieldName = indexSpecElem.fieldNameStringData();
- auto it = allowedFieldNames.find(fieldName);
- if (it != allowedFieldNames.end() &&
- (it->second.empty() || it->second.count(indexName) != 0)) {
+ if (allowedFieldNames.count(fieldName)) {
indexSpecHandleFn(indexSpecElem, &builder);
} else {
LOGV2_WARNING(23878,
@@ -271,9 +263,9 @@ BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec)
BSONObj repairIndexSpec(const NamespaceString& ns,
const BSONObj& indexSpec,
- const std::map<StringData, std::set<IndexType>>& allowedFieldNames) {
- auto fixIndexSpecFn = [&indexSpec, &ns](const BSONElement& indexSpecElem,
- BSONObjBuilder* builder) {
+ const std::set<StringData>& allowedFieldNames) {
+ auto fixBoolIndexSpecFn = [&indexSpec, &ns](const BSONElement& indexSpecElem,
+ BSONObjBuilder* builder) {
StringData fieldName = indexSpecElem.fieldNameStringData();
if ((IndexDescriptor::kBackgroundFieldName == fieldName ||
IndexDescriptor::kUniqueFieldName == fieldName ||
@@ -287,28 +279,17 @@ BSONObj repairIndexSpec(const NamespaceString& ns,
"fieldName"_attr = redact(fieldName),
"indexSpec"_attr = redact(indexSpec));
builder->appendBool(fieldName, true);
- } else if (IndexDescriptor::kExpireAfterSecondsFieldName == fieldName &&
- !(indexSpecElem.isNumber() && !indexSpecElem.isNaN())) {
- LOGV2_WARNING(6835900,
- "Fixing expire field from TTL index spec",
- "namespace"_attr = redact(ns.toString()),
- "fieldName"_attr = redact(fieldName),
- "indexSpec"_attr = redact(indexSpec));
- builder->appendNumber(fieldName,
- durationCount<Seconds>(kExpireAfterSecondsForInactiveTTLIndex));
} else {
builder->append(indexSpecElem);
}
};
-
- return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, fixIndexSpecFn);
+ return buildRepairedIndexSpec(ns, indexSpec, allowedFieldNames, fixBoolIndexSpecFn);
}
StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& indexSpec) {
bool hasKeyPatternField = false;
bool hasIndexNameField = false;
bool hasNamespaceField = false;
- bool isTTLIndexWithNaNExpireAfterSeconds = false;
bool hasVersionField = false;
bool hasCollationField = false;
bool hasWeightsField = false;
@@ -497,9 +478,6 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
<< "' is only allowed when '" << IndexDescriptor::kKeyPatternFieldName
<< "' is {\"$**\": ±1}"};
}
- if (key.nFields() != 1) {
- return {ErrorCodes::CannotCreateIndex, "wildcard indexes do not allow compounding"};
- }
if (indexSpecElem.embeddedObject().isEmpty()) {
return {ErrorCodes::FailedToParse,
@@ -553,15 +531,12 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
IndexDescriptor::kTextVersionFieldName == indexSpecElemFieldName ||
IndexDescriptor::k2dIndexBitsFieldName == indexSpecElemFieldName ||
IndexDescriptor::k2dIndexMinFieldName == indexSpecElemFieldName ||
- IndexDescriptor::k2dIndexMaxFieldName == indexSpecElemFieldName ||
- IndexDescriptor::kBucketSizeFieldName == indexSpecElemFieldName) &&
+ IndexDescriptor::k2dIndexMaxFieldName == indexSpecElemFieldName) &&
!indexSpecElem.isNumber()) {
return {ErrorCodes::TypeMismatch,
str::stream() << "The field '" << indexSpecElemFieldName
<< "' must be a number, but got "
<< typeName(indexSpecElem.type())};
- } else if (IndexDescriptor::kExpireAfterSecondsFieldName == indexSpecElemFieldName) {
- isTTLIndexWithNaNExpireAfterSeconds = indexSpecElem.isNaN();
} else {
// We can assume field name is valid at this point. Validation of fieldname is handled
// prior to this in validateIndexSpecFieldNames().
@@ -633,19 +608,6 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
modifiedSpec = modifiedSpec.removeField(IndexDescriptor::kNamespaceFieldName);
}
- if (isTTLIndexWithNaNExpireAfterSeconds &&
- !skipTTLIndexNaNExpireAfterSecondsValidation.shouldFail()) {
- // We create a new index specification with the 'expireAfterSeconds' field set as
- // kExpireAfterSecondsForInactiveTTLIndex if the current value is NaN. A similar
- // treatment is done in repairIndexSpec(). This rewrites the 'expireAfterSeconds'
- // value to be compliant with the 'safeInt' IDL type for the listIndexes response.
- BSONObjBuilder builder;
- builder.appendNumber(IndexDescriptor::kExpireAfterSecondsFieldName,
- durationCount<Seconds>(kExpireAfterSecondsForInactiveTTLIndex));
- auto obj = builder.obj();
- modifiedSpec = modifiedSpec.addField(obj.firstElement());
- }
-
if (!hasVersionField) {
// We create a new index specification with the 'v' field set as 'defaultIndexVersion' if
// the field was omitted.
@@ -806,8 +768,7 @@ StatusWith<BSONObj> validateIndexSpecCollation(OperationContext* opCtx,
return indexSpec;
}
-Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds,
- ValidateExpireAfterSecondsMode mode) {
+Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds) {
if (expireAfterSeconds < 0) {
return {ErrorCodes::InvalidOptions,
str::stream() << "TTL index '" << IndexDescriptor::kExpireAfterSecondsFieldName
@@ -818,31 +779,16 @@ Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds,
<< "TTL index '" << IndexDescriptor::kExpireAfterSecondsFieldName
<< "' option must be within an acceptable range, try a lower number";
- if (mode == ValidateExpireAfterSecondsMode::kSecondaryTTLIndex) {
- // Relax epoch restriction on TTL indexes. This allows us to export and import existing
- // TTL indexes with large values or NaN for the 'expireAfterSeconds' field.
- // Additionally, the 'expireAfterSeconds' for TTL indexes is defined as safeInt (int32_t)
- // in the IDL for listIndexes and collMod. See list_indexes.idl and coll_mod.idl.
- if (expireAfterSeconds > std::numeric_limits<std::int32_t>::max()) {
- return {ErrorCodes::InvalidOptions, tooLargeErr};
- }
- } else {
- // Clustered collections with TTL.
- // Note that 'expireAfterSeconds' is defined as safeInt64 in the IDL for the create and
- // collMod commands. See create.idl and coll_mod.idl.
- // There are two cases where we can encounter an issue here.
- // The first case is when we try to cast to millseconds from seconds, which could cause an
- // overflow. The second case is where 'expireAfterSeconds' is larger than the current epoch
- // time. This isn't necessarily problematic for the general case, but for the specific case
- // of time series collections, we cluster the collection by an OID value, where the
- // timestamp portion is only a 32-bit unsigned integer offset of seconds since the epoch.
- if (expireAfterSeconds > std::numeric_limits<std::int64_t>::max() / 1000) {
- return {ErrorCodes::InvalidOptions, tooLargeErr};
- }
- auto expireAfterMillis = duration_cast<Milliseconds>(Seconds(expireAfterSeconds));
- if (expireAfterMillis > Date_t::now().toDurationSinceEpoch()) {
- return {ErrorCodes::InvalidOptions, tooLargeErr};
- }
+ // There are two cases where we can encounter an issue here.
+ // The first case is when we try to cast to millseconds from seconds, which could cause an
+ // overflow. The second case is where 'expireAfterSeconds' is larger than the current epoch
+ // time.
+ if (expireAfterSeconds > std::numeric_limits<std::int64_t>::max() / 1000) {
+ return {ErrorCodes::InvalidOptions, tooLargeErr};
+ }
+ auto expireAfterMillis = duration_cast<Milliseconds>(Seconds(expireAfterSeconds));
+ if (expireAfterMillis > Date_t::now().toDurationSinceEpoch()) {
+ return {ErrorCodes::InvalidOptions, tooLargeErr};
}
return Status::OK();
}
@@ -866,9 +812,7 @@ Status validateIndexSpecTTL(const BSONObj& indexSpec) {
<< "'. Index spec: " << indexSpec};
}
- if (auto status =
- validateExpireAfterSeconds(expireAfterSecondsElt.safeNumberLong(),
- ValidateExpireAfterSecondsMode::kSecondaryTTLIndex);
+ if (auto status = validateExpireAfterSeconds(expireAfterSecondsElt.safeNumberLong());
!status.isOK()) {
return {ErrorCodes::CannotCreateIndex,
str::stream() << status.reason() << ". Index spec: " << indexSpec};
diff --git a/src/mongo/db/catalog/index_key_validate.h b/src/mongo/db/catalog/index_key_validate.h
index 45b383fee1c..8f226749c21 100644
--- a/src/mongo/db/catalog/index_key_validate.h
+++ b/src/mongo/db/catalog/index_key_validate.h
@@ -43,51 +43,35 @@ class StatusWith;
namespace index_key_validate {
-// TTL indexes with 'expireAfterSeconds' are repaired with this duration, which is chosen to be
-// the largest possible value for the 'safeInt' type that can be returned in the listIndexes
-// response.
-constexpr auto kExpireAfterSecondsForInactiveTTLIndex =
- Seconds(std::numeric_limits<int32_t>::max());
-
-/**
- * Describe which field names are considered valid options when creating an index. If the set
- * associated with the field name is empty, the option is always valid, otherwise it will be allowed
- * only when creating the set of index types listed in the set.
- */
-static std::map<StringData, std::set<IndexType>> allowedFieldNames = {
- {IndexDescriptor::k2dIndexBitsFieldName, {IndexType::INDEX_2D}},
- {IndexDescriptor::k2dIndexMaxFieldName, {IndexType::INDEX_2D}},
- {IndexDescriptor::k2dIndexMinFieldName, {IndexType::INDEX_2D}},
- {IndexDescriptor::k2dsphereCoarsestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
- {IndexDescriptor::k2dsphereFinestIndexedLevel, {IndexType::INDEX_2DSPHERE}},
- {IndexDescriptor::k2dsphereVersionFieldName,
- {IndexType::INDEX_2DSPHERE, IndexType::INDEX_2DSPHERE_BUCKET}},
- {IndexDescriptor::kBackgroundFieldName, {}},
- {IndexDescriptor::kCollationFieldName, {}},
- {IndexDescriptor::kDefaultLanguageFieldName, {}},
- {IndexDescriptor::kDropDuplicatesFieldName, {}},
- {IndexDescriptor::kExpireAfterSecondsFieldName, {}},
- {IndexDescriptor::kHiddenFieldName, {}},
- {IndexDescriptor::kIndexNameFieldName, {}},
- {IndexDescriptor::kIndexVersionFieldName, {}},
- {IndexDescriptor::kKeyPatternFieldName, {}},
- {IndexDescriptor::kLanguageOverrideFieldName, {}},
- {IndexDescriptor::kNamespaceFieldName, {}},
- {IndexDescriptor::kPartialFilterExprFieldName, {}},
- {IndexDescriptor::kPathProjectionFieldName, {IndexType::INDEX_WILDCARD}},
- {IndexDescriptor::kSparseFieldName, {}},
- {IndexDescriptor::kStorageEngineFieldName, {}},
- {IndexDescriptor::kTextVersionFieldName, {IndexType::INDEX_TEXT}},
- {IndexDescriptor::kUniqueFieldName, {}},
- {IndexDescriptor::kWeightsFieldName, {IndexType::INDEX_TEXT}},
- {IndexDescriptor::kOriginalSpecFieldName, {}},
- {IndexDescriptor::kPrepareUniqueFieldName, {}},
+static std::set<StringData> allowedFieldNames = {
+ IndexDescriptor::k2dIndexBitsFieldName,
+ IndexDescriptor::k2dIndexMaxFieldName,
+ IndexDescriptor::k2dIndexMinFieldName,
+ IndexDescriptor::k2dsphereCoarsestIndexedLevel,
+ IndexDescriptor::k2dsphereFinestIndexedLevel,
+ IndexDescriptor::k2dsphereVersionFieldName,
+ IndexDescriptor::kBackgroundFieldName,
+ IndexDescriptor::kCollationFieldName,
+ IndexDescriptor::kDefaultLanguageFieldName,
+ IndexDescriptor::kDropDuplicatesFieldName,
+ IndexDescriptor::kExpireAfterSecondsFieldName,
+ IndexDescriptor::kHiddenFieldName,
+ IndexDescriptor::kIndexNameFieldName,
+ IndexDescriptor::kIndexVersionFieldName,
+ IndexDescriptor::kKeyPatternFieldName,
+ IndexDescriptor::kLanguageOverrideFieldName,
+ IndexDescriptor::kNamespaceFieldName,
+ IndexDescriptor::kPartialFilterExprFieldName,
+ IndexDescriptor::kPathProjectionFieldName,
+ IndexDescriptor::kSparseFieldName,
+ IndexDescriptor::kStorageEngineFieldName,
+ IndexDescriptor::kTextVersionFieldName,
+ IndexDescriptor::kUniqueFieldName,
+ IndexDescriptor::kWeightsFieldName,
+ IndexDescriptor::kOriginalSpecFieldName,
+ IndexDescriptor::kPrepareUniqueFieldName,
// Index creation under legacy writeMode can result in an index spec with an _id field.
- {"_id", {}},
- // TODO SERVER-76108: Field names are not validated to match index type. This was used for the
- // removed 'geoHaystack' index type, but users could have set it for other index types as well.
- // We need to keep allowing it until FCV upgrade is implemented to clean this up.
- {"bucketSize"_sd, {}}};
+ "_id"};
/**
* Checks if the key is valid for building an index according to the validation rules for the given
@@ -108,12 +92,12 @@ StatusWith<BSONObj> validateIndexSpec(OperationContext* opCtx, const BSONObj& in
BSONObj removeUnknownFields(const NamespaceString& ns, const BSONObj& indexSpec);
/**
- * Returns a new index spec with boolean values in correct types and unknown field names removed.
+ * Returns a new index spec with boolean values in correct types and unkown field names removed.
*/
-BSONObj repairIndexSpec(const NamespaceString& ns,
- const BSONObj& indexSpec,
- const std::map<StringData, std::set<IndexType>>& allowedFieldNames =
- index_key_validate::allowedFieldNames);
+BSONObj repairIndexSpec(
+ const NamespaceString& ns,
+ const BSONObj& indexSpec,
+ const std::set<StringData>& allowedFieldNames = index_key_validate::allowedFieldNames);
/**
* Performs additional validation for _id index specifications. This should be called after
@@ -137,14 +121,9 @@ StatusWith<BSONObj> validateIndexSpecCollation(OperationContext* opCtx,
const CollatorInterface* defaultCollator);
/**
- * Validates the the 'expireAfterSeconds' value for a TTL index or clustered collection.
+ * Validates the the 'expireAfterSeconds' value for a TTL index..
*/
-enum class ValidateExpireAfterSecondsMode {
- kSecondaryTTLIndex,
- kClusteredTTLIndex,
-};
-Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds,
- ValidateExpireAfterSecondsMode mode);
+Status validateExpireAfterSeconds(std::int64_t expireAfterSeconds);
/**
* Returns true if 'indexSpec' refers to a TTL index.
@@ -166,7 +145,7 @@ bool isIndexAllowedInAPIVersion1(const IndexDescriptor& indexDesc);
* Optional filtering function to adjust allowed index field names at startup.
* Set it in a MONGO_INITIALIZER with 'FilterAllowedIndexFieldNames' as a dependant.
*/
-extern std::function<void(std::map<StringData, std::set<IndexType>>& allowedIndexFieldNames)>
+extern std::function<void(std::set<StringData>& allowedIndexFieldNames)>
filterAllowedIndexFieldNames;
} // namespace index_key_validate
diff --git a/src/mongo/db/catalog/index_key_validate_test.cpp b/src/mongo/db/catalog/index_key_validate_test.cpp
index 659a0bfb7b2..2c25d1b8791 100644
--- a/src/mongo/db/catalog/index_key_validate_test.cpp
+++ b/src/mongo/db/catalog/index_key_validate_test.cpp
@@ -318,100 +318,5 @@ TEST(IndexKeyValidateTest, Background) {
nullptr, fromjson("{key: {a: 1}, name: 'index', background: []}")));
}
-TEST(IndexKeyValidateTest, RemoveUnkownFieldsFromIndexSpecs) {
- ASSERT(fromjson("{key: {a: 1}, name: 'index'}")
- .binaryEqual(index_key_validate::removeUnknownFields(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', safe: true, force: true}"))));
-}
-
-TEST(IndexKeyValidateTest, UpdateTTLIndexNaNExpireAfterSeconds) {
- ASSERT(BSON("key" << BSON("a" << 1) << "name"
- << "index"
- << "expireAfterSeconds" << std::numeric_limits<int32_t>::max()
- << IndexDescriptor::kIndexVersionFieldName << IndexVersion::kV2)
- .binaryEqual(unittest::assertGet(index_key_validate::validateIndexSpec(
- nullptr, fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: NaN}")))));
-}
-
-TEST(IndexKeyValidateTest, ValidateAfterSecondsAcceptsFloatingPointNumber) {
- auto spec = unittest::assertGet(index_key_validate::validateIndexSpec(
- nullptr, fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: 123.456}")));
-
- // TTLMonitor extracts 'expireAfterSeconds' using BSONElement::safeNumberLong().
- ASSERT_EQUALS(spec["expireAfterSeconds"].safeNumberLong(), 123LL);
-}
-
-TEST(IndexKeyValidateTest, RepairIndexSpecs) {
- ASSERT(fromjson("{key: {a: 1}, name: 'index'}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', safe: true, force: true}"))));
-
- ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', sparse: 'true'}"))));
-
- ASSERT(fromjson("{key: {a: 1}, name: 'index', background: true}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', background: '1'}"))));
-
- ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true, background: true}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', sparse: 'true', background: '1'}"))));
-
- ASSERT(fromjson("{key: {a: 1}, name: 'index', sparse: true, background: true}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', sparse: 'true', background: '1', safe: "
- "true, force: true}"))));
-
- ASSERT(fromjson("{key: {a: 1}, name: 'index'}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', weights: {key: 1, name: 1}}"))));
-
- ASSERT(fromjson("{key: {'a': 'text'}, name: 'index', weights: {key: 1, name: 1}}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {'a': 'text'}, name: 'index', weights: {key: 1, name: 1}}"))));
-
- ASSERT(fromjson("{key: {'$**': 'text'}, name: 'index', weights: {key: 1, name: 1}}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {'$**': 'text'}, name: 'index', weights: {key: 1, name: 1}}"))));
-
- ASSERT(fromjson("{key: {'data.loc' : '2dsphere_bucket'}, 'name': 'loc_2dsphere', "
- "'2dsphereIndexVersion' : 3}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {'data.loc' : '2dsphere_bucket'}, 'name': 'loc_2dsphere', "
- "'2dsphereIndexVersion' : 3}"))));
-
- ASSERT(
- fromjson("{key: {a: 1, 'name': 'text'}, name: 'index', weights: {key: 1, name: 1}}")
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson(
- "{key: {a: 1, 'name': 'text'}, name: 'index', weights: {key: 1, name: 1}}"))));
-
- ASSERT(BSON("key" << BSON("a" << 1) << "name"
- << "index"
- << "expireAfterSeconds" << std::numeric_limits<int32_t>::max())
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: NaN}"))));
-
- ASSERT(BSON("key" << BSON("a" << 1) << "name"
- << "index"
- << "expireAfterSeconds" << std::numeric_limits<int32_t>::max())
- .binaryEqual(index_key_validate::repairIndexSpec(
- NamespaceString("coll"),
- fromjson("{key: {a: 1}, name: 'index', expireAfterSeconds: '123'}"))));
-}
-
} // namespace
} // namespace mongo
diff --git a/src/mongo/db/catalog/index_repair.cpp b/src/mongo/db/catalog/index_repair.cpp
index 4e863cec74c..6effefcd909 100644
--- a/src/mongo/db/catalog/index_repair.cpp
+++ b/src/mongo/db/catalog/index_repair.cpp
@@ -31,7 +31,7 @@
#include "mongo/base/status_with.h"
#include "mongo/db/catalog/validate_state.h"
#include "mongo/db/catalog_raii.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/index/index_access_method.h"
#include "mongo/logv2/log_debug.h"
diff --git a/src/mongo/db/catalog/list_indexes.cpp b/src/mongo/db/catalog/list_indexes.cpp
index 660c4068f83..ce1515a4c13 100644
--- a/src/mongo/db/catalog/list_indexes.cpp
+++ b/src/mongo/db/catalog/list_indexes.cpp
@@ -38,7 +38,7 @@
#include "mongo/bson/bsonobj.h"
#include "mongo/bson/bsonobjbuilder.h"
#include "mongo/db/catalog/clustered_collection_util.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop_failpoint_helpers.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/namespace_string.h"
diff --git a/src/mongo/db/catalog/local_oplog_info.cpp b/src/mongo/db/catalog/local_oplog_info.cpp
index d48a5277288..d7e51eb2892 100644
--- a/src/mongo/db/catalog/local_oplog_info.cpp
+++ b/src/mongo/db/catalog/local_oplog_info.cpp
@@ -38,7 +38,6 @@
#include "mongo/db/storage/recovery_unit.h"
#include "mongo/db/vector_clock_mutable.h"
#include "mongo/util/assert_util.h"
-#include "mongo/util/timer.h"
namespace mongo {
namespace {
@@ -112,8 +111,6 @@ std::vector<OplogSlot> LocalOplogInfo::getNextOpTimes(OperationContext* opCtx, s
invariant(_oplog);
fassert(28560, _oplog->getRecordStore()->oplogDiskLocRegister(opCtx, ts, orderedCommit));
}
-
- Timer oplogSlotDurationTimer;
std::vector<OplogSlot> oplogSlots(count);
for (std::size_t i = 0; i < count; i++) {
oplogSlots[i] = {Timestamp(ts.asULL() + i), term};
@@ -122,25 +119,8 @@ std::vector<OplogSlot> LocalOplogInfo::getNextOpTimes(OperationContext* opCtx, s
// If we abort a transaction that has reserved an optime, we should make sure to update the
// stable timestamp if necessary, since this oplog hole may have been holding back the stable
// timestamp.
- opCtx->recoveryUnit()->onRollback([replCoord, oplogSlotDurationTimer]() {
- replCoord->attemptToAdvanceStableTimestamp();
-
- // Transactions can commit on a different thread with a different client and opCtx.
- if (auto opCtx = cc().getOperationContext()) {
- // Sum the oplog slot durations. An operation may participate in multiple transactions.
- CurOp::get(opCtx)->debug().totalOplogSlotDurationMicros +=
- Microseconds(oplogSlotDurationTimer.elapsed());
- }
- });
-
- opCtx->recoveryUnit()->onCommit([oplogSlotDurationTimer](boost::optional<Timestamp>) {
- // Transactions can commit on a different thread with a different Client and opCtx.
- if (auto opCtx = cc().getOperationContext()) {
- // Sum the oplog slot durations. An operation may participate in multiple transactions.
- CurOp::get(opCtx)->debug().totalOplogSlotDurationMicros +=
- Microseconds(oplogSlotDurationTimer.elapsed());
- }
- });
+ opCtx->recoveryUnit()->onRollback(
+ [replCoord]() { replCoord->attemptToAdvanceStableTimestamp(); });
return oplogSlots;
}
diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp
index a0e8382959e..324f6e489ae 100644
--- a/src/mongo/db/catalog/multi_index_block.cpp
+++ b/src/mongo/db/catalog/multi_index_block.cpp
@@ -41,7 +41,7 @@
#include "mongo/db/catalog/collection_catalog.h"
#include "mongo/db/catalog/multi_index_block_gen.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/index/multikey_paths.h"
#include "mongo/db/multi_key_path_tracker.h"
#include "mongo/db/op_observer.h"
@@ -185,7 +185,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(OperationContext* opCtx,
const BSONObj& spec,
OnInitFn onInit) {
const auto indexes = std::vector<BSONObj>(1, spec);
- return init(opCtx, collection, indexes, onInit, /*forRecovery=*/false, boost::none);
+ return init(opCtx, collection, indexes, onInit, boost::none);
}
StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(
@@ -193,7 +193,6 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(
CollectionWriter& collection,
const std::vector<BSONObj>& indexSpecs,
OnInitFn onInit,
- bool forRecovery,
const boost::optional<ResumeIndexInfo>& resumeInfo) {
invariant(opCtx->lockState()->isCollectionLockedForMode(collection->ns(), MODE_X),
str::stream() << "Collection " << collection->ns() << " with UUID "
@@ -247,31 +246,27 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(
for (size_t i = 0; i < indexSpecs.size(); i++) {
BSONObj info = indexSpecs[i];
- if (!forRecovery) {
- // We skip this step when initializing unfinished index builds during startup
- // recovery as they are already in the index catalog.
- StatusWith<BSONObj> statusWithInfo =
- collection->getIndexCatalog()->prepareSpecForCreate(
- opCtx, collection.get(), info, resumeInfo);
- Status status = statusWithInfo.getStatus();
- if (!status.isOK()) {
- // If we were given two identical indexes to build, we will run into an error
- // trying to set up the same index a second time in this for-loop. This is the
- // only way to encounter this error because callers filter out ready/in-progress
- // indexes and start the build while holding a lock throughout.
- if (status == ErrorCodes::IndexBuildAlreadyInProgress) {
- invariant(indexSpecs.size() > 1,
- str::stream() << "Collection: " << collection->ns() << " ("
- << _collectionUUID
- << "), Index spec: " << indexSpecs.front());
- return {ErrorCodes::OperationFailed,
- "Cannot build two identical indexes. Try again without duplicate "
- "indexes."};
- }
- return status;
+ StatusWith<BSONObj> statusWithInfo =
+ collection->getIndexCatalog()->prepareSpecForCreate(
+ opCtx, collection.get(), info, resumeInfo);
+ Status status = statusWithInfo.getStatus();
+ if (!status.isOK()) {
+ // If we were given two identical indexes to build, we will run into an error trying
+ // to set up the same index a second time in this for-loop. This is the only way to
+ // encounter this error because callers filter out ready/in-progress indexes and
+ // start the build while holding a lock throughout.
+ if (status == ErrorCodes::IndexBuildAlreadyInProgress) {
+ invariant(indexSpecs.size() > 1,
+ str::stream()
+ << "Collection: " << collection->ns() << " (" << _collectionUUID
+ << "), Index spec: " << indexSpecs.front());
+ return {
+ ErrorCodes::OperationFailed,
+ "Cannot build two identical indexes. Try again without duplicate indexes."};
}
- info = statusWithInfo.getValue();
+ return status;
}
+ info = statusWithInfo.getValue();
indexInfoObjs.push_back(info);
boost::optional<TimeseriesOptions> options = collection->getTimeseriesOptions();
@@ -307,7 +302,7 @@ StatusWith<std::vector<BSONObj>> MultiIndexBlock::init(
status = index.block->initForResume(
opCtx, collection.getWritableCollection(), *stateInfo, resumeInfo->getPhase());
} else {
- status = index.block->init(opCtx, collection.getWritableCollection(), forRecovery);
+ status = index.block->init(opCtx, collection.getWritableCollection());
}
if (!status.isOK())
return status;
@@ -443,7 +438,7 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(
// Unlock before hanging so replication recognizes we've completed.
collection.yield();
Locker::LockSnapshot lockInfo;
- opCtx->lockState()->saveLockStateAndUnlock(&lockInfo);
+ invariant(opCtx->lockState()->saveLockStateAndUnlock(&lockInfo));
LOGV2(4585201,
"Hanging index build with no locks due to "
@@ -563,7 +558,7 @@ Status MultiIndexBlock::insertAllDocumentsInCollection(
// Unlock before hanging so replication recognizes we've completed.
collection.yield();
Locker::LockSnapshot lockInfo;
- opCtx->lockState()->saveLockStateAndUnlock(&lockInfo);
+ invariant(opCtx->lockState()->saveLockStateAndUnlock(&lockInfo));
LOGV2(20390,
"Hanging index build with no locks due to "
@@ -690,10 +685,10 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx,
// collection to have it.
if (_containsIndexBuildOnTimeseriesMeasurement &&
*collection->getTimeseriesBucketsMayHaveMixedSchemaData()) {
- auto docHasMixedSchemaData =
+ bool docHasMixedSchemaData =
collection->doesTimeseriesBucketsDocContainMixedSchemaData(doc);
- if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue()) {
+ if (docHasMixedSchemaData) {
LOGV2(6057700,
"Detected mixed-schema data in time-series bucket collection",
logAttrs(collection->ns()),
@@ -709,8 +704,7 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx,
auto replCoord = repl::ReplicationCoordinator::get(opCtx);
const bool replSetAndNotPrimary = !replCoord->canAcceptWritesFor(opCtx, collection->ns());
- if (docHasMixedSchemaData.isOK() && docHasMixedSchemaData.getValue() &&
- !replSetAndNotPrimary) {
+ if (docHasMixedSchemaData && !replSetAndNotPrimary) {
return timeseriesMixedSchemaDataFailure(collection.get());
}
}
@@ -727,6 +721,7 @@ Status MultiIndexBlock::_insert(OperationContext* opCtx,
try {
idxStatus = _indexes[i].bulk->insert(opCtx,
collection,
+ _indexes[i].block->getPooledBuilder(),
doc,
loc,
_indexes[i].options,
@@ -978,21 +973,15 @@ Status MultiIndexBlock::commit(OperationContext* opCtx,
onCommit();
- // We can't update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag here as it
- // requires the change to be driven by the router role. It means that subsequent index builds
- // and other systems needs to treat this collection as-if it contains mixed-schema data even if
- // it might not. We log a warning that can be used to initiate changing the flag. Note: just
- // because this node doesn't contain mixed-schema it doesn't mean that other shards can't have
- // mixed schema data. This flag needs to be consistent across the shards.
+ // Update the 'timeseriesBucketsMayHaveMixedSchemaData' catalog entry flag to false in order to
+ // allow subsequent index builds to skip checking bucket documents for mixed-schema data.
if (_containsIndexBuildOnTimeseriesMeasurement && !_timeseriesBucketContainsMixedSchemaData) {
boost::optional<bool> mayContainMixedSchemaData =
collection->getTimeseriesBucketsMayHaveMixedSchemaData();
invariant(mayContainMixedSchemaData);
if (*mayContainMixedSchemaData) {
- LOGV2_WARNING(9301400,
- "Index build finished for time-series collection marked as containing "
- "mixed schema buckets without detecting any buckets with mixed schema.");
+ collection->setTimeseriesBucketsMayHaveMixedSchemaData(opCtx, false);
}
}
diff --git a/src/mongo/db/catalog/multi_index_block.h b/src/mongo/db/catalog/multi_index_block.h
index 5220f8dd6f0..840770595cf 100644
--- a/src/mongo/db/catalog/multi_index_block.h
+++ b/src/mongo/db/catalog/multi_index_block.h
@@ -114,7 +114,6 @@ public:
CollectionWriter& collection,
const std::vector<BSONObj>& specs,
OnInitFn onInit,
- bool forRecovery,
const boost::optional<ResumeIndexInfo>& resumeInfo = boost::none);
StatusWith<std::vector<BSONObj>> init(OperationContext* opCtx,
CollectionWriter& collection,
diff --git a/src/mongo/db/catalog/multi_index_block_test.cpp b/src/mongo/db/catalog/multi_index_block_test.cpp
index e9153ba35dc..2fb9caf7371 100644
--- a/src/mongo/db/catalog/multi_index_block_test.cpp
+++ b/src/mongo/db/catalog/multi_index_block_test.cpp
@@ -90,11 +90,8 @@ TEST_F(MultiIndexBlockTest, CommitWithoutInsertingDocuments) {
AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X);
CollectionWriter coll(operationContext(), autoColl);
- auto specs = unittest::assertGet(indexer->init(operationContext(),
- coll,
- std::vector<BSONObj>(),
- MultiIndexBlock::kNoopOnInitFn,
- /*forRecovery=*/false));
+ auto specs = unittest::assertGet(indexer->init(
+ operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn));
ASSERT_EQUALS(0U, specs.size());
ASSERT_OK(indexer->dumpInsertsFromBulk(operationContext(), coll.get()));
@@ -116,11 +113,8 @@ TEST_F(MultiIndexBlockTest, CommitAfterInsertingSingleDocument) {
AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X);
CollectionWriter coll(operationContext(), autoColl);
- auto specs = unittest::assertGet(indexer->init(operationContext(),
- coll,
- std::vector<BSONObj>(),
- MultiIndexBlock::kNoopOnInitFn,
- /*forRecovery=*/false));
+ auto specs = unittest::assertGet(indexer->init(
+ operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn));
ASSERT_EQUALS(0U, specs.size());
ASSERT_OK(
@@ -152,11 +146,8 @@ TEST_F(MultiIndexBlockTest, AbortWithoutCleanupAfterInsertingSingleDocument) {
AutoGetCollection autoColl(operationContext(), getNSS(), MODE_X);
CollectionWriter coll(operationContext(), autoColl);
- auto specs = unittest::assertGet(indexer->init(operationContext(),
- coll,
- std::vector<BSONObj>(),
- MultiIndexBlock::kNoopOnInitFn,
- /*forRecovery=*/false));
+ auto specs = unittest::assertGet(indexer->init(
+ operationContext(), coll, std::vector<BSONObj>(), MultiIndexBlock::kNoopOnInitFn));
ASSERT_EQUALS(0U, specs.size());
ASSERT_OK(
indexer->insertSingleDocumentForInitialSyncOrRecovery(operationContext(),
diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp
index 01bc87f7e2b..186e4a38330 100644
--- a/src/mongo/db/catalog/rename_collection.cpp
+++ b/src/mongo/db/catalog/rename_collection.cpp
@@ -43,8 +43,8 @@
#include "mongo/db/catalog/list_indexes.h"
#include "mongo/db/catalog/local_oplog_info.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
#include "mongo/db/concurrency/lock_state.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
@@ -116,10 +116,7 @@ Status checkSourceAndTargetNamespaces(OperationContext* opCtx,
str::stream() << "Source collection " << source.ns() << " does not exist");
}
- if (sourceColl->getCollectionOptions().encryptedFieldConfig &&
- !AuthorizationSession::get(opCtx->getClient())
- ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
- ActionType::setUserWriteBlockMode)) {
+ if (sourceColl->getCollectionOptions().encryptedFieldConfig) {
return Status(ErrorCodes::IllegalOperation, "Cannot rename an encrypted collection");
}
@@ -132,10 +129,7 @@ Status checkSourceAndTargetNamespaces(OperationContext* opCtx,
return Status(ErrorCodes::NamespaceExists,
str::stream() << "a view already exists with that name: " << target);
} else {
- if (targetColl->getCollectionOptions().encryptedFieldConfig &&
- !AuthorizationSession::get(opCtx->getClient())
- ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
- ActionType::setUserWriteBlockMode)) {
+ if (targetColl->getCollectionOptions().encryptedFieldConfig) {
return Status(ErrorCodes::IllegalOperation,
"Cannot rename to an existing encrypted collection");
}
@@ -339,28 +333,6 @@ Status renameCollectionWithinDB(OperationContext* opCtx,
if (!status.isOK())
return status;
- if (options.originalCollectionOptions) {
- // Check target collection options match expected.
- const BSONObj collectionOptions =
- targetColl ? targetColl->getCollectionOptions().toBSON() : BSONObj();
- status = checkTargetCollectionOptionsMatch(
- target, options.originalCollectionOptions.get(), collectionOptions);
- if (!status.isOK()) {
- return status;
- }
- }
- if (options.originalIndexes) {
- // Check target collection indexes match expected.
- const auto currentIndexes =
- listIndexesEmptyListIfMissing(opCtx, target, ListIndexesInclude::Nothing);
- status = checkTargetCollectionIndexesMatch(
- target, options.originalIndexes.get(), currentIndexes);
-
- if (!status.isOK()) {
- return status;
- }
- }
-
AutoStatsTracker statsTracker(
opCtx,
source,
@@ -650,10 +622,7 @@ Status renameBetweenDBs(OperationContext* opCtx,
// Copy the index descriptions from the source collection.
std::vector<BSONObj> indexesToCopy;
- for (auto sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator(
- opCtx,
- IndexCatalog::InclusionPolicy::kReady | IndexCatalog::InclusionPolicy::kUnfinished |
- IndexCatalog::InclusionPolicy::kFrozen);
+ for (auto sourceIndIt = sourceColl->getIndexCatalog()->getIndexIterator(opCtx, true);
sourceIndIt->more();) {
auto descriptor = sourceIndIt->next()->descriptor();
if (descriptor->isIdIndex()) {
@@ -795,44 +764,44 @@ Status renameBetweenDBs(OperationContext* opCtx,
void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx,
const NamespaceString& sourceNs,
const NamespaceString& targetNs,
- const RenameCollectionOptions& options) {
- // Pass in originalIndexes and originalCollectionOptions to be evaluated later,
- // under a collection lock in renameCollectionWithinDB.
- validateAndRunRenameCollection(opCtx, sourceNs, targetNs, options);
-}
+ const RenameCollectionOptions& options,
+ std::list<BSONObj> originalIndexes,
+ BSONObj originalCollectionOptions) {
+ AutoGetDb dbLock(opCtx, targetNs.db(), MODE_X);
+ auto collection = dbLock.getDb()
+ ? CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, targetNs)
+ : nullptr;
+ BSONObj collectionOptions = {};
+ if (collection) {
+ // We do not include the UUID field in the options comparison. It is ok if the target
+ // collection was dropped and recreated, as long as the new target collection has the same
+ // options and indexes as the original one did. This is mainly to support concurrent $out
+ // to the same collection.
+ collectionOptions = collection->getCollectionOptions().toBSON().removeField("uuid");
+ }
-Status checkTargetCollectionOptionsMatch(const NamespaceString& targetNss,
- const BSONObj& expectedOptions,
- const BSONObj& currentOptions) {
- // We do not include the UUID field in the options comparison. It is ok if the target collection
- // was dropped and recreated, as long as the new target collection has the same options and
- // indexes as the original one did. This is mainly to support concurrent $out to the same
- // collection.
- if (SimpleBSONObjComparator::kInstance.evaluate(expectedOptions.removeField("uuid") !=
- currentOptions.removeField("uuid"))) {
- return Status(ErrorCodes::CommandFailed,
- str::stream()
- << "collection options of target collection " << targetNss.toString()
- << " changed during processing. Original options: " << expectedOptions
- << ", new options: " << currentOptions);
- };
- return Status::OK();
-}
+ uassert(ErrorCodes::CommandFailed,
+ str::stream() << "collection options of target collection " << targetNs.ns()
+ << " changed during processing. Original options: "
+ << originalCollectionOptions << ", new options: " << collectionOptions,
+ SimpleBSONObjComparator::kInstance.evaluate(
+ originalCollectionOptions.removeField("uuid") == collectionOptions));
+
+ auto currentIndexes =
+ listIndexesEmptyListIfMissing(opCtx, targetNs, ListIndexesInclude::Nothing);
-Status checkTargetCollectionIndexesMatch(const NamespaceString& targetNss,
- const std::list<BSONObj>& expectedIndexes,
- const std::list<BSONObj>& currentIndexes) {
UnorderedFieldsBSONObjComparator comparator;
- if (expectedIndexes.size() != currentIndexes.size() ||
- !(std::equal(expectedIndexes.begin(),
- expectedIndexes.end(),
- currentIndexes.begin(),
- [&](auto& lhs, auto& rhs) { return comparator.compare(lhs, rhs) == 0; }))) {
- return Status(ErrorCodes::CommandFailed,
- str::stream() << "indexes of target collection " << targetNss.toString()
- << " changed during processing.");
- }
- return Status::OK();
+ uassert(
+ ErrorCodes::CommandFailed,
+ str::stream() << "indexes of target collection " << targetNs.ns()
+ << " changed during processing.",
+ originalIndexes.size() == currentIndexes.size() &&
+ std::equal(originalIndexes.begin(),
+ originalIndexes.end(),
+ currentIndexes.begin(),
+ [&](auto& lhs, auto& rhs) { return comparator.compare(lhs, rhs) == 0; }));
+
+ validateAndRunRenameCollection(opCtx, sourceNs, targetNs, options);
}
void validateNamespacesForRenameCollection(OperationContext* opCtx,
@@ -886,21 +855,8 @@ void validateNamespacesForRenameCollection(OperationContext* opCtx,
!source.isSystemDotViews() && !target.isSystemDotViews());
uassert(ErrorCodes::IllegalOperation,
- "renaming system.users collection or renaming to system.users is not allowed",
- !source.isSystemDotUsers() && !target.isSystemDotUsers());
-
- if (source.isTimeseriesBucketsCollection()) {
- uassert(ErrorCodes::IllegalOperation,
- "Renaming system.buckets collections is not allowed",
- AuthorizationSession::get(opCtx->getClient())
- ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
- ActionType::setUserWriteBlockMode));
-
- uassert(ErrorCodes::IllegalOperation,
- str::stream() << "Cannot rename time-series buckets collection {" << source.ns()
- << "} to a non-time-series buckets namespace {" << target.ns() << "}",
- target.isTimeseriesBucketsCollection());
- }
+ "Renaming system.buckets collections is not allowed",
+ !source.isTimeseriesBucketsCollection());
}
void validateAndRunRenameCollection(OperationContext* opCtx,
diff --git a/src/mongo/db/catalog/rename_collection.h b/src/mongo/db/catalog/rename_collection.h
index ef0d4f8d9d8..253a358f89c 100644
--- a/src/mongo/db/catalog/rename_collection.h
+++ b/src/mongo/db/catalog/rename_collection.h
@@ -53,35 +53,14 @@ struct RenameCollectionOptions {
bool markFromMigrate = false;
boost::optional<UUID> expectedSourceUUID;
boost::optional<UUID> expectedTargetUUID;
- boost::optional<UUID> newTargetCollectionUuid;
- boost::optional<std::list<BSONObj>> originalIndexes;
- boost::optional<BSONObj> originalCollectionOptions;
};
void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx,
const NamespaceString& sourceNs,
const NamespaceString& targetNs,
- const RenameCollectionOptions& options);
-
-/**
- * Checks that CollectionOptions 'expectedOptions' and 'currentOptions' are equal, except for the
- * 'uuid' field. Returns a CommandFailed status otherwise.
- * To be used by doLocalRenameIfOptionsAndIndexesHaveNotChanged and also its sharding-aware
- * equivalent in RenameCollectionCoordinator.
- */
-Status checkTargetCollectionOptionsMatch(const NamespaceString& targetNss,
- const BSONObj& expectedOptions,
- const BSONObj& currentOptions);
-
-/**
- * Checks that the lists of index specs 'expectedIndexes' and 'currentIndexes' are equal.
- * To be used by doLocalRenameIfOptionsAndIndexesHaveNotChanged and also its sharding-aware
- * equivalent in RenameCollectionCoordinator. Returns a CommandFailed status if indexes do not
- * match.
- */
-Status checkTargetCollectionIndexesMatch(const NamespaceString& targetNss,
- const std::list<BSONObj>& expectedIndexes,
- const std::list<BSONObj>& currentIndexes);
+ const RenameCollectionOptions& options,
+ std::list<BSONObj> originalIndexes,
+ BSONObj collectionOptions);
Status renameCollection(OperationContext* opCtx,
const NamespaceString& source,
diff --git a/src/mongo/db/catalog/rename_collection_test.cpp b/src/mongo/db/catalog/rename_collection_test.cpp
index b092cf4fffe..215069fd9b7 100644
--- a/src/mongo/db/catalog/rename_collection_test.cpp
+++ b/src/mongo/db/catalog/rename_collection_test.cpp
@@ -40,7 +40,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/rename_collection.h"
#include "mongo/db/client.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
#include "mongo/db/index/index_descriptor.h"
#include "mongo/db/index_builds_coordinator.h"
diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp
deleted file mode 100644
index 66ed55b6e57..00000000000
--- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.cpp
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include <algorithm>
-#include <cstring>
-#include <fmt/format.h>
-#include <pcrecpp.h>
-
-#include "mongo/base/string_data.h"
-#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
-#include "mongo/util/ctype.h"
-
-namespace mongo {
-
-const static StaticImmortal<pcrecpp::RE> appMetadataRegex(
- R"re(((?<=^|,)\s*(?:app_metadata|\"app_metadata\")\s*[=:]\s*[({[]\s*))re");
-
-static pcrecpp::RE flagMatchRegex(StringData flagName) {
- // This check is overly strict, but it suffices for now and ensures that both:
- // - The flag name is a valid WiredTiger identifier, and
- // - It can be used in the regular expression without needing to escape it
- invariant(std::all_of(flagName.begin(), flagName.end(), ctype::isAlpha));
-
- // Some examples of possible matches:
- // `flag=false`
- // `flag:true`
- // `flag`
- // ` "flag" = false `
- // ` "flag" `
- return pcrecpp::RE(fmt::format(
- R"re(((?<=[,({{[])\s*(?:{0}|\"{0}\")(?:\s*[=:]\s*(true|false))?\s*(?=[,)}}\]])))re",
- flagName));
-}
-
-static std::map<StringData, boost::optional<bool>> getFlagsFromWtConfigStringAppMetadata(
- const std::string& configString, const std::vector<StringData>& flagNames) {
- std::map<StringData, boost::optional<bool>> flags;
-
- for (const auto& flagName : flagNames) {
- auto flagRegex = flagMatchRegex(flagName);
- pcrecpp::StringPiece fullMatch, flagValueStr;
- auto matches = flagRegex.PartialMatch(configString, &fullMatch, &flagValueStr);
- flags.emplace(flagName,
- matches ? boost::optional<bool>(flagValueStr == "" || flagValueStr == "true")
- : boost::none);
- }
-
- return flags;
-}
-
-std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson(
- const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames) {
- auto configString = WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions);
- return getFlagsFromWtConfigStringAppMetadata(configString.value_or(""), flagNames);
-}
-
-boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions,
- StringData flagName) {
- return getFlagsFromStorageEngineBson(storageEngineOptions, {flagName})[flagName];
-}
-
-// Finds or adds the 'app_metadata=(...)' struct inside a WiredTiger config string
-// Returns the position inside the struct (after the delimiter, before the first key-value)
-static size_t findOrAddAppMetadataStructToConfigString(std::string& configString) {
- pcrecpp::StringPiece fullMatch;
- auto matches = appMetadataRegex->PartialMatch(configString, &fullMatch);
- if (!matches)
- configString += configString.empty() ? "app_metadata=()" : ",app_metadata=()";
- return matches ? (fullMatch.data() + fullMatch.size() - configString.data())
- : configString.size() - 1;
-}
-
-// Expand a [pos, len) range inside a config string to include a leading or trailing comma separator
-static void expandRangeToIncludeSeparator(const std::string& configString,
- size_t& pos,
- size_t& len) {
- if (pos > 0 && configString[pos - 1] == ',') {
- pos--;
- len++;
- } else if (pos + len < configString.size() && configString[pos + len] == ',') {
- len++;
- }
-}
-
-static void setFlagsToWtConfigStringAppMetadata(
- std::string& configString, const std::map<StringData, boost::optional<bool>>& flags) {
- auto metadataPos = findOrAddAppMetadataStructToConfigString(configString);
-
- for (const auto& [flagName, flagValue] : flags) {
- auto flagRegex = flagMatchRegex(flagName);
- pcrecpp::StringPiece fullMatch;
- // "- 1" allows the positive lookbehind (?<=) at the start of the regex to work
- auto matches = flagRegex.PartialMatch(&configString[metadataPos - 1], &fullMatch);
- if (matches) {
- size_t pos = fullMatch.data() - configString.data(), len = fullMatch.size();
-
- if (flagValue.has_value()) { // Replace existing flag
- auto flagItem = fmt::format("{}={}", flagName, *flagValue);
- configString.replace(pos, len, flagItem);
- } else { // Unset existing flag
- expandRangeToIncludeSeparator(configString, pos, len);
- configString.erase(pos, len);
- }
- } else if (flagValue.has_value()) { // Add new flag
- auto metadataEmpty = strchr(")]}", configString[metadataPos]) != nullptr;
- auto flagItem = fmt::format("{}={}{}", flagName, *flagValue, metadataEmpty ? "" : ",");
- configString.insert(metadataPos, flagItem);
- }
- }
-}
-
-BSONObj setFlagsToStorageEngineBson(const BSONObj& storageEngineOptions,
- const std::map<StringData, boost::optional<bool>>& flags) {
- auto configString =
- WiredTigerUtil::getConfigStringFromStorageOptions(storageEngineOptions).value_or("");
- setFlagsToWtConfigStringAppMetadata(configString, flags);
-
- // Both for safety, and because the regex-based parser can not handle some theoretical cases,
- // sanity check that the resulting string is a valid WiredTiger configuration string
- auto configStringObj = BSON(WiredTigerUtil::kConfigStringField << configString);
- tassert(9218600,
- "The resulting WiredTiger configuration string is not valid",
- WiredTigerUtil::checkTableCreationOptions(configStringObj.firstElement()).isOK());
-
- return WiredTigerUtil::setConfigStringToStorageOptions(storageEngineOptions, configString);
-}
-
-BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions,
- StringData flagName,
- boost::optional<bool> flagValue) {
- return setFlagsToStorageEngineBson(storageEngineOptions, {{flagName, flagValue}});
-}
-
-} // namespace mongo
diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h
deleted file mode 100644
index cb4a1b20e2f..00000000000
--- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include <boost/optional/optional.hpp>
-#include <map>
-#include <vector>
-
-#include "mongo/base/string_data.h"
-#include "mongo/bson/bsonobj.h"
-
-namespace mongo {
-
-/**
- * Utility functions to get or set boolean flags from/to a storage engine options object
- * (see `CollectionOptions::storageEngine`).
- *
- * The idea is that for exceptional (workaround) purposes, we can use the storage engine
- * options object as a flexible structure where new fields can be added retroactively,
- * unlike the other parts of the catalog which generally have non-flexible / strict validations.
- * For more information, see: SERVER-91195, SERVER-92186.
- */
-
-std::map<StringData, boost::optional<bool>> getFlagsFromStorageEngineBson(
- const BSONObj& storageEngineOptions, const std::vector<StringData>& flagNames);
-
-boost::optional<bool> getFlagFromStorageEngineBson(const BSONObj& storageEngineOptions,
- StringData flagName);
-
-[[nodiscard]] BSONObj setFlagsToStorageEngineBson(
- const BSONObj& storageEngineOptions, const std::map<StringData, boost::optional<bool>>& flags);
-
-[[nodiscard]] BSONObj setFlagToStorageEngineBson(const BSONObj& storageEngineOptions,
- StringData flagName,
- boost::optional<bool> flagValue);
-
-} // namespace mongo
diff --git a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp b/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp
deleted file mode 100644
index 5305668e8fc..00000000000
--- a/src/mongo/db/catalog/storage_engine_collection_options_flags_parser_test.cpp
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Copyright (C) 2024-present MongoDB, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the Server Side Public License, version 1,
- * as published by MongoDB, Inc.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * Server Side Public License for more details.
- *
- * You should have received a copy of the Server Side Public License
- * along with this program. If not, see
- * <http://www.mongodb.com/licensing/server-side-public-license>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the Server Side Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/catalog/storage_engine_collection_options_flags_parser.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
-#include "mongo/unittest/bson_test_util.h"
-#include "mongo/unittest/unittest.h"
-
-namespace mongo {
-namespace {
-
-static BSONObj makeStorageEngineWithConfigString(StringData configString) {
- return BSON(kWiredTigerEngineName << BSON(WiredTigerUtil::kConfigStringField << configString));
-}
-
-static BSONObj addExtraFields(const BSONObj& storageEngineOptions) {
- auto wtObj = storageEngineOptions[kWiredTigerEngineName].Obj();
- return storageEngineOptions.addFields(BSON("dummy1"
- << "value1" << kWiredTigerEngineName
- << wtObj.addFields(BSON("dummy2"
- << "value2"))));
-}
-
-TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenNoWiredTigerConfigString) {
- auto options = BSONObj();
-
- auto flag = getFlagFromStorageEngineBson(options, "flagA");
-
- ASSERT_EQ(boost::none, flag);
-}
-
-TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenOptionsDoesNotContainMetadata) {
- auto options = makeStorageEngineWithConfigString("access_pattern_hint=random");
-
- auto flag = getFlagFromStorageEngineBson(options, "flagA");
-
- ASSERT_EQ(boost::none, flag);
-}
-
-TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataDoesNotContainTheFlag) {
- auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1)");
-
- auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"});
-
- ASSERT_EQ(boost::none, flags["flagA"]);
- ASSERT_EQ(boost::none, flags["flagB"]);
-}
-
-TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsASingleFlag) {
- auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=true)");
-
- auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"});
-
- ASSERT_EQ(true, flags["flagA"]);
- ASSERT_EQ(boost::none, flags["flagB"]);
-}
-
-TEST(StorageEngineFlagsParserTest, GetValueWhenMetadataContainsMultipleFlags) {
- auto options = makeStorageEngineWithConfigString("app_metadata=(flagB=true,flagA=false)");
-
- auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"});
-
- ASSERT_EQ(false, flags["flagA"]);
- ASSERT_EQ(true, flags["flagB"]);
-}
-
-TEST(StorageEngineFlagsParserTest, GetEmptyOptionalWhenMetadataContainsAnInvalidValue) {
- auto options =
- makeStorageEngineWithConfigString("app_metadata=(flagB=(hello=world),flagA=true)");
-
- auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"});
-
- ASSERT_EQ(true, flags["flagA"]);
- ASSERT_EQ(boost::none, flags["flagB"]);
-}
-
-TEST(StorageEngineFlagsParserTest, GetTrueWhenMetadataContainsAKeyWithNoValue) {
- auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)");
-
- auto flag = getFlagFromStorageEngineBson(options, "flagA");
-
- ASSERT_EQ(true, flag);
-}
-
-TEST(StorageEngineFlagsParserTest, GetIgnoresUnknownStorageEngineFields) {
- auto options = addExtraFields(makeStorageEngineWithConfigString("app_metadata=(flagA=true)"));
-
- auto flag = getFlagFromStorageEngineBson(options, "flagA");
-
- ASSERT_EQ(true, flag);
-}
-
-TEST(StorageEngineFlagsParserTest, GetHandlesTrickyFormatting) {
- auto options = addExtraFields(makeStorageEngineWithConfigString(
- " access_pattern_hint = random , \"app_metadata\" : [ x=y , "
- "\"flagB\": true , z : t ]"));
-
- auto flags = getFlagsFromStorageEngineBson(options, {"flagA", "flagB"});
-
- ASSERT_EQ(boost::none, flags["flagA"]);
- ASSERT_EQ(true, flags["flagB"]);
-}
-
-TEST(StorageEngineFlagsParserTest, AddFlagToEmptyStorageEngineBson) {
- auto options = BSONObj();
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", true);
-
- ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)"));
-}
-
-TEST(StorageEngineFlagsParserTest, AddFlagToEmptyConfigString) {
- auto options = makeStorageEngineWithConfigString("");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", true);
-
- ASSERT_BSONOBJ_EQ(newOptions, makeStorageEngineWithConfigString("app_metadata=(flagA=true)"));
-}
-
-TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithNoMetadata) {
- auto options = makeStorageEngineWithConfigString("access_pattern_hint=random");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", true);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)"));
-}
-
-TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithEmptyMetadata) {
- auto options = makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=()");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", false);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)"));
-}
-
-TEST(StorageEngineFlagsParserTest, AddFlagToExistingConfigStringWithOtherFlags) {
- auto options = makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(hello2=world2,flagB=true)");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", false);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(flagA=false,hello2=world2,flagB=true)"));
-}
-
-TEST(StorageEngineFlagsParserTest, SetExistingFlag) {
- auto options =
- makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=false)");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", true);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(flagA=true)"));
-}
-
-TEST(StorageEngineFlagsParserTest, RemoveExistingFlag) {
- auto options = makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(flagB=true,x=y,z=t)");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagB", boost::none);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString("access_pattern_hint=random,app_metadata=(x=y,z=t)"));
-}
-
-TEST(StorageEngineFlagsParserTest, SetMultipleFlags) {
- auto options = makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t,flagC=true)");
-
- auto newOptions = setFlagsToStorageEngineBson(
- options, {{"flagB", false}, {"flagA", true}, {"flagC", boost::none}});
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(flagA=true,x=y,flagB=false,z=t)"));
-}
-
-TEST(StorageEngineFlagsParserTest, SetFlagWhenMetadataContainsAKeyWithNoValue) {
- auto options = makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA)");
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagA", false);
-
- ASSERT_BSONOBJ_EQ(
- newOptions,
- makeStorageEngineWithConfigString("app_metadata=(formatVersion=1,flagA=false)"));
-}
-
-TEST(StorageEngineFlagsParserTest, SetPreservesUnknownStorageEngineFields) {
- auto options = addExtraFields(makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(x=y,flagB=false,z=t)"));
-
- auto newOptions = setFlagToStorageEngineBson(options, "flagB", true);
-
- auto expected = addExtraFields(makeStorageEngineWithConfigString(
- "access_pattern_hint=random,app_metadata=(x=y,flagB=true,z=t)"));
- ASSERT_BSONOBJ_EQ(newOptions, expected);
-}
-
-TEST(StorageEngineFlagsParserTest, SetHandlesTrickyFormatting) {
- auto options = addExtraFields(makeStorageEngineWithConfigString(
- " access_pattern_hint = random , \"app_metadata\" : [ x=y , "
- "\"flagB\": false , z : t , flagC : true ]"));
-
- auto newOptions = setFlagsToStorageEngineBson(
- options, {{"flagA", false}, {"flagB", true}, {"flagC", boost::none}});
-
- auto expected = addExtraFields(makeStorageEngineWithConfigString(
- " access_pattern_hint = random , \"app_metadata\" : [ flagA=false,x=y "
- " ,flagB=true, z : t ]"));
- ASSERT_BSONOBJ_EQ(newOptions, expected);
-}
-
-} // namespace
-} // namespace mongo
diff --git a/src/mongo/db/catalog/uncommitted_catalog_updates.cpp b/src/mongo/db/catalog/uncommitted_catalog_updates.cpp
index 86424f3d71a..67491ba5319 100644
--- a/src/mongo/db/catalog/uncommitted_catalog_updates.cpp
+++ b/src/mongo/db/catalog/uncommitted_catalog_updates.cpp
@@ -117,7 +117,7 @@ void UncommittedCatalogUpdates::_createCollection(OperationContext* opCtx,
// This will throw when registering a namespace which is already in use.
CollectionCatalog::write(opCtx, [&, coll = createdColl](CollectionCatalog& catalog) {
- catalog.registerCollection(opCtx, coll);
+ catalog.registerCollection(opCtx, uuid, coll);
});
opCtx->recoveryUnit()->onRollback([opCtx, uuid]() {
diff --git a/src/mongo/db/catalog/validate_adaptor.cpp b/src/mongo/db/catalog/validate_adaptor.cpp
index 9dc4a8dc71b..acbef39ba9c 100644
--- a/src/mongo/db/catalog/validate_adaptor.cpp
+++ b/src/mongo/db/catalog/validate_adaptor.cpp
@@ -41,7 +41,7 @@
#include "mongo/db/catalog/index_catalog.h"
#include "mongo/db/catalog/index_consistency.h"
#include "mongo/db/catalog/throttle_cursor.h"
-#include "mongo/db/concurrency/exception_util.h"
+#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/curop.h"
#include "mongo/db/index/index_access_method.h"
#include "mongo/db/index/index_descriptor.h"
@@ -53,10 +53,6 @@
#include "mongo/db/storage/execution_context.h"
#include "mongo/db/storage/key_string.h"
#include "mongo/db/storage/record_store.h"
-#include "mongo/db/storage/storage_parameters_gen.h"
-#include "mongo/db/timeseries/flat_bson.h"
-#include "mongo/db/timeseries/timeseries_constants.h"
-#include "mongo/db/timeseries/timeseries_options.h"
#include "mongo/logv2/log.h"
#include "mongo/rpc/object_check.h"
#include "mongo/util/fail_point.h"
@@ -67,8 +63,6 @@ namespace mongo {
namespace {
MONGO_FAIL_POINT_DEFINE(crashOnMultikeyValidateFailure);
-MONGO_FAIL_POINT_DEFINE(failIndexKeyOrdering);
-MONGO_FAIL_POINT_DEFINE(failRecordStoreTraversal);
// Set limit for size of corrupted records that will be reported.
const long long kMaxErrorSizeBytes = 1 * 1024 * 1024;
@@ -78,9 +72,6 @@ const long long kInterruptIntervalNumBytes = 50 * 1024 * 1024; // 50MB.
static constexpr const char* kSchemaValidationFailedReason =
"Detected one or more documents not compliant with the collection's schema. Check logs for log "
"id 5363500.";
-static constexpr const char* kTimeseriesValidationInconsistencyReason =
- "Detected one or more documents in this collection incompatible with time-series "
- "specifications. For more info, see logs with log id 6698300.";
/**
* Validate that for each record in a clustered RecordStore the record key (RecordId) matches the
@@ -127,197 +118,32 @@ void schemaValidationFailed(CollectionValidation::ValidateState* state,
state->setCollectionSchemaViolated();
- if (result != Collection::SchemaValidationResult::kPass) {
+ // TODO SERVER-65078: remove the testing proctor check.
+ // When testing is enabled, only warn about non-compliant documents to prevent test failures.
+ if (TestingProctor::instance().isEnabled() ||
+ Collection::SchemaValidationResult::kWarn == result) {
results->warnings.push_back(kSchemaValidationFailedReason);
+ } else if (Collection::SchemaValidationResult::kError == result) {
+ results->errors.push_back(kSchemaValidationFailedReason);
+ results->valid = false;
}
}
-/**
- * Checks the value of the bucket's version and if it matches the types of 'data' fields.
- */
-Status _validateTimeseriesControlVersion(const BSONObj& recordBson) {
- int controlVersion = recordBson.getField(timeseries::kBucketControlFieldName)
- .Obj()
- .getField(timeseries::kBucketControlVersionFieldName)
- .Number();
- if (controlVersion != 1 && controlVersion != 2) {
- return Status(
- ErrorCodes::BadValue,
- fmt::format("Invalid value for 'control.version'. Expected 1 or 2, but got {}.",
- controlVersion));
- }
- auto dataType = controlVersion == 1 ? BSONType::Object : BSONType::BinData;
- // In addition to checking dataType, make sure that closed buckets have BinData Column subtype
- auto isCorrectType = [&](BSONElement el) {
- if (controlVersion == 1) {
- return el.type() == BSONType::Object;
- } else {
- return el.type() == BSONType::BinData && el.binDataType() == BinDataType::Column;
- }
- };
- BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj();
- for (BSONObjIterator bi(data); bi.more();) {
- BSONElement e = bi.next();
- if (!isCorrectType(e)) {
- return Status(ErrorCodes::TypeMismatch,
- fmt::format("Mismatch between time-series schema version and data field "
- "type. Expected type {}, but got {}.",
- mongo::typeName(dataType),
- mongo::typeName(e.type())));
- }
- }
- return Status::OK();
-}
-
-/**
- * Checks the equivalence between the min and max fields in 'control' for a bucket and
- * the corresponding value in 'data'.
- */
-Status _validateTimeseriesMinMax(const BSONObj& recordBson, const CollectionPtr& coll) {
- BSONObj data = recordBson.getField(timeseries::kBucketDataFieldName).Obj();
- BSONObj control = recordBson.getField(timeseries::kBucketControlFieldName).Obj();
- BSONObj controlMin = control.getField(timeseries::kBucketControlMinFieldName).Obj();
- BSONObj controlMax = control.getField(timeseries::kBucketControlMaxFieldName).Obj();
-
- auto dataFields = data.getFieldNames<std::set<std::string>>();
- auto controlMinFields = controlMin.getFieldNames<std::set<std::string>>();
- auto controlMaxFields = controlMax.getFieldNames<std::set<std::string>>();
-
- // Checks that the number of 'control.min' and 'control.max' fields agrees with number of 'data'
- // fields.
- if (dataFields.size() != controlMinFields.size() ||
- dataFields.size() != controlMaxFields.size()) {
- return Status(
- ErrorCodes::BadValue,
- fmt::format(
- "Mismatch between the number of time-series control fields and the number "
- "of data fields. "
- "Control had {} min fields and {} max fields, but observed data had {} fields.",
- controlMinFields.size(),
- controlMaxFields.size(),
- dataFields.size()));
- };
-
- // Used when checking min timestamp, which is rounded down by granularity.
- auto granularity = coll->getTimeseriesOptions()->getGranularity();
-
- // Validates that the 'control.min' and 'control.max' field values agree with 'data' field
- // values.
- for (auto fieldName : dataFields) {
- timeseries::MinMax minmax;
- auto field = data.getField(fieldName);
-
- for (BSONElement el : field.Obj()) {
- minmax.update(el.wrap(fieldName), boost::none, coll->getDefaultCollator());
- }
-
- auto controlFieldMin = controlMin.getField(fieldName);
- auto controlFieldMax = controlMax.getField(fieldName);
- auto min = minmax.min();
- auto max = minmax.max();
-
- // Checks whether the min and max values between 'control' and 'data' match, taking
- // timestamp granularity into account.
- auto checkMinAndMaxMatch = [&]() {
- if (fieldName == coll->getTimeseriesOptions()->getTimeField()) {
- return controlFieldMin.Date() ==
- timeseries::roundTimestampToGranularity(min.getField(fieldName).Date(),
- granularity) &&
- controlFieldMax.Date() == max.getField(fieldName).Date();
- } else {
- return controlFieldMin.wrap().woCompare(min) == 0 &&
- controlFieldMax.wrap().woCompare(max) == 0;
- }
- };
-
- if (!checkMinAndMaxMatch()) {
- return Status(
- ErrorCodes::BadValue,
- fmt::format(
- "Mismatch between time-series control and observed min or max for field {}. "
- "Control had min {} and max {}, but observed data had min {} and max {}.",
- fieldName,
- controlFieldMin.toString(),
- controlFieldMax.toString(),
- min.toString(),
- max.toString()));
- }
- }
-
- return Status::OK();
-}
-
-/**
- * Validates the consistency of a time-series bucket.
- */
-Status _validateTimeSeriesBucketRecord(const CollectionPtr& collection,
- const BSONObj& recordBson,
- ValidateResults* results) {
-
- if (Status status = _validateTimeseriesControlVersion(recordBson); !status.isOK()) {
- return status;
- }
-
- int version = recordBson.getField(timeseries::kBucketControlFieldName)
- .Obj()
- .getField(timeseries::kBucketControlVersionFieldName)
- .Number();
-
- // TODO(SERVER-67023): Check closed bucket as part of validation.
- if (version == 1) {
- if (Status status = _validateTimeseriesMinMax(recordBson, collection); !status.isOK()) {
- return status;
- }
- }
-
-
- return Status::OK();
-}
-
-
-void _timeseriesValidationFailed(CollectionValidation::ValidateState* state,
- ValidateResults* results) {
- if (state->isTimeseriesDataInconsistent()) {
- // Only report the warning message once.
- return;
- }
- state->setTimeseriesDataInconsistent();
-
- results->warnings.push_back(kTimeseriesValidationInconsistencyReason);
-}
-
-BSONObj rehydrateKey(const BSONObj& keyPattern, const BSONObj& indexKey) {
- // We need to rehydrate the indexKey for improved readability.
- // {"": ObjectId(...)} -> {"_id": ObjectId(...)}
- auto keysIt = keyPattern.begin();
- auto valuesIt = indexKey.begin();
-
- BSONObjBuilder b;
- while (keysIt != keyPattern.end()) {
- // keysIt and valuesIt must have the same number of elements.
- invariant(valuesIt != indexKey.end());
- b.appendAs(*valuesIt, keysIt->fieldName());
- keysIt++;
- valuesIt++;
- }
- return b.obj();
-}
} // namespace
Status ValidateAdaptor::validateRecord(OperationContext* opCtx,
const RecordId& recordId,
const RecordData& record,
size_t* dataSize,
- ValidateResults* results,
- ValidationVersion validationVersion) {
- const Status status = validateBSON(record.data(), record.size(), validationVersion);
+ ValidateResults* results) {
+ const Status status = validateBSON(record.data(), record.size());
if (!status.isOK())
return status;
BSONObj recordBson = record.toBson();
*dataSize = recordBson.objsize();
- if (MONGO_unlikely(_validateState->logDiagnostics())) {
+ if (MONGO_unlikely(_validateState->extraLoggingForTest())) {
LOGV2(4666601, "[validate]", "recordId"_attr = recordId, "recordData"_attr = recordBson);
}
@@ -361,26 +187,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx,
{multikeyMetadataKeys->begin(), multikeyMetadataKeys->end()},
*documentMultikeyPaths);
- auto printMultikeyMetadata = [&]() {
- LOGV2(7556100,
- "Index is not multikey but document has multikey data",
- "indexName"_attr = descriptor->indexName(),
- "recordId"_attr = recordId,
- "record"_attr = redact(recordBson));
- for (auto& key : *documentKeySet) {
- auto indexKey = KeyString::toBsonSafe(key.getBuffer(),
- key.getSize(),
- iam->getSortedDataInterface()->getOrdering(),
- key.getTypeBits());
- const BSONObj rehydratedKey = rehydrateKey(descriptor->keyPattern(), indexKey);
- LOGV2(7556101,
- "Index key for document with multikey inconsistency",
- "indexName"_attr = descriptor->indexName(),
- "recordId"_attr = recordId,
- "indexKey"_attr = redact(rehydratedKey));
- }
- };
-
if (!index->isMultikey(opCtx, coll) && shouldBeMultikey) {
if (_validateState->fixErrors()) {
writeConflictRetry(opCtx, "setIndexAsMultikey", coll->ns().ns(), [&] {
@@ -398,17 +204,10 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx,
<< " set to multikey.");
results->repaired = true;
} else {
- printMultikeyMetadata();
-
auto& curRecordResults = (results->indexResultsMap)[descriptor->indexName()];
- const std::string msg = fmt::format(
- "Index {} is not multikey but document with RecordId({}) and {} has multikey "
- "data, "
- "{} key(s)",
- descriptor->indexName(),
- recordId.toString(),
- recordBson.getField("_id").toString(),
- documentKeySet->size());
+ std::string msg = str::stream() << "Index " << descriptor->indexName()
+ << " is not multikey but has more than one"
+ << " key in document " << recordId;
curRecordResults.errors.push_back(msg);
curRecordResults.valid = false;
if (crashOnMultikeyValidateFailure.shouldFail()) {
@@ -436,8 +235,6 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx,
<< " multikey paths updated.");
results->repaired = true;
} else {
- printMultikeyMetadata();
-
std::string msg = str::stream()
<< "Index " << descriptor->indexName()
<< " multikey paths do not cover a document. RecordId: " << recordId;
@@ -470,7 +267,7 @@ Status ValidateAdaptor::validateRecord(OperationContext* opCtx,
for (const auto& keyString : *documentKeySet) {
try {
_totalIndexKeys++;
- _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId, results);
+ _indexConsistency->addDocKey(opCtx, keyString, &indexInfo, recordId);
} catch (...) {
return exceptionToStatus();
}
@@ -483,16 +280,15 @@ namespace {
// Ensures that index entries are in increasing or decreasing order.
void _validateKeyOrder(OperationContext* opCtx,
const IndexCatalogEntry* index,
- const KeyStringEntry& currKey,
- const KeyStringEntry& prevKey,
+ const KeyString::Value& currKey,
+ const KeyString::Value& prevKey,
IndexValidateResults* results) {
auto descriptor = index->descriptor();
bool unique = descriptor->unique();
// KeyStrings will be in strictly increasing order because all keys are sorted and they are in
// the format (Key, RID), and all RecordIDs are unique.
- if (currKey.keyString.compare(prevKey.keyString) <= 0 ||
- MONGO_unlikely(failIndexKeyOrdering.shouldFail())) {
+ if (currKey.compare(prevKey) <= 0) {
if (results && results->valid) {
results->errors.push_back(str::stream()
<< "index '" << descriptor->indexName()
@@ -506,20 +302,21 @@ void _validateKeyOrder(OperationContext* opCtx,
if (unique) {
// Unique indexes must not have duplicate keys.
- int cmp = currKey.loc.isLong()
- ? currKey.keyString.compareWithoutRecordIdLong(prevKey.keyString)
- : currKey.keyString.compareWithoutRecordIdStr(prevKey.keyString);
+ int cmp = currKey.compareWithoutRecordIdLong(prevKey);
if (cmp != 0) {
return;
}
if (results && results->valid) {
- auto bsonKey =
- KeyString::toBson(currKey.keyString, Ordering::make(descriptor->keyPattern()));
+ auto bsonKey = KeyString::toBson(currKey, Ordering::make(descriptor->keyPattern()));
+ auto firstRecordId =
+ KeyString::decodeRecordIdLongAtEnd(prevKey.getBuffer(), prevKey.getSize());
+ auto secondRecordId =
+ KeyString::decodeRecordIdLongAtEnd(currKey.getBuffer(), currKey.getSize());
results->errors.push_back(str::stream() << "Unique index '" << descriptor->indexName()
<< "' has duplicate key: " << bsonKey
- << ", first record: " << prevKey.loc
- << ", second record: " << currKey.loc);
+ << ", first record: " << firstRecordId
+ << ", second record: " << secondRecordId);
}
if (results) {
results->valid = false;
@@ -538,6 +335,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
IndexInfo& indexInfo = _indexConsistency->getIndexInfo(indexName);
int64_t numKeys = 0;
+ bool isFirstEntry = true;
+
// The progress meter will be inactive after traversing the record store to allow the message
// and the total to be set to different values.
if (!_progress->isActive()) {
@@ -552,7 +351,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
KeyString::Builder firstKeyStringBuilder(
version, BSONObj(), indexInfo.ord, KeyString::Discriminator::kExclusiveBefore);
KeyString::Value firstKeyString = firstKeyStringBuilder.getValueCopy();
- boost::optional<KeyStringEntry> prevIndexKeyStringEntry;
+ KeyString::Value prevIndexKeyStringValue;
// Ensure that this index has an open index cursor.
const auto indexCursorIt = _validateState->getIndexCursors().find(indexName);
@@ -583,8 +382,9 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
bool foundOldUniqueIndexKeys = false;
while (indexEntry) {
- if (prevIndexKeyStringEntry) {
- _validateKeyOrder(opCtx, index, *indexEntry, *prevIndexKeyStringEntry, &indexResults);
+ if (!isFirstEntry) {
+ _validateKeyOrder(
+ opCtx, index, indexEntry->keyString, prevIndexKeyStringValue, &indexResults);
}
if (!foundOldUniqueIndexKeys && !descriptor->isIdIndex() && descriptor->unique() &&
@@ -615,7 +415,8 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
_progress->hit();
numKeys++;
- prevIndexKeyStringEntry = indexEntry;
+ isFirstEntry = false;
+ prevIndexKeyStringValue = indexEntry->keyString;
if (numKeys % kInterruptIntervalNumRecords == 0) {
// Periodically checks for interrupts and yields.
@@ -631,7 +432,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
"Error advancing index cursor",
"error"_attr = ex.toString(),
"index"_attr = indexName,
- "prevKey"_attr = prevIndexKeyStringEntry->keyString.toString());
+ "prevKey"_attr = prevIndexKeyStringValue.toString());
}
throw;
}
@@ -717,8 +518,7 @@ void ValidateAdaptor::traverseIndex(OperationContext* opCtx,
void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
ValidateResults* results,
- BSONObjBuilder* output,
- ValidationVersion validationVersion) {
+ BSONObjBuilder* output) {
_numRecords = 0; // need to reset it because this function can be called more than once.
long long dataSizeTotal = 0;
long long interruptIntervalNumBytes = 0;
@@ -742,10 +542,9 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
// Because the progress meter is intended as an approximation, it's sufficient to get the number
// of records when we begin traversing, even if this number may deviate from the final number.
- const auto& coll = _validateState->getCollection();
const char* curopMessage = "Validate: scanning documents";
- const auto totalRecords = coll->getRecordStore()->numRecords(opCtx);
- const auto rs = coll->getRecordStore();
+ const auto totalRecords = _validateState->getCollection()->getRecordStore()->numRecords(opCtx);
+ const auto rs = _validateState->getCollection()->getRecordStore();
{
stdx::unique_lock<Client> lk(*opCtx->getClient());
_progress.set(CurOp::get(opCtx)->setProgress_inlock(curopMessage, totalRecords));
@@ -756,9 +555,6 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
return;
}
- bool bucketMixedSchemaDataError = false;
- bool bucketMinMaxMalformedError = false;
- bool bucketMixedSchemaDataWarning = false;
bool corruptRecordsSizeLimitWarning = false;
const std::unique_ptr<SeekableRecordThrottleCursor>& traverseRecordStoreCursor =
_validateState->getTraverseRecordStoreCursor();
@@ -772,25 +568,11 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
interruptIntervalNumBytes += dataSize;
dataSizeTotal += dataSize;
size_t validatedSize = 0;
- Status status = validateRecord(
- opCtx, record->id, record->data, &validatedSize, results, validationVersion);
-
- // Log the out-of-order entries as errors.
- //
- // Validate uses a DataCorruptionDetectionMode::kLogAndContinue mode such that data
- // corruption errors are logged without throwing, so certain checks must be duplicated here
- // as well.
- if ((prevRecordId.isValid() && prevRecordId > record->id) ||
- MONGO_unlikely(failRecordStoreTraversal.shouldFail())) {
- // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list and
- // not care about deduplication.
- static constexpr auto kErrorMessage = "Detected out-of-order documents. See logs.";
- if (results->valid ||
- std::find(results->errors.begin(), results->errors.end(), kErrorMessage) ==
- results->errors.end()) {
- results->errors.push_back(kErrorMessage);
- results->valid = false;
- }
+ Status status = validateRecord(opCtx, record->id, record->data, &validatedSize, results);
+
+ // RecordStores are required to return records in RecordId order.
+ if (prevRecordId.isValid()) {
+ invariant(prevRecordId < record->id);
}
// validatedSize = dataSize is not a general requirement as some storage engines may use
@@ -821,14 +603,8 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
results->numRemovedCorruptRecords++;
_numRecords--;
} else {
- // TODO SERVER-78040: Clean this up once we can insert errors blindly into the list
- // and not care about deduplication.
- static constexpr auto kErrorMessage =
- "Detected one or more invalid documents. See logs.";
- if (results->valid ||
- std::find(results->errors.begin(), results->errors.end(), kErrorMessage) ==
- results->errors.end()) {
- results->errors.push_back(kErrorMessage);
+ if (results->valid) {
+ results->errors.push_back("Detected one or more invalid documents. See logs.");
results->valid = false;
}
@@ -847,76 +623,18 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
// If the document is not corrupted, validate the document against this collection's
// schema validator. Don't treat invalid documents as errors since documents can bypass
// document validation when being inserted or updated.
- auto result = coll->checkValidation(opCtx, record->data.toBson());
+ auto result =
+ _validateState->getCollection()->checkValidation(opCtx, record->data.toBson());
if (result.first != Collection::SchemaValidationResult::kPass) {
LOGV2_WARNING(5363500,
"Document is not compliant with the collection's schema",
- logAttrs(coll->ns()),
+ logAttrs(_validateState->getCollection()->ns()),
"recordId"_attr = record->id,
"reason"_attr = result.second);
nNonCompliantDocuments++;
schemaValidationFailed(_validateState, result.first, results);
- } else if (coll->getTimeseriesOptions()) {
- // Checks for time-series collection consistency.
- Status bucketStatus =
- _validateTimeSeriesBucketRecord(coll, record->data.toBson(), results);
-
- // This log id should be kept in sync with the associated warning messages that are
- // returned to the client.
- if (!bucketStatus.isOK()) {
- LOGV2_WARNING(6698300,
- "Document is not compliant with time-series specifications",
- logAttrs(coll->ns()),
- "recordId"_attr = record->id,
- "reason"_attr = bucketStatus);
- nNonCompliantDocuments++;
- _timeseriesValidationFailed(_validateState, results);
- } else {
- auto containsMixedSchemaDataResponse =
- coll->doesTimeseriesBucketsDocContainMixedSchemaData(record->data.toBson());
- if (!containsMixedSchemaDataResponse.isOK() && !bucketMinMaxMalformedError) {
- bucketMinMaxMalformedError = true;
- LOGV2_WARNING(8469900,
- "Detected a time-series bucket with malformed min/max values",
- logAttrs(coll->ns()),
- "bucketId"_attr = record->id,
- "error"_attr = containsMixedSchemaDataResponse.getStatus());
- results->errors.push_back(
- str::stream()
- << "Detected a time-series bucket with malformed min/max values");
- results->valid = false;
- } else if (containsMixedSchemaDataResponse.isOK() &&
- containsMixedSchemaDataResponse.getValue()) {
- bool mixedSchemaAllowed =
- coll->getTimeseriesBucketsMayHaveMixedSchemaData().value_or(true);
- if (mixedSchemaAllowed && !bucketMixedSchemaDataWarning) {
- bucketMixedSchemaDataWarning = true;
- LOGV2_WARNING(8469901,
- "Detected a time-series bucket with mixed schema data",
- logAttrs(coll->ns()),
- "bucketId"_attr = record->id);
- results->warnings.push_back(
- str::stream()
- << "Detected a time-series bucket with mixed schema data");
- } else if (!mixedSchemaAllowed && !bucketMixedSchemaDataError) {
- bucketMixedSchemaDataError = true;
- LOGV2_WARNING(8469902,
- "Detected a time-series bucket with mixed schema data "
- "when timeseriesBucketsMayHaveMixedSchemaData is false. "
- "You can run the collMod command to set this flag",
- logAttrs(coll->ns()),
- "bucketId"_attr = record->id);
- results->errors.push_back(
- str::stream()
- << "Detected a time-series bucket with mixed schema data when "
- "timeseriesBucketsMayHaveMixedSchemaData is false. You can run "
- "the collMod command to set this flag");
- results->valid = false;
- }
- }
- }
}
}
@@ -939,18 +657,20 @@ void ValidateAdaptor::traverseRecordStore(OperationContext* opCtx,
<< " invalid documents.");
}
- const auto fastCount = coll->numRecords(opCtx);
+ const auto fastCount = _validateState->getCollection()->numRecords(opCtx);
if (_validateState->shouldEnforceFastCount() && fastCount != _numRecords) {
- results->errors.push_back(
- str::stream() << "fast count (" << fastCount << ") does not match number of records ("
- << _numRecords << ") for collection '" << coll->ns() << "'");
+ results->errors.push_back(str::stream() << "fast count (" << fastCount
+ << ") does not match number of records ("
+ << _numRecords << ") for collection '"
+ << _validateState->getCollection()->ns() << "'");
results->valid = false;
}
// Do not update the record store stats if we're in the background as we've validated a
// checkpoint and it may not have the most up-to-date changes.
if (results->valid && !_validateState->isBackground()) {
- coll->getRecordStore()->updateStatsAfterRepair(opCtx, _numRecords, dataSizeTotal);
+ _validateState->getCollection()->getRecordStore()->updateStatsAfterRepair(
+ opCtx, _numRecords, dataSizeTotal);
}
}
diff --git a/src/mongo/db/catalog/validate_adaptor.h b/src/mongo/db/catalog/validate_adaptor.h
index 99c0a2fceb1..ba548ca6358 100644
--- a/src/mongo/db/catalog/validate_adaptor.h
+++ b/src/mongo/db/catalog/validate_adaptor.h
@@ -57,8 +57,7 @@ public:
const RecordId& recordId,
const RecordData& record,
size_t* dataSize,
- ValidateResults* results,
- ValidationVersion validationVersion = currentValidationVersion);
+ ValidateResults* results);
/**
* Traverses the index getting index entries to validate them and keep track of the index keys
@@ -75,8 +74,7 @@ public:
*/
void traverseRecordStore(OperationContext* opCtx,
ValidateResults* results,
- BSONObjBuilder* output,
- ValidationVersion validationVersion);
+ BSONObjBuilder* output);
/**
* Validates that the number of document keys matches the number of index keys previously
diff --git a/src/mongo/db/catalog/validate_results.h b/src/mongo/db/catalog/validate_results.h
index c02235a68bc..baa6d78ad42 100644
--- a/src/mongo/db/catalog/validate_results.h
+++ b/src/mongo/db/catalog/validate_results.h
@@ -30,7 +30,6 @@
#pragma once
#include <map>
-#include <set>
#include <string>
#include <vector>
@@ -45,6 +44,7 @@ struct IndexValidateResults {
std::vector<std::string> errors;
std::vector<std::string> warnings;
int64_t keysTraversed = 0;
+ int64_t keysTraversedFromFullValidate = 0;
int64_t keysRemovedFromRecordStore = 0;
};
@@ -60,9 +60,6 @@ struct ValidateResults {
std::vector<BSONObj> extraIndexEntries;
std::vector<BSONObj> missingIndexEntries;
std::vector<RecordId> corruptRecords;
- // Timestamps (startTs, startDurable, stopTs, stopDurableTs) related to records
- // with validation errors. See WiredTigerRecordStore::printRecordMetadata().
- std::set<Timestamp> recordTimestamps;
long long numRemovedCorruptRecords = 0;
long long numRemovedExtraIndexEntries = 0;
long long numInsertedMissingIndexEntries = 0;
diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp
index 01f80ac61cc..434950b9540 100644
--- a/src/mongo/db/catalog/validate_state.cpp
+++ b/src/mongo/db/catalog/validate_state.cpp
@@ -54,13 +54,12 @@ ValidateState::ValidateState(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
- const AdditionalOptions& additionalOptions,
- bool logDiagnostics)
+ bool turnOnExtraLoggingForTest)
: _nss(nss),
_mode(mode),
_repairMode(repairMode),
_dataThrottle(opCtx),
- _logDiagnostics(logDiagnostics) {
+ _extraLoggingForTest(turnOnExtraLoggingForTest) {
// Subsequent re-locks will use the UUID when 'background' is true.
if (isBackground()) {
@@ -81,26 +80,13 @@ ValidateState::ValidateState(OperationContext* opCtx,
_collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss);
if (!_collection) {
- auto view = CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss);
- if (!view) {
- uasserted(ErrorCodes::NamespaceNotFound,
- str::stream() << "Collection '" << _nss << "' does not exist to validate.");
- } else {
- // Uses the bucket collection in place of the time-series collection view.
- if (!view->timeseries()) {
- uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view");
- }
- _nss = _nss.makeTimeseriesBucketsNamespace();
- if (isBackground()) {
- _collectionLock.emplace(opCtx, _nss, MODE_IS);
- } else {
- _collectionLock.emplace(opCtx, _nss, MODE_X);
- }
- _collection = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, _nss);
+ if (CollectionCatalog::get(opCtx)->lookupView(opCtx, _nss)) {
+ uasserted(ErrorCodes::CommandNotSupportedOnView, "Cannot validate a view");
}
- }
- _validationVersion = additionalOptions.validationVersion;
+ uasserted(ErrorCodes::NamespaceNotFound,
+ str::stream() << "Collection '" << _nss << "' does not exist to validate.");
+ }
// RepairMode is incompatible with the ValidateModes kBackground and
// kForegroundFullEnforceFastCount.
@@ -252,16 +238,15 @@ void ValidateState::initializeCursors(OperationContext* opCtx) {
const IndexCatalog* indexCatalog = _collection->getIndexCatalog();
// The index iterator for ready indexes is timestamp-aware and will only return indexes that
// are visible at our read time.
- const auto it = indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady);
+ const std::unique_ptr<IndexCatalog::IndexIterator> it =
+ indexCatalog->getIndexIterator(opCtx, /*includeUnfinished*/ false);
while (it->more()) {
const IndexCatalogEntry* entry = it->next();
const IndexDescriptor* desc = entry->descriptor();
auto iam = entry->accessMethod()->asSortedData();
- if (!iam) {
- _skippedIndexes.emplace(desc->indexName());
+ if (!iam)
continue;
- }
_indexCursors.emplace(
desc->indexName(),
diff --git a/src/mongo/db/catalog/validate_state.h b/src/mongo/db/catalog/validate_state.h
index bb8f431d2d6..df796c686ce 100644
--- a/src/mongo/db/catalog/validate_state.h
+++ b/src/mongo/db/catalog/validate_state.h
@@ -55,12 +55,15 @@ class ValidateState {
ValidateState& operator=(const ValidateState&) = delete;
public:
+ /**
+ * 'turnOnExtraLoggingForTest' turns on extra logging for test debugging. This parameter is for
+ * unit testing only.
+ */
ValidateState(OperationContext* opCtx,
const NamespaceString& nss,
ValidateMode mode,
RepairMode repairMode,
- const AdditionalOptions& additionalOptions,
- bool logDiagnostics);
+ bool turnOnExtraLoggingForTest = false);
const NamespaceString& nss() const {
return _nss;
@@ -93,13 +96,6 @@ public:
_collectionSchemaViolated = true;
}
- bool isTimeseriesDataInconsistent() {
- return _timeseriesDataInconsistency;
- }
- void setTimeseriesDataInconsistent() {
- _timeseriesDataInconsistency = true;
- }
-
bool fixErrors() const {
return _repairMode == RepairMode::kFixErrors;
}
@@ -127,10 +123,6 @@ public:
return _indexes;
}
- const StringSet& getSkippedIndexes() const {
- return _skippedIndexes;
- }
-
/**
* Map of index names to index cursors.
*/
@@ -170,13 +162,11 @@ public:
/**
* Indicates whether extra logging should occur during validation.
+ *
+ * This is for unit testing only. Intended to improve diagnosibility.
*/
- bool logDiagnostics() {
- return _logDiagnostics;
- }
-
- ValidationVersion validationVersion() const {
- return _validationVersion;
+ bool extraLoggingForTest() {
+ return _extraLoggingForTest;
}
boost::optional<Timestamp> getValidateTimestamp() {
@@ -221,8 +211,6 @@ private:
ValidateMode _mode;
RepairMode _repairMode;
bool _collectionSchemaViolated = false;
- bool _timeseriesDataInconsistency = false;
- ValidationVersion _validationVersion = currentValidationVersion;
boost::optional<ShouldNotConflictWithSecondaryBatchApplicationBlock> _noPBWM;
boost::optional<Lock::GlobalLock> _globalLock;
@@ -247,10 +235,6 @@ private:
std::unique_ptr<SeekableRecordThrottleCursor> _traverseRecordStoreCursor;
std::unique_ptr<SeekableRecordThrottleCursor> _seekRecordStoreCursor;
- // Stores the set of indexes that will not be validated for some reason, e.g. they are not
- // ready.
- StringSet _skippedIndexes;
-
RecordId _firstRecordId;
DataThrottle _dataThrottle;
@@ -258,8 +242,8 @@ private:
// Used to detect when the catalog is re-opened while yielding locks.
uint64_t _catalogGeneration;
- // Can be set to obtain better insight into what validate sees/does.
- bool _logDiagnostics;
+ // Can be set by unit tests to obtain better insight into what validate sees/does.
+ bool _extraLoggingForTest;
boost::optional<Timestamp> _validateTs = boost::none;
};
diff --git a/src/mongo/db/catalog/validate_state_test.cpp b/src/mongo/db/catalog/validate_state_test.cpp
index 3e4222da54b..e8e4bfbbbd9 100644
--- a/src/mongo/db/catalog/validate_state_test.cpp
+++ b/src/mongo/db/catalog/validate_state_test.cpp
@@ -157,9 +157,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError
CollectionValidation::ValidateState(opCtx,
kNss,
CollectionValidation::ValidateMode::kForeground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false),
+ CollectionValidation::RepairMode::kNone),
AssertionException,
ErrorCodes::NamespaceNotFound);
@@ -167,9 +165,7 @@ TEST_F(ValidateStateTest, NonExistentCollectionShouldThrowNamespaceNotFoundError
CollectionValidation::ValidateState(opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false),
+ CollectionValidation::RepairMode::kNone),
AssertionException,
ErrorCodes::NamespaceNotFound);
}
@@ -188,9 +184,7 @@ TEST_F(ValidateStateTest, UncheckpointedCollectionShouldBeAbleToInitializeCursor
opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
// Assert that cursors are able to created on the new collection.
validateState.initializeCursors(opCtx);
// There should only be a first record id if cursors were initialized successfully.
@@ -217,9 +211,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) {
opCtx,
kNss,
CollectionValidation::ValidateMode::kForeground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
validateState.initializeCursors(opCtx);
// Make sure all of the indexes were found and cursors opened against them. Including the
@@ -237,9 +229,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexes) {
opCtx,
kNss,
CollectionValidation::ValidateMode::kForeground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexes().size(), 5);
}
@@ -266,9 +256,7 @@ TEST_F(ValidateStateTest, OpenCursorsOnAllIndexesWithBackground) {
opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
validateState.initializeCursors(opCtx);
// We should be able to open a cursor on each index.
@@ -305,9 +293,7 @@ TEST_F(ValidateStateTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatWereL
opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexes().size(), 3);
}
@@ -320,9 +306,7 @@ TEST_F(ValidateStateTest, CursorsAreNotOpenedAgainstCheckpointedIndexesThatWereL
opCtx,
kNss,
CollectionValidation::ValidateMode::kBackground,
- CollectionValidation::RepairMode::kNone,
- /*additionalOptions=*/{},
- /*logDiagnostics=*/false);
+ CollectionValidation::RepairMode::kNone);
validateState.initializeCursors(opCtx);
ASSERT_EQ(validateState.getIndexes().size(), 3);
}
diff --git a/src/mongo/db/catalog/views_for_database.cpp b/src/mongo/db/catalog/views_for_database.cpp
index 404297c11a3..776cf5e3266 100644
--- a/src/mongo/db/catalog/views_for_database.cpp
+++ b/src/mongo/db/catalog/views_for_database.cpp
@@ -61,8 +61,47 @@ std::shared_ptr<const ViewDefinition> ViewsForDatabase::lookup(const NamespaceSt
}
Status ViewsForDatabase::reload(OperationContext* opCtx) {
+ auto reloadCallback = [&](const BSONObj& view) -> Status {
+ BSONObj collationSpec = view.hasField("collation") ? view["collation"].Obj() : BSONObj();
+ auto collator = parseCollator(opCtx, collationSpec);
+ if (!collator.isOK()) {
+ return collator.getStatus();
+ }
+
+ NamespaceString viewName(view["_id"].str());
+
+ auto pipeline = view["pipeline"].Obj();
+ for (auto&& stage : pipeline) {
+ if (BSONType::Object != stage.type()) {
+ return Status(ErrorCodes::InvalidViewDefinition,
+ str::stream() << "View 'pipeline' entries must be objects, but "
+ << viewName.toString()
+ << " has a pipeline element of type " << stage.type());
+ }
+ }
+
+ auto viewDef = std::make_shared<ViewDefinition>(viewName.db(),
+ viewName.coll(),
+ view["viewOn"].str(),
+ pipeline,
+ std::move(collator.getValue()));
+
+ if (!viewName.isOnInternalDb() && !viewName.isSystem()) {
+ if (viewDef->timeseries()) {
+ stats.userTimeseries += 1;
+ } else {
+ stats.userViews += 1;
+ }
+ } else {
+ stats.internal += 1;
+ }
+
+ viewMap[viewName.ns()] = std::move(viewDef);
+ return Status::OK();
+ };
+
try {
- durable->iterate(opCtx, [&](const BSONObj& view) { return _insert(opCtx, view); });
+ durable->iterate(opCtx, reloadCallback);
} catch (const DBException& ex) {
auto status = ex.toStatus();
LOGV2(22547,
@@ -71,60 +110,9 @@ Status ViewsForDatabase::reload(OperationContext* opCtx) {
"error"_attr = status);
return status;
}
- valid = true;
- return Status::OK();
-}
-
-Status ViewsForDatabase::insert(OperationContext* opCtx, const BSONObj& view) {
- auto status = _insert(opCtx, view);
- if (!status.isOK()) {
- LOGV2(5387000,
- "Could not insert view",
- "db"_attr = durable->getName(),
- "error"_attr = status);
- return status;
- }
valid = true;
- return Status::OK();
-};
-
-Status ViewsForDatabase::_insert(OperationContext* opCtx, const BSONObj& view) {
- BSONObj collationSpec = view.hasField("collation") ? view["collation"].Obj() : BSONObj();
- auto collator = parseCollator(opCtx, collationSpec);
- if (!collator.isOK()) {
- return collator.getStatus();
- }
-
- NamespaceString viewName(view["_id"].str());
-
- auto pipeline = view["pipeline"].Obj();
- for (auto&& stage : pipeline) {
- if (BSONType::Object != stage.type()) {
- return Status(ErrorCodes::InvalidViewDefinition,
- str::stream() << "View 'pipeline' entries must be objects, but "
- << viewName.toString() << " has a pipeline element of type "
- << stage.type());
- }
- }
-
- auto viewDef = std::make_shared<ViewDefinition>(viewName.db(),
- viewName.coll(),
- view["viewOn"].str(),
- pipeline,
- std::move(collator.getValue()));
-
- if (!viewName.isOnInternalDb() && !viewName.isSystem()) {
- if (viewDef->timeseries()) {
- stats.userTimeseries += 1;
- } else {
- stats.userViews += 1;
- }
- } else {
- stats.internal += 1;
- }
- viewMap[viewName.ns()] = std::move(viewDef);
return Status::OK();
}
@@ -147,8 +135,7 @@ Status ViewsForDatabase::validateCollation(OperationContext* opCtx,
Status ViewsForDatabase::upsertIntoGraph(OperationContext* opCtx,
const ViewDefinition& viewDef,
- const PipelineValidatorFn& validatePipeline,
- const bool needsValidation) {
+ const PipelineValidatorFn& validatePipeline) {
// Performs the insert into the graph.
auto doInsert = [this, opCtx, &validatePipeline](const ViewDefinition& viewDef,
bool needsValidation) -> Status {
@@ -203,7 +190,7 @@ Status ViewsForDatabase::upsertIntoGraph(OperationContext* opCtx,
// is simply a no-op.
viewGraph.remove(viewDef.name());
- return doInsert(viewDef, needsValidation);
+ return doInsert(viewDef, true);
}
} // namespace mongo
diff --git a/src/mongo/db/catalog/views_for_database.h b/src/mongo/db/catalog/views_for_database.h
index ab4329bf9d9..914adf60df7 100644
--- a/src/mongo/db/catalog/views_for_database.h
+++ b/src/mongo/db/catalog/views_for_database.h
@@ -93,11 +93,6 @@ public:
Status reload(OperationContext* opCtx);
/**
- * Inserts the view into the view map.
- */
- Status insert(OperationContext* opCtx, const BSONObj& view);
-
- /**
* Returns Status::OK if each view namespace in 'refs' has the same default collation as
* 'view'. Otherwise, returns ErrorCodes::OptionNotSupportedOnView.
*/
@@ -108,17 +103,11 @@ public:
/**
* Parses the view definition pipeline, attempts to upsert into the view graph, and
* refreshes the graph if necessary. Returns an error status if the resulting graph
- * would be invalid. needsValidation can be set to false if the view already exists in the
- * durable view catalog and skips checking that the resulting dependency graph is acyclic and
- * within the maximum depth.
+ * would be invalid.
*/
Status upsertIntoGraph(OperationContext* opCtx,
const ViewDefinition& viewDef,
- const PipelineValidatorFn&,
- bool needsValidation);
-
-private:
- Status _insert(OperationContext* opCtx, const BSONObj& view);
+ const PipelineValidatorFn&);
};
} // namespace mongo