diff options
Diffstat (limited to 'src/mongo/db/s')
235 files changed, 3612 insertions, 12474 deletions
diff --git a/src/mongo/db/s/README.md b/src/mongo/db/s/README.md index c0a3323ecd2..04c3ac2adcc 100644 --- a/src/mongo/db/s/README.md +++ b/src/mongo/db/s/README.md @@ -1178,9 +1178,6 @@ If mongos does not support `hello`, the `helloOk` flag is ignored. A new driver not see "helloOk: true" in the response and must continue to send `isMaster` on this connection. Old drivers will not specify this flag at all, so the behavior remains the same. -When mongos establishes outgoing connections to mongod nodes in the cluster, it always uses `hello` -rather than `isMaster`. - #### Code references * [isMaster command](https://github.com/mongodb/mongo/blob/r4.8.0-alpha/src/mongo/s/commands/cluster_is_master_cmd.cpp#L248) for mongos. * [hello command](https://github.com/mongodb/mongo/blob/r4.8.0-alpha/src/mongo/s/commands/cluster_is_master_cmd.cpp#L64) for mongos. @@ -1204,7 +1201,7 @@ consistent. When a DDL request is received by a router, it gets forwarded to the of the targeted database. For the sake of clarity, createDatabase is the only DDL operation that cannot possibly get forwarded to the database primary but is instead routed to the config server, as the database may not exist yet. -##### Serialization and joinability of DDL operations +##### Serialization and joinability of DDL operations When a primary shard receives a DDL request, it tries to construct a DDL coordinator performing the following steps: - Acquire the [distributed lock for the database](https://github.com/mongodb/mongo/blob/908e394d39b223ce498fde0d40e18c9200c188e2/src/mongo/db/s/sharding_ddl_coordinator.cpp#L155). This ensures that at most one DDL operation at a time will run for namespaces belonging to the same database on that particular primary node. - Acquire the distributed lock for the [collection](https://github.com/mongodb/mongo/blob/908e394d39b223ce498fde0d40e18c9200c188e2/src/mongo/db/s/sharding_ddl_coordinator.cpp#L171) (or [collections](https://github.com/mongodb/mongo/blob/908e394d39b223ce498fde0d40e18c9200c188e2/src/mongo/db/s/sharding_ddl_coordinator.cpp#L181)) involved in the operation. @@ -1217,7 +1214,7 @@ on the shard in order to join the ongoing operation if the options match (same o Once the distributed locks have been acquired, it is guaranteed that no other concurrent DDLs are happening for the same database, hence a DDL coordinator can safely start [executing the operation](https://github.com/mongodb/mongo/blob/master/src/mongo/db/s/sharding_ddl_coordinator.cpp#L207). -As first step, each coordinator is required to [majority commit a document](https://github.com/mongodb/mongo/blob/2ae2bcedfb7d48e64843dd56b9e4f107c56944b6/src/mongo/db/s/sharding_ddl_coordinator.h#L105-L116) - +As first step, each coordinator is required to [majority commit a document](https://github.com/mongodb/mongo/blob/2ae2bcedfb7d48e64843dd56b9e4f107c56944b6/src/mongo/db/s/sharding_ddl_coordinator.h#L105-L116) - that we will refer to as state document - containing all information regarding the running operation such as name of the DDL, namespaces involved and other metadata identifying the original request. At this point, the coordinator is entitled to start making both local and remote catalog modifications, eventually after blocking CRUD operations on the changing namespaces; when the execution reaches relevant diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index ed4697636bf..237dad8bcf8 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -17,7 +17,6 @@ env.Library( 'database_sharding_state.cpp', 'global_user_write_block_state.cpp', 'operation_sharding_state.cpp', - 'range_deletion_task.idl', 'shard_key_index_util.cpp', 'sharding_api_d_params.idl', 'sharding_migration_critical_section.cpp', @@ -33,7 +32,6 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/index_catalog', '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/range_arithmetic', - '$BUILD_DIR/mongo/db/server_feature_flags', '$BUILD_DIR/mongo/s/grid', '$BUILD_DIR/mongo/s/sharding_routing_table', ], @@ -43,25 +41,13 @@ env.Library( ) env.Library( - target='balancer_stats_registry', - source=[ - 'balancer_stats_registry.cpp', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/catalog_raii', - '$BUILD_DIR/mongo/db/dbdirectclient', - '$BUILD_DIR/mongo/db/repl/replica_set_aware_service', - '$BUILD_DIR/mongo/s/grid', - ], -) - -env.Library( target='sharding_runtime_d', source=[ 'active_migrations_registry.cpp', 'auto_split_vector.cpp', 'chunk_move_write_concern_options.cpp', 'chunk_split_state_driver.cpp', + 'balancer_stats_registry.cpp', 'chunk_splitter.cpp', 'collection_critical_section_document.idl', 'collection_sharding_runtime.cpp', @@ -69,8 +55,6 @@ env.Library( 'config_server_op_observer.cpp', 'global_index_metrics.cpp', 'metadata_manager.cpp', - 'migration_batch_fetcher.cpp', - 'migration_batch_inserter.cpp', 'migration_chunk_cloner_source_legacy.cpp', 'migration_chunk_cloner_source.cpp', 'migration_coordinator_document.idl', @@ -86,6 +70,7 @@ env.Library( 'op_observer_sharding_impl.cpp', 'periodic_balancer_config_refresher.cpp', 'periodic_sharded_index_consistency_checker.cpp', + 'range_deletion_task.idl', 'range_deletion_util.cpp', 'read_only_catalog_cache_loader.cpp', 'recoverable_critical_section_service.cpp', @@ -152,11 +137,9 @@ env.Library( '$BUILD_DIR/mongo/db/catalog/multi_index_block', '$BUILD_DIR/mongo/db/client_metadata_propagation_egress_hook', '$BUILD_DIR/mongo/db/commands/mongod_fcv', - '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/db_raii', '$BUILD_DIR/mongo/db/dbhelpers', '$BUILD_DIR/mongo/db/op_observer_impl', - '$BUILD_DIR/mongo/db/op_observer_util', '$BUILD_DIR/mongo/db/ops/write_ops_exec', '$BUILD_DIR/mongo/db/pipeline/aggregation_request_helper', '$BUILD_DIR/mongo/db/repl/abstract_async_component', @@ -171,7 +154,6 @@ env.Library( '$BUILD_DIR/mongo/s/query/cluster_aggregate', '$BUILD_DIR/mongo/s/sharding_api', '$BUILD_DIR/mongo/s/sharding_initialization', - 'forwardable_operation_metadata', 'sharding_api_d', 'sharding_catalog_manager', 'sharding_logging', @@ -188,7 +170,6 @@ env.Library( '$BUILD_DIR/mongo/db/session_catalog', '$BUILD_DIR/mongo/idl/server_parameter', '$BUILD_DIR/mongo/util/future_util', - 'balancer_stats_registry', ], ) @@ -303,17 +284,14 @@ env.Library( '$BUILD_DIR/mongo/db/commands/set_cluster_parameter_invocation', '$BUILD_DIR/mongo/db/commands/set_feature_compatibility_version_idl', '$BUILD_DIR/mongo/db/common', - '$BUILD_DIR/mongo/db/concurrency/exception_util', '$BUILD_DIR/mongo/db/dbdirectclient', '$BUILD_DIR/mongo/db/internal_transactions_feature_flag', - '$BUILD_DIR/mongo/db/keys_collection_util', '$BUILD_DIR/mongo/db/pipeline/process_interface/shardsvr_process_interface', '$BUILD_DIR/mongo/db/pipeline/sharded_agg_helpers', '$BUILD_DIR/mongo/db/repl/hello_command', '$BUILD_DIR/mongo/db/repl/read_concern_args', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replica_set_aware_service', - '$BUILD_DIR/mongo/db/repl/wait_for_majority_service', '$BUILD_DIR/mongo/db/rw_concern_d', '$BUILD_DIR/mongo/db/server_feature_flags', '$BUILD_DIR/mongo/db/snapshot_window_options', @@ -360,7 +338,6 @@ env.Library( 'config/configsvr_clear_jumbo_flag_command.cpp', 'config/configsvr_collmod_command.cpp', 'config/configsvr_commit_chunk_migration_command.cpp', - 'config/configsvr_commit_move_primary_command.cpp', 'config/configsvr_commit_reshard_collection_command.cpp', 'config/configsvr_configure_collection_balancing.cpp', 'config/configsvr_control_balancer_command.cpp', @@ -401,6 +378,7 @@ env.Library( 'flush_routing_table_cache_updates_command.cpp', 'get_database_version_command.cpp', 'get_shard_version_command.cpp', + 'get_stats_for_balancing.idl', 'migration_chunk_cloner_source_legacy_commands.cpp', 'migration_destination_manager_legacy_commands.cpp', 'move_primary_coordinator_document.idl', @@ -489,7 +467,6 @@ env.Library( '$BUILD_DIR/mongo/s/commands/sharded_cluster_sharding_commands', '$BUILD_DIR/mongo/s/sharding_initialization', '$BUILD_DIR/mongo/s/sharding_router_api', - 'balancer_stats_registry', 'forwardable_operation_metadata', 'sharding_runtime_d', 'user_writes_recoverable_critical_section', @@ -502,7 +479,6 @@ env.Library( 'sessions_collection_config_server.cpp', ], LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/dbdirectclient', '$BUILD_DIR/mongo/db/pipeline/sharded_agg_helpers', '$BUILD_DIR/mongo/s/sessions_collection_sharded', @@ -574,7 +550,6 @@ env.CppUnitTest( 'global_index_metrics_test.cpp', 'implicit_collection_creation_test.cpp', 'metadata_manager_test.cpp', - 'migration_batch_fetcher_test.cpp', 'migration_chunk_cloner_source_legacy_test.cpp', 'migration_destination_manager_test.cpp', 'migration_session_id_test.cpp', @@ -592,7 +567,6 @@ env.CppUnitTest( 'resharding/resharding_donor_oplog_iterator_test.cpp', 'resharding/resharding_donor_recipient_common_test.cpp', 'resharding/resharding_donor_service_test.cpp', - 'resharding/resharding_future_util_test.cpp', 'resharding/resharding_metrics_new_test.cpp', 'resharding/resharding_metrics_test.cpp', 'resharding/resharding_oplog_applier_test.cpp', @@ -683,7 +657,6 @@ env.CppUnitTest( 'config/sharding_catalog_manager_clear_jumbo_flag_test.cpp', 'config/sharding_catalog_manager_commit_chunk_migration_test.cpp', 'config/sharding_catalog_manager_config_initialization_test.cpp', - 'config/sharding_catalog_manager_database_operations_test.cpp', 'config/sharding_catalog_manager_ensure_chunk_version_is_greater_than_test.cpp', 'config/sharding_catalog_manager_merge_chunks_test.cpp', 'config/sharding_catalog_manager_remove_shard_from_zone_test.cpp', diff --git a/src/mongo/db/s/active_migrations_registry.cpp b/src/mongo/db/s/active_migrations_registry.cpp index dcbf6ef4c97..3d275f9853d 100644 --- a/src/mongo/db/s/active_migrations_registry.cpp +++ b/src/mongo/db/s/active_migrations_registry.cpp @@ -136,29 +136,22 @@ StatusWith<ScopedReceiveChunk> ActiveMigrationsRegistry::registerReceiveChunk( OperationContext* opCtx, const NamespaceString& nss, const ChunkRange& chunkRange, - const ShardId& fromShardId, - bool waitForOngoingMigrations) { + const ShardId& fromShardId) { stdx::unique_lock<Latch> ul(_mutex); - if (waitForOngoingMigrations) { - opCtx->waitForConditionOrInterrupt(_chunkOperationsStateChangedCV, ul, [this] { - return !_migrationsBlocked && !_activeMoveChunkState && !_activeReceiveChunkState; - }); - } else { - opCtx->waitForConditionOrInterrupt( - _chunkOperationsStateChangedCV, ul, [this] { return !_migrationsBlocked; }); + opCtx->waitForConditionOrInterrupt( + _chunkOperationsStateChangedCV, ul, [this] { return !_migrationsBlocked; }); - if (_activeReceiveChunkState) { - return _activeReceiveChunkState->constructErrorStatus(); - } + if (_activeReceiveChunkState) { + return _activeReceiveChunkState->constructErrorStatus(); + } - if (_activeMoveChunkState) { - LOGV2(6386802, - "Rejecting receive chunk due to conflicting donate chunk in progress", - logAttrs(_activeMoveChunkState->args.getCommandParameter()), - "runningMigration"_attr = _activeMoveChunkState->args.toBSON({})); - return _activeMoveChunkState->constructErrorStatus(); - } + if (_activeMoveChunkState) { + LOGV2(6386802, + "Rejecting receive chunk due to conflicting donate chunk in progress", + logAttrs(_activeMoveChunkState->args.getCommandParameter()), + "runningMigration"_attr = _activeMoveChunkState->args.toBSON({})); + return _activeMoveChunkState->constructErrorStatus(); } _activeReceiveChunkState.emplace(nss, chunkRange, fromShardId); diff --git a/src/mongo/db/s/active_migrations_registry.h b/src/mongo/db/s/active_migrations_registry.h index 4d75a6845b0..c17180f5fb2 100644 --- a/src/mongo/db/s/active_migrations_registry.h +++ b/src/mongo/db/s/active_migrations_registry.h @@ -96,15 +96,12 @@ public: * ScopedReceiveChunk will unregister the migration when the ScopedReceiveChunk goes out of * scope. * - * Otherwise returns a ConflictingOperationInProgress error if waitForOngoingMigrations is false - * or waits for the ongoing migration/split/merge to finish and then registers the migration if - * waitForOngoingMigrations is true. + * Otherwise returns a ConflictingOperationInProgress error. */ StatusWith<ScopedReceiveChunk> registerReceiveChunk(OperationContext* opCtx, const NamespaceString& nss, const ChunkRange& chunkRange, - const ShardId& fromShardId, - bool waitForOngoingMigrations); + const ShardId& fromShardId); /** * If there are no migrations running on this shard, registers an active split or merge diff --git a/src/mongo/db/s/active_migrations_registry_test.cpp b/src/mongo/db/s/active_migrations_registry_test.cpp index 5ba12a1cd16..a84f159a074 100644 --- a/src/mongo/db/s/active_migrations_registry_test.cpp +++ b/src/mongo/db/s/active_migrations_registry_test.cpp @@ -236,8 +236,7 @@ TEST_F(MoveChunkRegistration, TestBlockingReceiveChunk) { _registry.registerReceiveChunk(opCtx.get(), NamespaceString("TestDB", "TestColl"), ChunkRange(BSON("Key" << -100), BSON("Key" << 100)), - ShardId("shard0001"), - false); + ShardId("shard0001")); ASSERT_OK(scopedReceiveChunk.getStatus()); @@ -331,8 +330,7 @@ TEST_F(MoveChunkRegistration, TestBlockingWhileReceiveInProgress) { _registry.registerReceiveChunk(operationContext(), NamespaceString("TestDB", "TestColl"), ChunkRange(BSON("Key" << -100), BSON("Key" << 100)), - ShardId("shard0001"), - false); + ShardId("shard0001")); ASSERT_OK(scopedReceiveChunk.getStatus()); // 3. Signal the registry locking thread that the registry is ready to be locked. diff --git a/src/mongo/db/s/add_shard_cmd.cpp b/src/mongo/db/s/add_shard_cmd.cpp index af64045ca6a..3bfc1cc4a18 100644 --- a/src/mongo/db/s/add_shard_cmd.cpp +++ b/src/mongo/db/s/add_shard_cmd.cpp @@ -72,14 +72,6 @@ public: auto addShardCmd = request(); auto shardIdUpsertCmd = add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd); - - // A request dispatched through a local client is served within the same thread that - // submits it (so that the opCtx needs to be used as the vehicle to pass the WC to the - // ServiceEntryPoint). - const auto originalWC = opCtx->getWriteConcern(); - ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); }); - opCtx->setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern); - DBDirectClient localClient(opCtx); BSONObj res; diff --git a/src/mongo/db/s/add_shard_util.cpp b/src/mongo/db/s/add_shard_util.cpp index 750dbf98d23..957c2626e26 100644 --- a/src/mongo/db/s/add_shard_util.cpp +++ b/src/mongo/db/s/add_shard_util.cpp @@ -73,6 +73,7 @@ BSONObj createShardIdentityUpsertForAddShard(const AddShard& addShardCmd) { return updateOp; }()); + request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); return request.toBSON(); } diff --git a/src/mongo/db/s/auto_split_vector.cpp b/src/mongo/db/s/auto_split_vector.cpp index 2dcb509f156..8be8c5dde8b 100644 --- a/src/mongo/db/s/auto_split_vector.cpp +++ b/src/mongo/db/s/auto_split_vector.cpp @@ -50,16 +50,7 @@ namespace mongo { namespace { -/* - * BSON arrays are serialized as BSON objects with the index of each element as a string key: for - * example, the array ["a","b","c"] is going to be serialized as {"0":"a","1":"b","2":"c"}. The - * minimum size for a BSON object is `BSONObj::kMinBSONLength`. - * - * Given that the `vector<BSONObj>` returned by `autoSplitVector` can't be greater than 16MB when - * serialized, pessimistically assume that each key occupies the highest possible number of bytes. - */ -const int estimatedAdditionalBytesPerItemInBSONArray{ - (int)std::to_string(BSONObjMaxUserSize / BSONObj::kMinBSONLength).length()}; +constexpr int estimatedAdditionalBytesPerItemInBSONArray{2}; constexpr int kMaxSplitPointsToReposition{3}; @@ -94,11 +85,9 @@ const std::tuple<BSONObj, BSONObj> getMinMaxExtendedBounds(const ShardKeyIndex& /* * Reshuffle fields according to the shard key pattern. */ -auto orderShardKeyFields(const BSONObj& keyPattern, const BSONObj& key) { - // Note: It is correct to hydrate the indexKey 'key' with 'keyPattern', because the index key - // pattern is a prefix of 'keyPattern'. - return dotted_path_support::extractElementsBasedOnTemplate(key.replaceFieldNames(keyPattern), - keyPattern); +auto orderShardKeyFields(const BSONObj& keyPattern, BSONObj& key) { + return dotted_path_support::extractElementsBasedOnTemplate( + prettyKey(keyPattern, key.getOwned()), keyPattern); } } // namespace diff --git a/src/mongo/db/s/auto_split_vector_command.cpp b/src/mongo/db/s/auto_split_vector_command.cpp index dc4effd9b15..821527cac7d 100644 --- a/src/mongo/db/s/auto_split_vector_command.cpp +++ b/src/mongo/db/s/auto_split_vector_command.cpp @@ -39,8 +39,7 @@ namespace mongo { namespace { -static constexpr int64_t kSmallestChunkSizeBytesSupported = 1024 * 1024; -static constexpr int64_t kBiggestChunkSizeBytesSupported = 1024 * 1024 * 1024; +static constexpr int64_t kSmallestChunkSizeSupported = 1024 * 1024; class AutoSplitVectorCommand final : public TypedCommand<AutoSplitVectorCommand> { public: @@ -76,19 +75,16 @@ public: const auto& req = request(); uassert(ErrorCodes::ErrorCodes::InvalidOptions, - str::stream() << "maxChunksSizeBytes must lie within the range [" - << kSmallestChunkSizeBytesSupported / (1024 * 1024) << "MB, " - << kBiggestChunkSizeBytesSupported / (1024 * 1024) << "MB]", - req.getMaxChunkSizeBytes() >= kSmallestChunkSizeBytesSupported && - req.getMaxChunkSizeBytes() <= kBiggestChunkSizeBytesSupported); + str::stream() << "maxChunksSizeBytes cannot be smaller than " + << kSmallestChunkSizeSupported, + req.getMaxChunkSizeBytes() >= kSmallestChunkSizeSupported); auto [splitPoints, continuation] = autoSplitVector(opCtx, ns(), req.getKeyPattern(), req.getMin(), req.getMax(), - req.getMaxChunkSizeBytes(), - req.getLimit()); + req.getMaxChunkSizeBytes()); Response autoSplitVectorResponse(std::move(splitPoints)); autoSplitVectorResponse.setContinuation(continuation); return autoSplitVectorResponse; diff --git a/src/mongo/db/s/balancer/balancer.cpp b/src/mongo/db/s/balancer/balancer.cpp index cef33998f40..4487672302a 100644 --- a/src/mongo/db/s/balancer/balancer.cpp +++ b/src/mongo/db/s/balancer/balancer.cpp @@ -52,7 +52,6 @@ #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/sharding_config_server_parameters_gen.h" #include "mongo/db/s/sharding_logging.h" -#include "mongo/db/server_feature_flags_gen.h" #include "mongo/executor/scoped_task_executor.h" #include "mongo/logv2/log.h" #include "mongo/s/balancer_configuration.h" @@ -78,9 +77,14 @@ using std::vector; namespace { MONGO_FAIL_POINT_DEFINE(overrideBalanceRoundInterval); -MONGO_FAIL_POINT_DEFINE(forceBalancerWarningChecks); -const Milliseconds kBalanceRoundDefaultInterval(10 * 1000); +const Seconds kBalanceRoundDefaultInterval(10); + +// Sleep between balancer rounds in the case where the last round found some chunks which needed to +// be balanced. This value should be set sufficiently low so that imbalanced clusters will quickly +// reach balanced state, but setting it too low may cause CRUD operations to start failing due to +// not being able to establish a stable shard version. +const Seconds kShortBalanceRoundInterval(1); /** * Balancer status response @@ -97,19 +101,10 @@ class BalanceRoundDetails { public: BalanceRoundDetails() : _executionTimer() {} - void setSucceeded(int numCandidateChunks, - int numChunksMoved, - int numImbalancedCachedCollections, - Milliseconds selectionTime, - Milliseconds throttleTime, - Milliseconds migrationTime) { + void setSucceeded(int candidateChunks, int chunksMoved) { invariant(!_errMsg); - _numCandidateChunks = numCandidateChunks; - _numChunksMoved = numChunksMoved; - _numImbalancedCachedCollections = numImbalancedCachedCollections; - _selectionTime = selectionTime; - _throttleTime = throttleTime; - _migrationTime = migrationTime; + _candidateChunks = candidateChunks; + _chunksMoved = chunksMoved; } void setFailed(const string& errMsg) { @@ -124,33 +119,52 @@ public: if (_errMsg) { builder.append("errmsg", *_errMsg); } else { - builder.append("candidateChunks", _numCandidateChunks); - builder.append("chunksMoved", _numChunksMoved); - builder.append("imbalancedCachedCollections", _numImbalancedCachedCollections); - BSONObjBuilder timeInfo{builder.subobjStart("times"_sd)}; - timeInfo.append("selectionTimeMillis"_sd, _selectionTime.count()); - timeInfo.append("throttleTimeMillis"_sd, _throttleTime.count()); - timeInfo.append("migrationTimeMillis"_sd, _migrationTime.count()); - timeInfo.done(); + builder.append("candidateChunks", _candidateChunks); + builder.append("chunksMoved", _chunksMoved); } return builder.obj(); } private: const Timer _executionTimer; - Milliseconds _selectionTime; - Milliseconds _throttleTime; - Milliseconds _migrationTime; // Set only on success - int _numCandidateChunks{0}; - int _numChunksMoved{0}; - int _numImbalancedCachedCollections{0}; + int _candidateChunks{0}; + int _chunksMoved{0}; // Set only on failure boost::optional<string> _errMsg; }; +/** + * Occasionally prints a log message with shard versions if the versions are not the same + * in the cluster. + */ +void warnOnMultiVersion(const vector<ClusterStatistics::ShardStatistics>& clusterStats) { + static const auto& majorMinorRE = *new pcrecpp::RE(R"re(^(\d+)\.(\d+)\.)re"); + auto&& vii = VersionInfoInterface::instance(); + auto hasMyVersion = [&](auto&& stat) { + int major; + int minor; + return majorMinorRE.PartialMatch(pcrecpp::StringPiece(stat.mongoVersion), &major, &minor) && + major == vii.majorVersion() && minor == vii.minorVersion(); + }; + + // If we're all the same version, don't message + if (std::all_of(clusterStats.begin(), clusterStats.end(), hasMyVersion)) + return; + + BSONObjBuilder shardVersions; + for (const auto& stat : clusterStats) { + shardVersions << stat.shardId << stat.mongoVersion; + } + + LOGV2_WARNING(21875, + "Multiversion cluster detected", + "localVersion"_attr = vii.version(), + "shardVersions"_attr = shardVersions.done()); +} + Status processManualMigrationOutcome(OperationContext* opCtx, const BSONObj& chunkMin, const NamespaceString& nss, @@ -215,136 +229,6 @@ const auto _balancerDecoration = ServiceContext::declareDecoration<Balancer>(); const ReplicaSetAwareServiceRegistry::Registerer<Balancer> _balancerRegisterer("Balancer"); -/** - * Returns the names of shards that are currently draining. When the balancer is disabled, draining - * shards are stuck in this state as chunks cannot be migrated. - */ -std::vector<std::string> getDrainingShardNames(OperationContext* opCtx) { - // Find the shards that are currently draining. - const auto configShard{Grid::get(opCtx)->shardRegistry()->getConfigShard()}; - const auto drainingShardsDocs{ - uassertStatusOK( - configShard->exhaustiveFindOnConfig(opCtx, - ReadPreferenceSetting{ReadPreference::Nearest}, - repl::ReadConcernLevel::kMajorityReadConcern, - NamespaceString::kConfigsvrShardsNamespace, - BSON(ShardType::draining << true), - BSONObj() /* No sorting */, - boost::none /* No limit */)) - .docs}; - - // Build the list of the draining shard names. - std::vector<std::string> drainingShardNames; - drainingShardNames.reserve(drainingShardsDocs.size()); - std::transform(drainingShardsDocs.begin(), - drainingShardsDocs.end(), - std::back_inserter(drainingShardNames), - [](const auto& shardDoc) { - const auto shardEntry{uassertStatusOK(ShardType::fromBSON(shardDoc))}; - return shardEntry.getName(); - }); - return drainingShardNames; -} - -class BalancerWarning { - // Time interval between checks on draining shards. - constexpr static Minutes kDrainingShardsCheckInterval{10}; - -public: - BalancerWarning() = default; - - void warnIfRequired(OperationContext* opCtx, BalancerSettingsType::BalancerMode balancerMode) { - if (Date_t::now() - _lastDrainingShardsCheckTime < kDrainingShardsCheckInterval && - MONGO_likely(!forceBalancerWarningChecks.shouldFail())) { - return; - } - _lastDrainingShardsCheckTime = Date_t::now(); - - LOGV2(7977401, "Performing balancer warning checks"); - - const auto drainingShardNames{getDrainingShardNames(opCtx)}; - if (drainingShardNames.empty()) { - return; - } - - if (balancerMode == BalancerSettingsType::BalancerMode::kOff) { - LOGV2_WARNING( - 6434000, - "Draining of removed shards cannot be completed because the balancer is disabled", - "shards"_attr = drainingShardNames); - return; - } - - _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled(opCtx, drainingShardNames); - } - -private: - void _warnIfDrainingShardHasChunksForCollectionWithBalancingDisabled( - OperationContext* opCtx, const std::vector<std::string>& drainingShardNames) { - // Balancer is on, emit warning if balancer is disabled for collections which have chunks in - // shards in draining mode. - const auto catalogClient = Grid::get(opCtx)->catalogClient(); - auto collections = - catalogClient->getCollections(opCtx, - {}, - repl::ReadConcernLevel::kMajorityReadConcern, - BSON(CollectionType::kNssFieldName << 1)); - if (collections.empty()) { - return; - } - - // Construct BSONArray of draining shard names. - const auto drainingShardNameArray = [&]() { - BSONArrayBuilder shardNameArrayBuilder; - std::for_each(drainingShardNames.begin(), - drainingShardNames.end(), - [&shardNameArrayBuilder](const auto& shardName) { - shardNameArrayBuilder.append(shardName); - }); - return shardNameArrayBuilder.arr(); - }(); - - // For each collection, check if the collection has balancing disabled. If it is disabled, - // checks if the collection has any chunks in any of the draining shards. In which case a - // warning is emitted. - for (const auto& collType : collections) { - if (!collType.getAllowBalance() || !collType.getAllowMigrations() || - !collType.getPermitMigrations()) { - const auto findQuery = - BSON(ChunkType::collectionUUID() << collType.getUuid() << ChunkType::shard() - << BSON("$in" << drainingShardNameArray)); - - auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto findResponse = uassertStatusOK(configShard->exhaustiveFindOnConfig( - opCtx, - ReadPreferenceSetting(ReadPreference::PrimaryOnly), - repl::ReadConcernLevel::kMajorityReadConcern, - ChunkType::ConfigNS, - findQuery, - BSONObj(), - boost::none)); - - const auto& chunks = findResponse.docs; - if (!chunks.empty()) { - stdx::unordered_set<std::string> shardsWithChunks; - std::for_each( - chunks.begin(), chunks.end(), [&shardsWithChunks](const BSONObj& chunkObj) { - shardsWithChunks.emplace(chunkObj.getStringField(ChunkType::shard())); - }); - LOGV2_WARNING( - 7977400, - "Draining of removed shards cannot be completed because the balancer is " - "disabled for a collection which has chunks in those shards", - "uuid"_attr = collType.getUuid(), - "nss"_attr = collType.getNss(), - "shardsWithChunks"_attr = shardsWithChunks); - } - } - } - } - - Date_t _lastDrainingShardsCheckTime{Date_t::fromMillisSinceEpoch(0)}; -}; } // namespace Balancer* Balancer::get(ServiceContext* serviceContext) { @@ -365,8 +249,7 @@ Balancer::Balancer() _defragmentationPolicy(std::make_unique<BalancerDefragmentationPolicyImpl>( _clusterStats.get(), _random, [this]() { _onActionsStreamPolicyStateUpdate(); })), _clusterChunksResizePolicy(std::make_unique<ClusterChunksResizePolicyImpl>( - [this] { _onActionsStreamPolicyStateUpdate(); })), - _imbalancedCollectionsCache(std::make_unique<stdx::unordered_set<NamespaceString>>()) {} + [this] { _onActionsStreamPolicyStateUpdate(); })) {} Balancer::~Balancer() { // Terminate the balancer thread so it doesn't leak memory. @@ -397,7 +280,6 @@ void Balancer::onBecomeArbiter() { void Balancer::initiateBalancer(OperationContext* opCtx) { stdx::lock_guard<Latch> scopedLock(_mutex); - _imbalancedCollectionsCache->clear(); invariant(_state == kStopped); _state = kRunning; @@ -409,11 +291,11 @@ void Balancer::initiateBalancer(OperationContext* opCtx) { void Balancer::interruptBalancer() { stdx::lock_guard<Latch> scopedLock(_mutex); - if (_state != kRunning) { + if (_state != kRunning) return; - } _state = kStopping; + _thread.detach(); // Interrupt the balancer thread if it has been started. We are guaranteed that the operation // context of that thread is still alive, because we hold the balancer mutex. @@ -428,10 +310,8 @@ void Balancer::interruptBalancer() { void Balancer::waitForBalancerToStop() { stdx::unique_lock<Latch> scopedLock(_mutex); + _joinCond.wait(scopedLock, [this] { return _state == kStopped; }); - if (_thread.joinable()) { - _thread.join(); - } } void Balancer::joinCurrentRound(OperationContext* opCtx) { @@ -554,16 +434,15 @@ void Balancer::report(OperationContext* opCtx, BSONObjBuilder* builder) { builder->append("mode", BalancerSettingsType::kBalancerModes[mode]); builder->append("inBalancerRound", _inBalancerRound); builder->append("numBalancerRounds", _numBalancerRounds); - builder->append("term", repl::ReplicationCoordinator::get(opCtx)->getTerm()); } void Balancer::_consumeActionStreamLoop() { - Client::initThread("BalancerSecondary"); - { - stdx::lock_guard<Client> lk(cc()); - cc().setSystemOperationKillableByStepdown(lk); - } + ScopeGuard onExitCleanup([this] { + _defragmentationPolicy->interruptAllDefragmentations(); + _clusterChunksResizePolicy->stop(); + }); + Client::initThread("BalancerSecondary"); auto opCtx = cc().makeOperationContext(); // This thread never refreshes balancerConfig - instead, it relies on the requests // performed by _mainThread() on each round to eventually see updated information. @@ -571,20 +450,6 @@ void Balancer::_consumeActionStreamLoop() { executor::ScopedTaskExecutor executor( Grid::get(opCtx.get())->getExecutorPool()->getFixedExecutor()); - ScopeGuard onExitCleanup([this, &executor] { - _defragmentationPolicy->interruptAllDefragmentations(); - _clusterChunksResizePolicy->stop(); - // Explicitly cancel and drain any outstanding streaming action already dispatched to the - // task executor. - executor->shutdown(); - executor->join(); - // When shutting down, the task executor may or may not invoke the - // _applyDefragmentationActionResponseToPolicy() callback for canceled streaming actions: to - // ensure a consistent state of the balancer after a step down, _outstandingStreamingOps - // needs then to be reset to 0 once all the tasks have been drained. - _outstandingStreamingOps.store(0); - }); - auto selectStream = [&]() -> ActionsStreamPolicy* { // This policy has higher priority - and once activated, it cannot be disabled through cfg // changes. @@ -598,6 +463,15 @@ void Balancer::_consumeActionStreamLoop() { return nullptr; }; + auto applyActionResponseTo = [this](const DefragmentationAction& action, + const DefragmentationActionResponse& response, + ActionsStreamPolicy* policy) { + invariant(_outstandingStreamingOps.addAndFetch(-1) >= 0); + ThreadClient tc("BalancerSecondaryThread::applyActionResponse", getGlobalServiceContext()); + auto opCtx = tc->makeOperationContext(); + policy->applyActionResult(opCtx.get(), action, response); + }; + auto applyThrottling = [lastActionTime = Date_t::fromMillisSinceEpoch(0)]() mutable { const Milliseconds throttle{chunkDefragmentationThrottlingMS.load()}; auto timeSinceLastAction = Date_t::now() - lastActionTime; @@ -641,23 +515,9 @@ void Balancer::_consumeActionStreamLoop() { "selectedStream"_attr = selectedStream->getName()); } - boost::optional<DefragmentationAction> nextAction; - try { - _newInfoOnStreamingActions.store(false); - nextAction = selectedStream->getNextStreamingAction(opCtx.get()); - } catch (const DBException& e) { - LOGV2_WARNING(7435001, - "Failed to get next action from action stream", - "error"_attr = redact(e), - "stream"_attr = selectedStream->getName()); - - _newInfoOnStreamingActions.store(true); - continue; - } - - if (!nextAction.is_initialized()) { - // No action was returned by this stream. This means that the stream is drained. - streamDrained = true; + _newInfoOnStreamingActions.store(false); + auto nextAction = selectedStream->getNextStreamingAction(opCtx.get()); + if ((streamDrained = !nextAction.is_initialized())) { continue; } @@ -674,10 +534,11 @@ void Balancer::_consumeActionStreamLoop() { mergeAction.chunkRange, mergeAction.collectionVersion) .thenRunOn(*executor) - .onCompletion([this, selectedStream, action = std::move(mergeAction)]( - const Status& status) { - _applyDefragmentationActionResponseToPolicy( - action, status, selectedStream); + .onCompletion([this, + selectedStream, + &applyActionResponseTo, + action = std::move(mergeAction)](const Status& status) { + applyActionResponseTo(action, status, selectedStream); }); }, [&, selectedStream](DataSizeInfo&& dataSizeAction) { @@ -689,15 +550,15 @@ void Balancer::_consumeActionStreamLoop() { dataSizeAction.chunkRange, dataSizeAction.version, dataSizeAction.keyPattern, - dataSizeAction.estimatedValue, - dataSizeAction.maxSize) + dataSizeAction.estimatedValue) .thenRunOn(*executor) - .onCompletion( - [this, selectedStream, action = std::move(dataSizeAction)]( - const StatusWith<DataSizeResponse>& swDataSize) { - _applyDefragmentationActionResponseToPolicy( - action, swDataSize, selectedStream); - }); + .onCompletion([this, + selectedStream, + &applyActionResponseTo, + action = std::move(dataSizeAction)]( + const StatusWith<DataSizeResponse>& swDataSize) { + applyActionResponseTo(action, swDataSize, selectedStream); + }); }, [&, selectedStream](AutoSplitVectorInfo&& splitVectorAction) { auto result = @@ -711,10 +572,12 @@ void Balancer::_consumeActionStreamLoop() { splitVectorAction.maxChunkSizeBytes) .thenRunOn(*executor) .onCompletion( - [this, selectedStream, action = std::move(splitVectorAction)]( + [this, + selectedStream, + &applyActionResponseTo, + action = std::move(splitVectorAction)]( const StatusWith<AutoSplitVectorResponse>& swSplitPoints) { - _applyDefragmentationActionResponseToPolicy( - action, swSplitPoints, selectedStream); + applyActionResponseTo(action, swSplitPoints, selectedStream); }); }, [&, selectedStream](SplitInfoWithKeyPattern&& splitAction) { @@ -730,10 +593,11 @@ void Balancer::_consumeActionStreamLoop() { splitAction.info.maxKey, splitAction.info.splitKeys) .thenRunOn(*executor) - .onCompletion([this, selectedStream, action = std::move(splitAction)]( - const Status& status) { - _applyDefragmentationActionResponseToPolicy( - action, status, selectedStream); + .onCompletion([this, + selectedStream, + &applyActionResponseTo, + action = std::move(splitAction)](const Status& status) { + applyActionResponseTo(action, status, selectedStream); }); }, [](MigrateInfo&& _) { @@ -746,12 +610,12 @@ void Balancer::_consumeActionStreamLoop() { void Balancer::_mainThread() { ON_BLOCK_EXIT([this] { - { - stdx::lock_guard<Latch> scopedLock(_mutex); - _state = kStopped; - LOGV2_DEBUG(21855, 1, "Balancer thread terminated"); - } + stdx::lock_guard<Latch> scopedLock(_mutex); + + _state = kStopped; _joinCond.notify_all(); + + LOGV2_DEBUG(21855, 1, "Balancer thread terminated"); }); Client::initThread("Balancer"); @@ -798,8 +662,6 @@ void Balancer::_mainThread() { LOGV2(6036606, "Balancer worker thread initialised. Entering main loop."); // Main balancer loop - auto lastMigrationTime = Date_t::fromMillisSinceEpoch(0); - BalancerWarning balancerWarning; while (!_stopRequested()) { BalanceRoundDetails roundDetails; @@ -820,9 +682,6 @@ void Balancer::_mainThread() { continue; } - // Warn before we skip the iteration due to balancing being disabled. - balancerWarning.warnIfRequired(opCtx.get(), balancerConfig->getBalancerMode()); - if (!balancerConfig->shouldBalance() || _stopRequested() || _clusterChunksResizePolicy->isActive()) { LOGV2_DEBUG(21859, 1, "Skipping balancing round because balancing is disabled"); @@ -830,31 +689,33 @@ void Balancer::_mainThread() { continue; } - boost::optional<Milliseconds> forcedBalancerRoundInterval(boost::none); - overrideBalanceRoundInterval.execute([&](const BSONObj& data) { - forcedBalancerRoundInterval = Milliseconds(data["intervalMs"].numberInt()); - LOGV2(21864, - "overrideBalanceRoundInterval: using customized balancing interval", - "balancerInterval"_attr = *forcedBalancerRoundInterval); - }); - // The current configuration is allowing the balancer to perform operations. // Unblock the secondary thread if needed. _defragmentationCondVar.notify_all(); + { + LOGV2_DEBUG(21860, + 1, + "Start balancing round. waitForDelete: {waitForDelete}, " + "secondaryThrottle: {secondaryThrottle}", + "Start balancing round", + "waitForDelete"_attr = balancerConfig->waitForDelete(), + "secondaryThrottle"_attr = + balancerConfig->getSecondaryThrottle().toBSON()); + + static Occasionally sampler; + if (sampler.tick()) { + warnOnMultiVersion(uassertStatusOK(_clusterStats->getStats(opCtx.get()))); + } - LOGV2_DEBUG(21860, - 1, - "Start balancing round. waitForDelete: {waitForDelete}, " - "secondaryThrottle: {secondaryThrottle}", - "Start balancing round", - "waitForDelete"_attr = balancerConfig->waitForDelete(), - "secondaryThrottle"_attr = balancerConfig->getSecondaryThrottle().toBSON()); - - // Collect and apply up-to-date configuration values on the cluster collections. - _defragmentationPolicy->startCollectionDefragmentations(opCtx.get()); + // Collect and apply up-to-date configuration values on the cluster collections. + { + OperationContext* ctx = opCtx.get(); + auto allCollections = Grid::get(ctx)->catalogClient()->getCollections(ctx, {}); + for (const auto& coll : allCollections) { + _defragmentationPolicy->startCollectionDefragmentation(ctx, coll); + } + } - // Split chunk to match zones boundaries - { Status status = _splitChunksIfNeeded(opCtx.get()); if (!status.isOK()) { LOGV2_WARNING(21878, @@ -864,83 +725,47 @@ void Balancer::_mainThread() { } else { LOGV2_DEBUG(21861, 1, "Done enforcing tag range boundaries."); } - } - - // Select and migrate chunks - { - Timer selectionTimer; - const std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats->getStats(opCtx.get())); - - stdx::unordered_set<ShardId> availableShards; - std::transform( - shardStats.begin(), - shardStats.end(), - std::inserter(availableShards, availableShards.end()), - [](const ClusterStatistics::ShardStatistics& shardStatistics) -> ShardId { - return shardStatistics.shardId; - }); + stdx::unordered_set<ShardId> usedShards; const auto chunksToDefragment = - _defragmentationPolicy->selectChunksToMove(opCtx.get(), &availableShards); + _defragmentationPolicy->selectChunksToMove(opCtx.get(), &usedShards); const auto chunksToRebalance = uassertStatusOK( - _chunkSelectionPolicy->selectChunksToMove(opCtx.get(), - shardStats, - &availableShards, - _imbalancedCollectionsCache.get())); - const Milliseconds selectionTimeMillis{selectionTimer.millis()}; + _chunkSelectionPolicy->selectChunksToMove(opCtx.get(), &usedShards)); if (chunksToRebalance.empty() && chunksToDefragment.empty()) { LOGV2_DEBUG(21862, 1, "No need to move any chunk"); _balancedLastTime = 0; - LOGV2_DEBUG(21863, 1, "End balancing round"); - _endRound(opCtx.get(), - forcedBalancerRoundInterval ? *forcedBalancerRoundInterval - : kBalanceRoundDefaultInterval); } else { - - // Sleep according to the migration throttling settings - const auto throttleTimeMillis = [&] { - const auto& minRoundinterval = forcedBalancerRoundInterval - ? *forcedBalancerRoundInterval - : Milliseconds(balancerMigrationsThrottlingMs.load()); - - const auto timeSinceLastMigration = Date_t::now() - lastMigrationTime; - if (timeSinceLastMigration < minRoundinterval) { - return minRoundinterval - timeSinceLastMigration; - } - return Milliseconds::zero(); - }(); - _sleepFor(opCtx.get(), throttleTimeMillis); - - // Migrate chunks - Timer migrationTimer; _balancedLastTime = _moveChunks(opCtx.get(), chunksToRebalance, chunksToDefragment); - lastMigrationTime = Date_t::now(); - const Milliseconds migrationTimeMillis{migrationTimer.millis()}; - // Complete round roundDetails.setSucceeded( static_cast<int>(chunksToRebalance.size() + chunksToDefragment.size()), - _balancedLastTime, - _imbalancedCollectionsCache->size(), - selectionTimeMillis, - throttleTimeMillis, - migrationTimeMillis); + _balancedLastTime); ShardingLogging::get(opCtx.get()) ->logAction(opCtx.get(), "balancer.round", "", roundDetails.toBSON()) .ignore(); - - LOGV2_DEBUG(6679500, 1, "End balancing round"); - // Migration throttling of `balancerMigrationsThrottlingMs` will be applied - // before the next call to _moveChunks, so don't sleep here. - _endRound(opCtx.get(), Milliseconds(0)); } + + LOGV2_DEBUG(21863, 1, "End balancing round"); } + + Milliseconds balancerInterval = + _balancedLastTime ? kShortBalanceRoundInterval : kBalanceRoundDefaultInterval; + + overrideBalanceRoundInterval.execute([&](const BSONObj& data) { + balancerInterval = Milliseconds(data["intervalMs"].numberInt()); + LOGV2(21864, + "overrideBalanceRoundInterval: using shorter balancing interval: " + "{balancerInterval}", + "overrideBalanceRoundInterval: using shorter balancing interval", + "balancerInterval"_attr = balancerInterval); + }); + + _endRound(opCtx.get(), balancerInterval); } catch (const DBException& e) { LOGV2(21865, "caught exception while doing balance: {error}", @@ -980,21 +805,6 @@ void Balancer::_mainThread() { LOGV2(21867, "CSRS balancer is now stopped"); } -void Balancer::_applyDefragmentationActionResponseToPolicy( - const DefragmentationAction& action, - const DefragmentationActionResponse& response, - ActionsStreamPolicy* policy) { - invariant(_outstandingStreamingOps.addAndFetch(-1) >= 0); - ThreadClient tc("BalancerSecondaryThread::applyActionResponse", getGlobalServiceContext()); - { - stdx::lock_guard<Client> lk(cc()); - cc().setSystemOperationKillableByStepdown(lk); - } - - auto opCtx = tc->makeOperationContext(); - policy->applyActionResult(opCtx.get(), action, response); -}; - bool Balancer::_stopRequested() { stdx::lock_guard<Latch> scopedLock(_mutex); return (_state != kRunning); @@ -1154,17 +964,13 @@ int Balancer::_moveChunks(OperationContext* opCtx, std::vector<std::pair<const MigrateInfo&, SemiFuture<void>>> rebalanceMigrationsAndResponses, defragmentationMigrationsAndResponses; auto requestMigration = [&](const MigrateInfo& migrateInfo) -> SemiFuture<void> { - auto maxChunkSizeBytes = [&]() { - if (migrateInfo.optMaxChunkSizeBytes.has_value()) { - return *migrateInfo.optMaxChunkSizeBytes; - } - - auto coll = Grid::get(opCtx)->catalogClient()->getCollection( - opCtx, migrateInfo.nss, repl::ReadConcernLevel::kMajorityReadConcern); - return coll.getMaxChunkSizeBytes().value_or(balancerConfig->getMaxChunkSizeBytes()); - }(); + auto coll = Grid::get(opCtx)->catalogClient()->getCollection( + opCtx, migrateInfo.nss, repl::ReadConcernLevel::kMajorityReadConcern); + auto maxChunkSizeBytes = + coll.getMaxChunkSizeBytes().value_or(balancerConfig->getMaxChunkSizeBytes()); - if (migrateInfo.maxKey.has_value()) { + if (serverGlobalParams.featureCompatibility.isLessThan( + multiversion::FeatureCompatibilityVersion::kVersion_6_0)) { // TODO SERVER-65322 only use `moveRange` once v6.0 branches out MoveChunkSettings settings(maxChunkSizeBytes, balancerConfig->getSecondaryThrottle(), @@ -1175,24 +981,18 @@ int Balancer::_moveChunks(OperationContext* opCtx, MoveRangeRequestBase requestBase(migrateInfo.to); requestBase.setWaitForDelete(balancerConfig->waitForDelete()); requestBase.setMin(migrateInfo.minKey); - requestBase.setMax(migrateInfo.maxKey); + if (!feature_flags::gNoMoreAutoSplitter.isEnabled( + serverGlobalParams.featureCompatibility)) { + // Issue the equivalent of a `moveChunk` if the auto-splitter is enabled + requestBase.setMax(migrateInfo.maxKey); + } ShardsvrMoveRange shardSvrRequest(migrateInfo.nss); shardSvrRequest.setDbName(NamespaceString::kAdminDb); shardSvrRequest.setMoveRangeRequestBase(requestBase); shardSvrRequest.setMaxChunkSizeBytes(maxChunkSizeBytes); shardSvrRequest.setFromShard(migrateInfo.from); - shardSvrRequest.setEpoch(migrateInfo.version.epoch()); - const auto forceJumbo = [&]() { - if (migrateInfo.forceJumbo == MoveChunkRequest::ForceJumbo::kForceManual) { - return ForceJumbo::kForceManual; - } - if (migrateInfo.forceJumbo == MoveChunkRequest::ForceJumbo::kForceBalancer) { - return ForceJumbo::kForceBalancer; - } - return ForceJumbo::kDoNotForce; - }(); - shardSvrRequest.setForceJumbo(forceJumbo); + shardSvrRequest.setEpoch(coll.getEpoch()); const auto [secondaryThrottle, wc] = getSecondaryThrottleAndWriteConcern(balancerConfig->getSecondaryThrottle()); shardSvrRequest.setSecondaryThrottle(secondaryThrottle); @@ -1236,32 +1036,10 @@ int Balancer::_moveChunks(OperationContext* opCtx, opCtx, migrateInfo.uuid, repl::ReadConcernLevel::kMajorityReadConcern); ShardingCatalogManager::get(opCtx)->splitOrMarkJumbo( - opCtx, collection.getNss(), migrateInfo.minKey, migrateInfo.getMaxChunkSizeBytes()); + opCtx, collection.getNss(), migrateInfo.minKey); continue; } - if (status == ErrorCodes::IndexNotFound && - gFeatureFlagShardKeyIndexOptionalHashedSharding.isEnabled( - serverGlobalParams.featureCompatibility)) { - - const auto cm = uassertStatusOK( - Grid::get(opCtx)->catalogCache()->getCollectionRoutingInfoWithRefresh( - opCtx, migrateInfo.nss)); - - if (cm.getShardKeyPattern().isHashedPattern()) { - LOGV2(78252, - "Turning off balancing for hashed collection because migration failed due to " - "missing shardkey index", - "migrateInfo"_attr = redact(migrateInfo.toString()), - "error"_attr = redact(status), - "collection"_attr = migrateInfo.nss); - - // Write to config.collections to turn off the balancer. - _disableBalancer(opCtx, migrateInfo.nss); - continue; - } - } - LOGV2(21872, "Migration {migrateInfo} failed with {error}", "Migration failed", @@ -1280,31 +1058,6 @@ int Balancer::_moveChunks(OperationContext* opCtx, return numChunksProcessed; } -void Balancer::_disableBalancer(OperationContext* opCtx, NamespaceString nss) { - const auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - - BatchedCommandRequest updateRequest([&]() { - write_ops::UpdateCommandRequest updateOp(CollectionType::ConfigNS); - updateOp.setUpdates({[&] { - write_ops::UpdateOpEntry entry; - entry.setQ(BSON(CollectionType::kNssFieldName << nss.ns())); - entry.setU(write_ops::UpdateModification::parseFromClassicUpdate( - BSON("$set" << BSON("noBalance" << true)))); - entry.setMulti(false); - entry.setUpsert(false); - return entry; - }()}); - return updateOp; - }()); - - auto response = configShard->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - updateRequest, - ShardingCatalogClient::kMajorityWriteConcern, - Shard::RetryPolicy::kIdempotent); - uassertStatusOK(response.toStatus()); -} - void Balancer::_onActionsStreamPolicyStateUpdate() { // On any internal update of the defragmentation/cluster chunks resize policy status, // wake up the thread consuming the stream of actions @@ -1322,20 +1075,6 @@ void Balancer::abortCollectionDefragmentation(OperationContext* opCtx, const Nam SharedSemiFuture<void> Balancer::applyLegacyChunkSizeConstraintsOnClusterData( OperationContext* opCtx) { - // Remove the maxChunkSizeBytes from config.system.collections to make it compatible with - // the balancing strategy based on the number of collection chunks - try { - ShardingCatalogManager::get(opCtx)->configureCollectionBalancing( - opCtx, - NamespaceString::kLogicalSessionsNamespace, - 0, - boost::none /*defragmentCollection*/, - boost::none /*enableAutoSplitter*/); - } catch (const ExceptionFor<ErrorCodes::NamespaceNotSharded>&) { - // config.system.collections does not appear in config.collections; continue. - } - - // Ensure now that each collection in the cluster complies with its "maxChunkSize" constraint const auto balancerConfig = Grid::get(opCtx)->getBalancerConfiguration(); uassertStatusOK(balancerConfig->refreshAndCheck(opCtx)); auto futureOutcome = @@ -1356,12 +1095,7 @@ BalancerCollectionStatusResponse Balancer::getBalancerStatusForNs(OperationConte uasserted(ErrorCodes::NamespaceNotSharded, "Collection unsharded or undefined"); } - - const auto maxChunkSizeBytes = getMaxChunkSizeBytes(opCtx, coll); - double maxChunkSizeMB = (double)maxChunkSizeBytes / (1024 * 1024); - // Keep only 2 decimal digits to return a readable value - maxChunkSizeMB = std::ceil(maxChunkSizeMB * 100.0) / 100.0; - + const auto maxChunkSizeMB = getMaxChunkSizeMB(opCtx, coll); BalancerCollectionStatusResponse response(maxChunkSizeMB, true /*balancerCompliant*/); auto setViolationOnResponse = [&response](const StringData& reason, const boost::optional<BSONObj>& details = diff --git a/src/mongo/db/s/balancer/balancer.h b/src/mongo/db/s/balancer/balancer.h index b84df5558bb..be31a053d0a 100644 --- a/src/mongo/db/s/balancer/balancer.h +++ b/src/mongo/db/s/balancer/balancer.h @@ -40,7 +40,6 @@ namespace mongo { -class ActionsStreamPolicy; class ChunkType; class ClusterStatistics; class BalancerCommandsScheduler; @@ -260,20 +259,6 @@ private: void _onActionsStreamPolicyStateUpdate(); - /** - * To be invoked on completion of an action requested to by an ActionStream policy to - * update the policy state (which will generate follow-up actions based on the received - * outcome). - */ - void _applyDefragmentationActionResponseToPolicy(const DefragmentationAction& action, - const DefragmentationActionResponse& response, - ActionsStreamPolicy* policy); - - /** - * Disables the balancer for a collection by setting `noBalance` to true. - */ - void _disableBalancer(OperationContext* opCtx, NamespaceString nss); - // Protects the state below Mutex _mutex = MONGO_MAKE_LATCH("Balancer::_mutex"); @@ -326,8 +311,6 @@ private: // TODO SERVER-65332 remove logic bound to this policy object When kLastLTS is 6.0 std::unique_ptr<ClusterChunksResizePolicy> _clusterChunksResizePolicy; - - std::unique_ptr<stdx::unordered_set<NamespaceString>> _imbalancedCollectionsCache; }; } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h b/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h index eb10bb6fd88..5aed229c202 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy.h @@ -74,11 +74,7 @@ public: * Potentially blocking method, which gives out a set of chunks to be moved. */ virtual StatusWith<MigrateInfoVector> selectChunksToMove( - OperationContext* opCtx, - const std::vector<ClusterStatistics::ShardStatistics>& shardStats, - stdx::unordered_set<ShardId>* availableShards, - stdx::unordered_set<NamespaceString>* imbalancedCollectionsCachePtr) = 0; - + OperationContext* opCtx, stdx::unordered_set<ShardId>* unavailableShards) = 0; /** * Given a valid namespace returns all the Migrations the balancer would need to perform diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp index 8deef770d27..c9a41e33c09 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.cpp @@ -40,7 +40,6 @@ #include "mongo/base/status_with.h" #include "mongo/bson/bsonobj_comparator_interface.h" #include "mongo/db/s/sharding_config_server_parameters_gen.h" -#include "mongo/db/s/sharding_util.h" #include "mongo/logv2/log.h" #include "mongo/platform/bits.h" #include "mongo/s/balancer_configuration.h" @@ -48,14 +47,8 @@ #include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/s/catalog_cache.h" -#include "mongo/s/chunk_manager.h" #include "mongo/s/grid.h" -#include "mongo/s/request_types/get_stats_for_balancing_gen.h" -#include "mongo/s/sharding_feature_flags_gen.h" #include "mongo/util/str.h" -#include "mongo/util/timer.h" - -MONGO_FAIL_POINT_DEFINE(overrideStatsForBalancingBatchSize); namespace mongo { @@ -70,92 +63,39 @@ StatusWith<DistributionStatus> createCollectionDistributionStatus( const NamespaceString& nss, const ShardStatisticsVector& allShards, const ChunkManager& chunkMgr) { + ShardToChunksMap shardToChunksMap; - auto swZoneInfo = - createCollectionZoneInfo(opCtx, nss, chunkMgr.getShardKeyPattern().getKeyPattern()); - if (!swZoneInfo.isOK()) { - return swZoneInfo.getStatus(); + // Makes sure there is an entry in shardToChunksMap for every shard, so empty shards will also + // be accounted for + for (const auto& stat : allShards) { + shardToChunksMap[stat.shardId]; } - return {DistributionStatus{nss, std::move(swZoneInfo.getValue()), chunkMgr}}; -} + chunkMgr.forEachChunk([&](const auto& chunkEntry) { + ChunkType chunk; + chunk.setCollectionUUID(chunkMgr.getUUID()); + chunk.setMin(chunkEntry.getMin()); + chunk.setMax(chunkEntry.getMax()); + chunk.setJumbo(chunkEntry.isJumbo()); + chunk.setShard(chunkEntry.getShardId()); + chunk.setVersion(chunkEntry.getLastmod()); -stdx::unordered_map<NamespaceString, CollectionDataSizeInfoForBalancing> -getDataSizeInfoForCollections(OperationContext* opCtx, - const std::vector<CollectionType>& collections) { - const auto balancerConfig = Grid::get(opCtx)->getBalancerConfiguration(); - uassertStatusOK(balancerConfig->refreshAndCheck(opCtx)); + shardToChunksMap[chunkEntry.getShardId()].push_back(chunk); - const auto shardRegistry = Grid::get(opCtx)->shardRegistry(); - const auto shardIds = shardRegistry->getAllShardIds(opCtx); + return true; + }); - // Map to be returned, incrementally populated with the collected statistics - stdx::unordered_map<NamespaceString, CollectionDataSizeInfoForBalancing> dataSizeInfoMap; + DistributionStatus distribution(nss, std::move(shardToChunksMap)); - std::vector<NamespaceWithOptionalUUID> namespacesWithUUIDsForStatsRequest; - for (const auto& coll : collections) { - const auto& nss = coll.getNss(); - const auto maxChunkSizeBytes = - coll.getMaxChunkSizeBytes().value_or(balancerConfig->getMaxChunkSizeBytes()); - - dataSizeInfoMap.emplace( - nss, - CollectionDataSizeInfoForBalancing(std::map<ShardId, int64_t>(), maxChunkSizeBytes)); - - NamespaceWithOptionalUUID nssWithUUID(nss); - nssWithUUID.setUUID(coll.getUuid()); - namespacesWithUUIDsForStatsRequest.push_back(nssWithUUID); - } - - ShardsvrGetStatsForBalancing req{namespacesWithUUIDsForStatsRequest}; - req.setScaleFactor(1); - const auto reqObj = req.toBSON({}); - - const auto executor = Grid::get(opCtx)->getExecutorPool()->getFixedExecutor(); - const auto responsesFromShards = - sharding_util::sendCommandToShards(opCtx, - NamespaceString::kAdminDb.toString(), - reqObj, - shardIds, - executor, - false /* throwOnError */); - - for (auto&& response : responsesFromShards) { - try { - const auto& shardId = response.shardId; - const auto errorContext = - "Failed to get stats for balancing from shard '{}'"_format(shardId.toString()); - const auto responseValue = - uassertStatusOKWithContext(std::move(response.swResponse), errorContext); - - const ShardsvrGetStatsForBalancingReply reply = - ShardsvrGetStatsForBalancingReply::parse( - IDLParserErrorContext("ShardsvrGetStatsForBalancingReply"), - std::move(responseValue.data)); - const auto collStatsFromShard = reply.getStats(); - - invariant(collStatsFromShard.size() == collections.size()); - for (const auto& stats : collStatsFromShard) { - invariant(dataSizeInfoMap.contains(stats.getNs())); - dataSizeInfoMap.at(stats.getNs()).shardToDataSizeMap[shardId] = stats.getCollSize(); - } - } catch (const ExceptionFor<ErrorCodes::ShardNotFound>& ex) { - // Handle `removeShard`: skip shards removed during a balancing round - LOGV2_DEBUG(6581603, - 1, - "Skipping shard for the current balancing round", - "error"_attr = redact(ex)); - } - } + const auto& keyPattern = chunkMgr.getShardKeyPattern().getKeyPattern(); - return dataSizeInfoMap; -} + // Cache the collection tags + auto status = ZoneInfo::addTagsFromCatalog(opCtx, nss, keyPattern, distribution.zoneInfo()); + if (!status.isOK()) { + return status; + } -const CollectionDataSizeInfoForBalancing getDataSizeInfoForCollection(OperationContext* opCtx, - const NamespaceString& nss) { - const auto coll = Grid::get(opCtx)->catalogClient()->getCollection(opCtx, nss); - std::vector<CollectionType> vec{coll}; - return std::move(getDataSizeInfoForCollections(opCtx, vec).at(nss)); + return {std::move(distribution)}; } /** @@ -225,12 +165,12 @@ private: * range boundaries. */ void getSplitCandidatesToEnforceTagRanges(const ChunkManager& cm, - const ZoneInfo& zoneInfo, + const DistributionStatus& distribution, SplitCandidatesBuffer* splitCandidates) { const auto& globalMax = cm.getShardKeyPattern().getKeyPattern().globalMax(); // For each tag range, find chunks that need to be split. - for (const auto& tagRangeEntry : zoneInfo.zoneRanges()) { + for (const auto& tagRangeEntry : distribution.tagRanges()) { const auto& tagRange = tagRangeEntry.second; const auto chunkAtZoneMin = cm.findIntersectingChunkWithSimpleCollation(tagRange.min); @@ -382,185 +322,60 @@ StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToSpli } StatusWith<MigrateInfoVector> BalancerChunkSelectionPolicyImpl::selectChunksToMove( - OperationContext* opCtx, - const std::vector<ClusterStatistics::ShardStatistics>& shardStats, - stdx::unordered_set<ShardId>* availableShards, - stdx::unordered_set<NamespaceString>* imbalancedCollectionsCachePtr) { - invariant(availableShards); - invariant(imbalancedCollectionsCachePtr); - - if (availableShards->size() < 2) { - return MigrateInfoVector{}; + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) { + auto shardStatsStatus = _clusterStats->getStats(opCtx); + if (!shardStatsStatus.isOK()) { + return shardStatsStatus.getStatus(); } - Timer chunksSelectionTimer; + const auto& shardStats = shardStatsStatus.getValue(); - auto collections = Grid::get(opCtx)->catalogClient()->getCollections( - opCtx, - {}, - repl::ReadConcernLevel::kMajorityReadConcern, - BSON(CollectionType::kNssFieldName << 1)); + if (shardStats.size() < 2) { + return MigrateInfoVector{}; + } + auto collections = Grid::get(opCtx)->catalogClient()->getCollections(opCtx, {}); if (collections.empty()) { return MigrateInfoVector{}; } MigrateInfoVector candidateChunks; - const uint32_t kStatsForBalancingBatchSize = [&]() { - auto batchSize = 100U; - overrideStatsForBalancingBatchSize.execute([&batchSize](const BSONObj& data) { - batchSize = data["size"].numberInt(); - LOGV2(7617200, "Overriding collections batch size", "size"_attr = batchSize); - }); - return batchSize; - }(); - - const uint32_t kMaxCachedCollectionsSize = 0.75 * kStatsForBalancingBatchSize; - - // Lambda function used to get a CollectionType leveraging the `collections` vector - // The `collections` vector must be sorted by nss when it is called - auto getCollectionTypeByNss = [&collections](const NamespaceString& nss) - -> std::pair<boost::optional<CollectionType>, std::vector<CollectionType>::iterator> { - // Using a lower_bound to perform a binary search on the `collections` vector - const auto collIt = - std::lower_bound(collections.begin(), - collections.end(), - nss, - [](const CollectionType& coll, const NamespaceString& ns) { - return coll.getNss() < ns; - }); - - if (collIt == collections.end() || collIt->getNss() != nss) { - return std::make_pair(boost::none, collections.end()); - } - return std::make_pair(*collIt, collIt); - }; + std::shuffle(collections.begin(), collections.end(), _random); + + for (const auto& coll : collections) { + const NamespaceString& nss(coll.getNss()); - // Lambda function to check if a collection is explicitly disabled for balancing - const auto canBalanceCollection = [](const CollectionType& coll) -> bool { if (!coll.getAllowBalance() || !coll.getAllowMigrations() || !coll.getPermitMigrations() || coll.getDefragmentCollection()) { LOGV2_DEBUG(5966401, 1, "Not balancing explicitly disabled collection", - "namespace"_attr = coll.getNss(), + "namespace"_attr = nss, "allowBalance"_attr = coll.getAllowBalance(), "allowMigrations"_attr = coll.getAllowMigrations(), "permitMigrations"_attr = coll.getPermitMigrations(), "defragmentCollection"_attr = coll.getDefragmentCollection()); - return false; - } - return true; - }; - - // Lambda function to select migrate candidates from a batch of collections - const auto processBatch = [&](std::vector<CollectionType>& collBatch) { - boost::optional<stdx::unordered_map<NamespaceString, CollectionDataSizeInfoForBalancing>> - collsDataSizeInfo; - if (feature_flags::gBalanceAccordingToDataSize.isEnabled( - serverGlobalParams.featureCompatibility)) { - collsDataSizeInfo.emplace(getDataSizeInfoForCollections(opCtx, collBatch)); - } - - std::shuffle(collBatch.begin(), collBatch.end(), _random); - for (const auto& coll : collBatch) { - - if (availableShards->size() < 2) { - break; - } - - const auto& nss = coll.getNss(); - - boost::optional<CollectionDataSizeInfoForBalancing> optDataSizeInfo; - if (collsDataSizeInfo.has_value()) { - optDataSizeInfo.emplace(std::move(collsDataSizeInfo->at(nss))); - } - - auto swMigrateCandidates = _getMigrateCandidatesForCollection( - opCtx, nss, shardStats, optDataSizeInfo, availableShards); - if (swMigrateCandidates == ErrorCodes::NamespaceNotFound) { - // Namespace got dropped before we managed to get to it, so just skip it - imbalancedCollectionsCachePtr->erase(nss); - continue; - } else if (!swMigrateCandidates.isOK()) { - LOGV2_WARNING(21853, - "Unable to balance collection", - "namespace"_attr = nss.ns(), - "error"_attr = swMigrateCandidates.getStatus()); - continue; - } - - candidateChunks.insert( - candidateChunks.end(), - std::make_move_iterator(swMigrateCandidates.getValue().first.begin()), - std::make_move_iterator(swMigrateCandidates.getValue().first.end())); - - const auto& migrateCandidates = swMigrateCandidates.getValue().first; - if (migrateCandidates.empty()) { - imbalancedCollectionsCachePtr->erase(nss); - } else if (imbalancedCollectionsCachePtr->size() < kMaxCachedCollectionsSize) { - imbalancedCollectionsCachePtr->insert(nss); - } - } - }; - - // To assess if a collection has chunks to migrate, we need to ask shards the size of that - // collection. For efficiency, we ask for a batch of collections per every shard request instead - // of a single request per collection - std::vector<CollectionType> collBatch; - - // The first batch is partially filled by the imbalanced cached collections - for (auto imbalancedNssIt = imbalancedCollectionsCachePtr->begin(); - imbalancedNssIt != imbalancedCollectionsCachePtr->end();) { - - const auto& [imbalancedColl, collIt] = getCollectionTypeByNss(*imbalancedNssIt); - - if (!imbalancedColl.has_value() || !canBalanceCollection(imbalancedColl.value())) { - // The collection was dropped or is no longer enabled for balancing. - imbalancedCollectionsCachePtr->erase(imbalancedNssIt++); continue; } - collBatch.push_back(imbalancedColl.value()); - ++imbalancedNssIt; - - // Remove the collection from the whole list of collections to avoid processing it twice - collections.erase(collIt); - } - - // Iterate all the remaining collections randomly - std::shuffle(collections.begin(), collections.end(), _random); - for (const auto& coll : collections) { - - if (canBalanceCollection(coll)) { - collBatch.push_back(coll); - } - - if (collBatch.size() == kStatsForBalancingBatchSize) { - processBatch(collBatch); - if (availableShards->size() < 2) { - return candidateChunks; - } - collBatch.clear(); - } - - const auto maxTimeMs = balancerChunksSelectionTimeoutMs.load(); - if (candidateChunks.size() > 0 && chunksSelectionTimer.millis() > maxTimeMs) { - LOGV2_DEBUG( - 7100900, - 1, - "Exceeded max time while searching for candidate chunks to migrate in this round.", - "maxTime"_attr = Milliseconds(maxTimeMs), - "chunksSelectionTime"_attr = chunksSelectionTimer.elapsed(), - "numCandidateChunks"_attr = candidateChunks.size()); - - return candidateChunks; + auto candidatesStatus = + _getMigrateCandidatesForCollection(opCtx, nss, shardStats, usedShards); + if (candidatesStatus == ErrorCodes::NamespaceNotFound) { + // Namespace got dropped before we managed to get to it, so just skip it + continue; + } else if (!candidatesStatus.isOK()) { + LOGV2_WARNING(21853, + "Unable to balance collection {namespace}: {error}", + "Unable to balance collection", + "namespace"_attr = nss.ns(), + "error"_attr = candidatesStatus.getStatus()); + continue; } - } - if (collBatch.size() > 0) { - processBatch(collBatch); + candidateChunks.insert(candidateChunks.end(), + std::make_move_iterator(candidatesStatus.getValue().first.begin()), + std::make_move_iterator(candidatesStatus.getValue().first.end())); } return candidateChunks; @@ -579,23 +394,9 @@ StatusWith<MigrateInfosWithReason> BalancerChunkSelectionPolicyImpl::selectChunk // doesn't. Grid::get(opCtx)->catalogClient()->getCollection(opCtx, nss); - stdx::unordered_set<ShardId> availableShards; - std::transform(shardStats.begin(), - shardStats.end(), - std::inserter(availableShards, availableShards.end()), - [](const ClusterStatistics::ShardStatistics& shardStatistics) -> ShardId { - return shardStatistics.shardId; - }); + stdx::unordered_set<ShardId> usedShards; - - boost::optional<CollectionDataSizeInfoForBalancing> optCollDataSizeInfo; - if (feature_flags::gBalanceAccordingToDataSize.isEnabled( - serverGlobalParams.featureCompatibility)) { - optCollDataSizeInfo.emplace(getDataSizeInfoForCollection(opCtx, nss)); - } - - auto candidatesStatus = _getMigrateCandidatesForCollection( - opCtx, nss, shardStats, optCollDataSizeInfo, &availableShards); + auto candidatesStatus = _getMigrateCandidatesForCollection(opCtx, nss, shardStats, &usedShards); if (!candidatesStatus.isOK()) { return candidatesStatus.getStatus(); } @@ -675,7 +476,7 @@ Status BalancerChunkSelectionPolicyImpl::checkMoveAllowed(OperationContext* opCt } return BalancerPolicy::isShardSuitableReceiver(*newShardIterator, - distribution.getTagForRange(chunk.getRange())); + distribution.getTagForChunk(chunk)); } StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::_getSplitCandidatesForCollection( @@ -688,26 +489,26 @@ StatusWith<SplitInfoVector> BalancerChunkSelectionPolicyImpl::_getSplitCandidate const auto& cm = routingInfoStatus.getValue(); - const auto swCollZoneInfo = - createCollectionZoneInfo(opCtx, nss, cm.getShardKeyPattern().getKeyPattern()); - if (!swCollZoneInfo.isOK()) { - return swCollZoneInfo.getStatus(); + const auto collInfoStatus = createCollectionDistributionStatus(opCtx, nss, shardStats, cm); + if (!collInfoStatus.isOK()) { + return collInfoStatus.getStatus(); } - const auto& collZoneInfo = swCollZoneInfo.getValue(); + + const DistributionStatus& distribution = collInfoStatus.getValue(); // Accumulate split points for the same chunk together SplitCandidatesBuffer splitCandidates(nss, cm.getVersion()); if (nss == NamespaceString::kLogicalSessionsNamespace) { - if (!collZoneInfo.allZones().empty()) { + if (!distribution.tags().empty()) { LOGV2_WARNING(4562401, "Ignoring zones for the sessions collection", - "tags"_attr = collZoneInfo.allZones()); + "tags"_attr = distribution.tags()); } getSplitCandidatesForSessionsCollection(opCtx, cm, &splitCandidates); } else { - getSplitCandidatesToEnforceTagRanges(cm, collZoneInfo, &splitCandidates); + getSplitCandidatesToEnforceTagRanges(cm, distribution, &splitCandidates); } return splitCandidates.done(); @@ -718,8 +519,7 @@ BalancerChunkSelectionPolicyImpl::_getMigrateCandidatesForCollection( OperationContext* opCtx, const NamespaceString& nss, const ShardStatisticsVector& shardStats, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - stdx::unordered_set<ShardId>* availableShards) { + stdx::unordered_set<ShardId>* usedShards) { auto routingInfoStatus = Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(opCtx, nss); if (!routingInfoStatus.isOK()) { @@ -737,7 +537,7 @@ BalancerChunkSelectionPolicyImpl::_getMigrateCandidatesForCollection( const DistributionStatus& distribution = collInfoStatus.getValue(); - for (const auto& tagRangeEntry : distribution.getZoneInfo().zoneRanges()) { + for (const auto& tagRangeEntry : distribution.tagRanges()) { const auto& tagRange = tagRangeEntry.second; const auto chunkAtZoneMin = cm.findIntersectingChunkWithSimpleCollation(tagRange.min); @@ -775,8 +575,7 @@ BalancerChunkSelectionPolicyImpl::_getMigrateCandidatesForCollection( return BalancerPolicy::balance( shardStats, distribution, - collDataSizeInfo, - availableShards, + usedShards, Grid::get(opCtx)->getBalancerConfiguration()->attemptToBalanceJumboChunks()); } diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h index 85cdbaaf509..76febe0557c 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h @@ -47,10 +47,7 @@ public: const NamespaceString& ns) override; StatusWith<MigrateInfoVector> selectChunksToMove( - OperationContext* opCtx, - const std::vector<ClusterStatistics::ShardStatistics>& shardStats, - stdx::unordered_set<ShardId>* availableShards, - stdx::unordered_set<NamespaceString>* imbalancedCollectionsCachePtr) override; + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) override; StatusWith<MigrateInfosWithReason> selectChunksToMove(OperationContext* opCtx, const NamespaceString& ns) override; @@ -80,8 +77,7 @@ private: OperationContext* opCtx, const NamespaceString& nss, const ShardStatisticsVector& shardStats, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - stdx::unordered_set<ShardId>* availableShards); + stdx::unordered_set<ShardId>* usedShards); // Source for obtaining cluster statistics. Not owned and must not be destroyed before the // policy object is destroyed. diff --git a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_test.cpp b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_test.cpp index 6152927cfe9..a06171ed7da 100644 --- a/src/mongo/db/s/balancer/balancer_chunk_selection_policy_test.cpp +++ b/src/mongo/db/s/balancer/balancer_chunk_selection_policy_test.cpp @@ -33,11 +33,7 @@ #include "mongo/db/s/balancer/balancer_chunk_selection_policy_impl.h" #include "mongo/db/s/balancer/cluster_statistics_impl.h" #include "mongo/db/s/balancer/migration_test_fixture.h" -#include "mongo/idl/server_parameter_test_util.h" -#include "mongo/logv2/log.h" #include "mongo/platform/random.h" -#include "mongo/s/balancer_configuration.h" -#include "mongo/s/request_types/get_stats_for_balancing_gen.h" #include "mongo/s/type_collection_common_types_gen.h" namespace mongo { @@ -58,24 +54,6 @@ protected: std::make_unique<BalancerChunkSelectionPolicyImpl>(_clusterStats.get(), _random)) {} /** - * Generates a default chunks distribution across shards with the form: - * [MinKey, 0), [0, 1), [1, 2) ... [N - 2, MaxKey) - */ - std::map<ShardId, std::vector<ChunkRange>> generateDefaultChunkRanges( - const std::vector<ShardId>& shards) { - - std::map<ShardId, std::vector<ChunkRange>> chunksPerShard; - for (auto i = 0U; i < shards.size(); ++i) { - const ShardId& shardId = shards[i]; - const auto min = (i == 0 ? kKeyPattern.globalMin() : BSON(kPattern << int(i - 1))); - const auto max = - (i == shards.size() - 1 ? kKeyPattern.globalMax() : BSON(kPattern << int(i))); - chunksPerShard[shardId].push_back(ChunkRange(min, max)); - } - return chunksPerShard; - } - - /** * Sets up mock network to expect a listDatabases command and returns a BSON response with * a dummy sizeOnDisk. */ @@ -97,162 +75,33 @@ protected: } /** - * Sets up mock network for all the shards to expect the commands executed for computing cluster - * stats, which include listDatabase and serverStatus. - */ - void expectGetStatsCommands(int numShards) { - for (int i = 0; i < numShards; i++) { - expectListDatabasesCommand(); - } - } - - /** - * Sets up mock network for all the shards to expect the command `_shardsvrGetStatsForBalancing` - * Given a request sent to a specific shard with below structure ... - * { - * "_shardsvrGetStatsForBalancing" : 1, - * "collections" : [ - * { - * "ns" : "TestDb.TestColl", - * "UUID" : "xxxx" - * }, - * ... - * ] - * } - * - * ... mocks a reply with the following structure: - * { - * "stats" : [ - * { - * "namespace" : "TestDb.TestColl", - * "collSize" : 12345, - * }, - * ... - * ] - * } + * Sets up mock network to expect a serverStatus command and returns a BSON response with + * a dummy version. */ - void expectGetStatsForBalancingCommands(const std::map<ShardId, int64_t>& collSizePerShard) { - const auto& numShards = collSizePerShard.size(); - for (auto i = 0U; i < numShards; ++i) { - BSONObjBuilder resultBuilder; - CommandHelpers::appendCommandStatusNoThrow(resultBuilder, Status::OK()); - - // Build a response for given request - onCommand([&](const RemoteCommandRequest& request) { - ASSERT(request.cmdObj[ShardsvrGetStatsForBalancing::kCommandName]); - - // Get `shardId` - ShardId shardId = getShardIdByHost(request.target); - resultBuilder.append("shardId", shardId); - - // Build `stats` array: [ {"namespace": <nss>, "collSize": <collSize>}, ...] - { - BSONArrayBuilder statsArrayBuilder(resultBuilder.subarrayStart("stats")); - - ASSERT_EQ(1, collSizePerShard.count(shardId)); - const auto& collSize = collSizePerShard.at(shardId); - - for (const auto& reqColl : - request.cmdObj[ShardsvrGetStatsForBalancing::kCollectionsFieldName] - .Array()) { - const auto nss = - NamespaceWithOptionalUUID::parse( - IDLParserErrorContext("BalancerChunkSelectionPolicyTest"), - reqColl.Obj()) - .getNs(); - - statsArrayBuilder.append(CollStatsForBalancing(nss, collSize).toBSON()); - } - } - return resultBuilder.obj(); - }); - } - } + void expectServerStatusCommand() { + BSONObjBuilder resultBuilder; + CommandHelpers::appendCommandStatusNoThrow(resultBuilder, Status::OK()); - /** - * Same as expectGetStatsForBalancingCommands with the difference that this function will expect - * only one migration between the specified shards - */ - void expectGetStatsForBalancingCommandsWithOneMigration(uint32_t numShards, - ShardId donorShardId, - ShardId recipientShardId) { - ASSERT_NE(donorShardId, recipientShardId); - - const auto maxChunkSizeBytes = - Grid::get(operationContext())->getBalancerConfiguration()->getMaxChunkSizeBytes(); - const auto defaultCollSizeOnShard = 2 * maxChunkSizeBytes; - const auto imbalancedCollSizeOnRecipient = maxChunkSizeBytes; - const auto imbalancedCollSizeOnDonor = 5 * maxChunkSizeBytes; - - for (auto i = 0U; i < numShards; ++i) { - BSONObjBuilder resultBuilder; - CommandHelpers::appendCommandStatusNoThrow(resultBuilder, Status::OK()); - - // Build a response for every given request - onCommand([&](const RemoteCommandRequest& request) { - ASSERT(request.cmdObj[ShardsvrGetStatsForBalancing::kCommandName]); - - // Get `shardId` - ShardId shardId = getShardIdByHost(request.target); - resultBuilder.append("shardId", shardId); - - // Build `stats` array: [ {"namespace": <nss>, "collSize": <collSize>}, ...] - { - bool firstColl = true; - BSONArrayBuilder statsArrayBuilder(resultBuilder.subarrayStart("stats")); - for (const auto& reqColl : - request.cmdObj[ShardsvrGetStatsForBalancing::kCollectionsFieldName] - .Array()) { - const auto nss = - NamespaceWithOptionalUUID::parse( - IDLParserErrorContext("BalancerChunkSelectionPolicyTest"), - reqColl.Obj()) - .getNs(); - - const auto collSize = [&]() { - if (firstColl && shardId == donorShardId) { - return imbalancedCollSizeOnDonor; - } else if (firstColl && shardId == recipientShardId) { - return imbalancedCollSizeOnRecipient; - } - return defaultCollSizeOnShard; - }(); - - statsArrayBuilder.append(CollStatsForBalancing(nss, collSize).toBSON()); - firstColl = false; - } - } - return resultBuilder.obj(); - }); - } + onCommand([&resultBuilder](const RemoteCommandRequest& request) { + ASSERT(request.cmdObj["serverStatus"]); + resultBuilder.append("version", "MONGO_VERSION"); + return resultBuilder.obj(); + }); } /** - * Sets up a collection and its chunks according to the given range distribution across - * shards + * Sets up mock network for all the shards to expect the commands executed for computing cluster + * stats, which include listDatabase and serverStatus. */ - UUID setUpCollectionWithChunks( - const NamespaceString& ns, - const std::map<ShardId, std::vector<ChunkRange>>& chunksPerShard) { - const UUID collUuid = UUID::gen(); - ChunkVersion version(2, 0, OID::gen(), Timestamp(42)); - - for (const auto& [shardId, chunkRanges] : chunksPerShard) { - for (const auto& chunkRange : chunkRanges) { - setUpChunk(collUuid, chunkRange.getMin(), chunkRange.getMax(), shardId, version); - version.incMinor(); - } - version.incMajor(); + void expectGetStatsCommands(int numShards) { + for (int i = 0; i < numShards; i++) { + expectListDatabasesCommand(); + expectServerStatusCommand(); } - - setUpCollection(ns, collUuid, version); - - return collUuid; } /** - * Returns a new BSON object with the zone encoded using the legacy field "tags" - * (to mimic the expected schema of config.shards) + * Returns a new BSON object with the tags appended. */ BSONObj appendTags(const BSONObj shardBSON, std::vector<std::string> tags) { BSONObjBuilder appendedShardBSON(shardBSON); @@ -268,21 +117,8 @@ protected: BalancerRandomSource _random; std::unique_ptr<ClusterStatistics> _clusterStats; std::unique_ptr<BalancerChunkSelectionPolicy> _chunkSelectionPolicy; - stdx::unordered_set<NamespaceString> _imbalancedCollectionsCache; }; -stdx::unordered_set<ShardId> getAllShardIds( - const std::vector<ClusterStatistics::ShardStatistics>& shardStats) { - stdx::unordered_set<ShardId> shards; - std::transform(shardStats.begin(), - shardStats.end(), - std::inserter(shards, shards.end()), - [](const ClusterStatistics::ShardStatistics& shardStaticstics) -> ShardId { - return shardStaticstics.shardId; - }); - return shards; -} - TEST_F(BalancerChunkSelectionTest, TagRangesOverlap) { // Set up two shards in the metadata. ASSERT_OK(catalogClient()->insertConfigDocument( @@ -337,8 +173,6 @@ TEST_F(BalancerChunkSelectionTest, TagRangesOverlap) { } TEST_F(BalancerChunkSelectionTest, TagRangeMaxNotAlignedWithChunkMax) { - RAIIServerParameterControllerForTest featureFlagBalanceAccordingToDataSize{ - "featureFlagBalanceAccordingToDataSize", false}; // Set up two shards in the metadata. ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), ShardType::ConfigNS, @@ -375,12 +209,9 @@ TEST_F(BalancerChunkSelectionTest, TagRangeMaxNotAlignedWithChunkMax) { shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - _imbalancedCollectionsCache.clear(); - auto candidateChunksStatus = _chunkSelectionPolicy.get()->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); + stdx::unordered_set<ShardId> usedShards; + auto candidateChunksStatus = + _chunkSelectionPolicy.get()->selectChunksToMove(opCtx.get(), &usedShards); ASSERT_OK(candidateChunksStatus.getStatus()); // The balancer does not bubble up the IllegalOperation error, but it is expected @@ -450,8 +281,6 @@ TEST_F(BalancerChunkSelectionTest, ShardedTimeseriesCollectionsCanBeAutoSplitted } TEST_F(BalancerChunkSelectionTest, ShardedTimeseriesCollectionsCanBeBalanced) { - RAIIServerParameterControllerForTest featureFlagBalanceAccordingToDataSize{ - "featureFlagBalanceAccordingToDataSize", false}; // Set up two shards in the metadata. ASSERT_OK(catalogClient()->insertConfigDocument( operationContext(), ShardType::ConfigNS, kShard0, kMajorityWriteConcern)); @@ -487,13 +316,9 @@ TEST_F(BalancerChunkSelectionTest, ShardedTimeseriesCollectionsCanBeBalanced) { shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - _imbalancedCollectionsCache.clear(); - auto candidateChunksStatus = _chunkSelectionPolicy.get()->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); + stdx::unordered_set<ShardId> usedShards; + auto candidateChunksStatus = + _chunkSelectionPolicy.get()->selectChunksToMove(opCtx.get(), &usedShards); ASSERT_OK(candidateChunksStatus.getStatus()); ASSERT_EQUALS(1, candidateChunksStatus.getValue().size()); @@ -503,375 +328,5 @@ TEST_F(BalancerChunkSelectionTest, ShardedTimeseriesCollectionsCanBeBalanced) { future.default_timed_get(); } -TEST_F(BalancerChunkSelectionTest, AllImbalancedCollectionsShouldEventuallyBeSelectedForBalancing) { - // Set up two shards in the metadata. - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard0, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard1, - kMajorityWriteConcern)); - - // Set up database - setUpDatabase(kDbName, kShardId0); - - // Override collections batch size to 4 for speeding up the test - FailPointEnableBlock overrideBatchSizeGuard("overrideStatsForBalancingBatchSize", - BSON("size" << 4)); - - // Set up 7 imbalanced collections (more than `kStatsForBalancingBatchSize`) - const int numCollections = 7; - const int maxIterations = 1000; - - for (auto i = 0; i < numCollections; ++i) { - const std::string collName = "TestColl" + std::to_string(i); - setUpCollectionWithChunks(NamespaceString(kDbName, collName), - generateDefaultChunkRanges({kShardId0, kShardId1})); - } - - std::set<NamespaceString> collectionsSelected; - _imbalancedCollectionsCache.clear(); - - auto i = 0; - for (; i < maxIterations; ++i) { - - auto future = launchAsync([this, &collectionsSelected]() { - ThreadClient tc(getServiceContext()); - auto opCtx = Client::getCurrent()->makeOperationContext(); - - // Requests chunks to be relocated requires running commands on each shard to - // get shard statistics. Set up dummy hosts for the source shards. - shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - auto chunksToMoveWithStatus = _chunkSelectionPolicy->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); - ASSERT_OK(chunksToMoveWithStatus.getStatus()); - - for (const auto& chunkToMove : chunksToMoveWithStatus.getValue()) { - collectionsSelected.insert(chunkToMove.nss); - } - }); - - expectGetStatsCommands(2 /*numShards*/); - - // Collection size distribution for each collection: - // Shard0 -> 512 MB - // Shard1 -> 0 MB - expectGetStatsForBalancingCommands( - {{kShardId0, 512 * 1024 * 1024 /*Bytes*/}, {kShardId1, 0 /*Bytes*/}}); - - future.default_timed_get(); - - if (collectionsSelected.size() == numCollections) { - break; - } - } - - LOGV2(6867000, - "AllImbalancedCollectionsShouldEventuallyBeSelectedForBalancing test results", - "numCollectionsSelected"_attr = collectionsSelected.size(), - "iterations"_attr = i); - - // Check that all collections were selected for balancing at least once. - ASSERT_EQ(numCollections, collectionsSelected.size()); -} - -TEST_F(BalancerChunkSelectionTest, SelectedCollectionsShouldBeCached) { - // Set up two shards in the metadata. - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard0, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard1, - kMajorityWriteConcern)); - - // Set up database - setUpDatabase(kDbName, kShardId0); - - // Set up 4 collections - const int numCollections = 4; - for (auto i = 0; i < numCollections; ++i) { - const std::string collName = "TestColl" + std::to_string(i); - setUpCollectionWithChunks(NamespaceString(kDbName, collName), - generateDefaultChunkRanges({kShardId0, kShardId1})); - } - - std::set<NamespaceString> collectionsSelected; - _imbalancedCollectionsCache.clear(); - - for (auto i = 0; i < 5; ++i) { - - auto future = launchAsync([this, &collectionsSelected]() { - ThreadClient tc(getServiceContext()); - auto opCtx = Client::getCurrent()->makeOperationContext(); - - // Requests chunks to be relocated requires running commands on each shard to - // get shard statistics. Set up dummy hosts for the source shards. - shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - auto chunksToMoveWithStatus = _chunkSelectionPolicy->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); - ASSERT_OK(chunksToMoveWithStatus.getStatus()); - - for (const auto& chunkToMove : chunksToMoveWithStatus.getValue()) { - collectionsSelected.insert(chunkToMove.nss); - } - }); - - expectGetStatsCommands(2 /*numShards*/); - - // Collection size distribution for each collection: - // Shard0 -> 512 MB - // Shard1 -> 0 MB - expectGetStatsForBalancingCommands( - {{kShardId0, 512 * 1024 * 1024 /*Bytes*/}, {kShardId1, 0 /*Bytes*/}}); - - future.default_timed_get(); - } - - // Check that all selected collections are cached - for (const auto& coll : collectionsSelected) { - ASSERT_TRUE(_imbalancedCollectionsCache.count(coll)); - } - ASSERT_EQ(_imbalancedCollectionsCache.size(), collectionsSelected.size()); -} - -TEST_F(BalancerChunkSelectionTest, CachedCollectionsShouldBeSelected) { - // Set up two shards in the metadata. - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard0, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard1, - kMajorityWriteConcern)); - - // Set up database - setUpDatabase(kDbName, kShardId0); - - _imbalancedCollectionsCache.clear(); - std::vector<NamespaceString> allCollections; - - // Set up 4 collections and add all them into the imbalanced collections cache - const int numCollections = 4; - for (auto i = 0; i < numCollections; ++i) { - NamespaceString nss(kDbName, "TestColl" + std::to_string(i)); - allCollections.push_back(nss); - setUpCollectionWithChunks(nss, generateDefaultChunkRanges({kShardId0, kShardId1})); - - _imbalancedCollectionsCache.insert(nss); - } - - std::set<NamespaceString> collectionsSelected; - - for (auto i = 0; i < 1000; ++i) { - - auto future = launchAsync([this, &collectionsSelected]() { - ThreadClient tc(getServiceContext()); - auto opCtx = Client::getCurrent()->makeOperationContext(); - - // Requests chunks to be relocated requires running commands on each shard to - // get shard statistics. Set up dummy hosts for the source shards. - shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - auto chunksToMoveWithStatus = _chunkSelectionPolicy->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); - ASSERT_OK(chunksToMoveWithStatus.getStatus()); - - for (const auto& chunkToMove : chunksToMoveWithStatus.getValue()) { - collectionsSelected.insert(chunkToMove.nss); - } - }); - - expectGetStatsCommands(2 /*numShards*/); - - // Collection size distribution for each collection: - // Shard0 -> 512 MB - // Shard1 -> 0 MB - expectGetStatsForBalancingCommands( - {{kShardId0, 512 * 1024 * 1024 /*Bytes*/}, {kShardId1, 0 /*Bytes*/}}); - - future.default_timed_get(); - - if (collectionsSelected.size() == allCollections.size()) { - break; - } - } - - // Check that all selected collections are cached - for (const auto& nss : allCollections) { - ASSERT_TRUE(collectionsSelected.count(nss)); - } - ASSERT_EQ(allCollections.size(), collectionsSelected.size()); -} - -TEST_F(BalancerChunkSelectionTest, MaxTimeToScheduleBalancingOperationsExceeded) { - // Set up 4 shards in the metadata. - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard0, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard1, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard2, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard3, - kMajorityWriteConcern)); - - // Set up database - setUpDatabase(kDbName, kShardId0); - - // Override collections batch size to 4 for speeding up the test - FailPointEnableBlock overrideBatchSizeGuard("overrideStatsForBalancingBatchSize", - BSON("size" << 4)); - - // Set up 5 collections to process more than 1 batch - for (auto i = 0U; i < 5; ++i) { - NamespaceString ns{kDbName, "coll" + std::to_string(i)}; - setUpCollectionWithChunks( - ns, generateDefaultChunkRanges({kShardId0, kShardId1, kShardId2, kShardId3})); - } - - auto future = launchAsync([&] { - ThreadClient tc(getServiceContext()); - auto opCtx = Client::getCurrent()->makeOperationContext(); - - // Requesting chunks to be relocated requires running commands on each shard to get - // shard statistics. Set up dummy hosts for the source shards. - shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - shardTargeterMock(opCtx.get(), kShardId2)->setFindHostReturnValue(kShardHost2); - shardTargeterMock(opCtx.get(), kShardId3)->setFindHostReturnValue(kShardHost3); - - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - _imbalancedCollectionsCache.clear(); - - // Forcing timeout to exceed by setting it to 0 - RAIIServerParameterControllerForTest balancerChunksSelectionTimeoutMsIsZero( - "balancerChunksSelectionTimeoutMs", 0); - - auto candidateChunksStatus = _chunkSelectionPolicy.get()->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); - - ASSERT_OK(candidateChunksStatus.getStatus()); - - // We know that timeout exceeded because we only got 1 migration instead of the 2 migrations - // expected in a normal scenario with 4 shards - ASSERT_EQUALS(1U, candidateChunksStatus.getValue().size()); - }); - - expectGetStatsCommands(4); - - // We need to get at least 1 migration per batch since the timeout only exceeds when balancer - // has found at least one candidate migration On the other side, we must get less than 2 - // migrations per batch since the maximum number of migrations per balancing round is 2 (with 4 - // shards) - expectGetStatsForBalancingCommandsWithOneMigration( - 4 /*numShards*/, kShardId0 /*donor*/, kShardId1 /*recipient*/); - - future.default_timed_get(); -} - -TEST_F(BalancerChunkSelectionTest, MakeSureMoreThanOneBatchIsProcessedIfNeeded) { - // Set up 4 shards in the metadata. - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard0, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard1, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard2, - kMajorityWriteConcern)); - ASSERT_OK(catalogClient()->insertConfigDocument(operationContext(), - NamespaceString::kConfigsvrShardsNamespace, - kShard3, - kMajorityWriteConcern)); - - // Set up database - setUpDatabase(kDbName, kShardId0); - - // Override collections batch size to 4 for speeding up the test - FailPointEnableBlock overrideBatchSizeGuard("overrideStatsForBalancingBatchSize", - BSON("size" << 4)); - - // Set up 5 collections to process 2 batches - for (auto i = 0; i < 5; ++i) { - NamespaceString ns{kDbName, "coll" + std::to_string(i)}; - setUpCollectionWithChunks( - ns, generateDefaultChunkRanges({kShardId0, kShardId1, kShardId2, kShardId3})); - } - - auto future = launchAsync([&] { - ThreadClient tc(getServiceContext()); - auto opCtx = Client::getCurrent()->makeOperationContext(); - - // Requesting chunks to be relocated requires running commands on each shard to get - // shard statistics. Set up dummy hosts for the source shards. - shardTargeterMock(opCtx.get(), kShardId0)->setFindHostReturnValue(kShardHost0); - shardTargeterMock(opCtx.get(), kShardId1)->setFindHostReturnValue(kShardHost1); - shardTargeterMock(opCtx.get(), kShardId2)->setFindHostReturnValue(kShardHost2); - shardTargeterMock(opCtx.get(), kShardId3)->setFindHostReturnValue(kShardHost3); - - std::vector<ClusterStatistics::ShardStatistics> shardStats = - uassertStatusOK(_clusterStats.get()->getStats(opCtx.get())); - auto availableShards = getAllShardIds(shardStats); - - _imbalancedCollectionsCache.clear(); - - // Forcing timeout to exceed - auto candidateChunksStatus = _chunkSelectionPolicy.get()->selectChunksToMove( - opCtx.get(), shardStats, &availableShards, &_imbalancedCollectionsCache); - - ASSERT_OK(candidateChunksStatus.getStatus()); - - // We know that timeout exceeded because we only got 1 migration instead of the 2 migrations - // expected in a normal scenario with 4 shards - ASSERT_EQUALS(2U, candidateChunksStatus.getValue().size()); - }); - - expectGetStatsCommands(4); - - // We are scheduling one migration on the first batch to make sure that the second batch is - // processed - expectGetStatsForBalancingCommandsWithOneMigration( - 4 /*numShards*/, kShardId0 /*donor*/, kShardId1 /*recipient*/); - expectGetStatsForBalancingCommandsWithOneMigration( - 4 /*numShards*/, kShardId2 /*donor*/, kShardId3 /*recipient*/); - - future.default_timed_get(); -} } // namespace } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler.h b/src/mongo/db/s/balancer/balancer_commands_scheduler.h index 2a16209a2c0..a75e18e944e 100644 --- a/src/mongo/db/s/balancer/balancer_commands_scheduler.h +++ b/src/mongo/db/s/balancer/balancer_commands_scheduler.h @@ -148,8 +148,7 @@ public: const ChunkRange& chunkRange, const ChunkVersion& version, const KeyPattern& keyPattern, - bool estimatedValue, - int64_t maxSize) = 0; + bool estimatedValue) = 0; virtual SemiFuture<void> requestMoveRange(OperationContext* opCtx, const ShardsvrMoveRange& request, diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp index 8348125a79a..5847ede7e55 100644 --- a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp +++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.cpp @@ -184,7 +184,6 @@ const std::string DataSizeCommandInfo::kKeyPattern = "keyPattern"; const std::string DataSizeCommandInfo::kMinValue = "min"; const std::string DataSizeCommandInfo::kMaxValue = "max"; const std::string DataSizeCommandInfo::kEstimatedValue = "estimate"; -const std::string DataSizeCommandInfo::kMaxSizeValue = "maxSize"; const std::string SplitChunkCommandInfo::kCommandName = "splitChunk"; const std::string SplitChunkCommandInfo::kShardName = "from"; @@ -256,12 +255,11 @@ SemiFuture<void> BalancerCommandsSchedulerImpl::requestMoveChunk( auto externalClientInfo = issuedByRemoteUser ? boost::optional<ExternalClientInfo>(opCtx) : boost::none; - invariant(migrateInfo.maxKey.has_value(), "Bound not present when requesting move chunk"); auto commandInfo = std::make_shared<MoveChunkCommandInfo>(migrateInfo.nss, migrateInfo.from, migrateInfo.to, migrateInfo.minKey, - *migrateInfo.maxKey, + migrateInfo.maxKey, commandSettings.maxChunkSizeBytes, commandSettings.secondaryThrottle, commandSettings.waitForDelete, @@ -360,15 +358,13 @@ SemiFuture<DataSizeResponse> BalancerCommandsSchedulerImpl::requestDataSize( const ChunkRange& chunkRange, const ChunkVersion& version, const KeyPattern& keyPattern, - bool estimatedValue, - int64_t maxSize) { + bool estimatedValue) { auto commandInfo = std::make_shared<DataSizeCommandInfo>(nss, shardId, keyPattern.toBSON(), chunkRange.getMin(), chunkRange.getMax(), estimatedValue, - maxSize, version); return _buildAndEnqueueNewRequest(opCtx, std::move(commandInfo)) @@ -380,8 +376,7 @@ SemiFuture<DataSizeResponse> BalancerCommandsSchedulerImpl::requestDataSize( } long long sizeBytes = remoteResponse.data["size"].number(); long long numObjects = remoteResponse.data["numObjects"].number(); - bool maxSizeReached = remoteResponse.data["maxReached"].trueValue(); - return DataSizeResponse(sizeBytes, numObjects, maxSizeReached); + return DataSizeResponse(sizeBytes, numObjects); }) .semi(); } @@ -427,53 +422,49 @@ CommandSubmissionResult BalancerCommandsSchedulerImpl::_submit( LOGV2_DEBUG( 5847203, 2, "Balancer command request submitted for execution", "reqId"_attr = params.id); bool distLockTaken = false; - try { - const auto shardWithStatus = - Grid::get(opCtx)->shardRegistry()->getShard(opCtx, params.commandInfo->getTarget()); - if (!shardWithStatus.isOK()) { - return CommandSubmissionResult(params.id, distLockTaken, shardWithStatus.getStatus()); - } - const auto shardHostWithStatus = shardWithStatus.getValue()->getTargeter()->findHost( - opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}); - if (!shardHostWithStatus.isOK()) { - return CommandSubmissionResult( - params.id, distLockTaken, shardHostWithStatus.getStatus()); - } + const auto shardWithStatus = + Grid::get(opCtx)->shardRegistry()->getShard(opCtx, params.commandInfo->getTarget()); + if (!shardWithStatus.isOK()) { + return CommandSubmissionResult(params.id, distLockTaken, shardWithStatus.getStatus()); + } - if (params.commandInfo->requiresRecoveryOnCrash()) { - auto writeStatus = persistRecoveryInfo(opCtx, *(params.commandInfo)); - if (!writeStatus.isOK()) { - return CommandSubmissionResult(params.id, distLockTaken, writeStatus); - } - } + const auto shardHostWithStatus = shardWithStatus.getValue()->getTargeter()->findHost( + opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}); + if (!shardHostWithStatus.isOK()) { + return CommandSubmissionResult(params.id, distLockTaken, shardHostWithStatus.getStatus()); + } - const executor::RemoteCommandRequest remoteCommand = - executor::RemoteCommandRequest(shardHostWithStatus.getValue(), - params.commandInfo->getTargetDb(), - params.commandInfo->serialise(), - opCtx); - auto onRemoteResponseReceived = - [this, - requestId = params.id](const executor::TaskExecutor::RemoteCommandCallbackArgs& args) { - _applyCommandResponse(requestId, args.response); - }; - - if (params.commandInfo->requiresDistributedLock()) { - Status lockAcquisitionResponse = - _distributedLocks.acquireFor(opCtx, params.commandInfo->getNameSpace()); - if (!lockAcquisitionResponse.isOK()) { - return CommandSubmissionResult(params.id, distLockTaken, lockAcquisitionResponse); - } - distLockTaken = true; + if (params.commandInfo->requiresRecoveryOnCrash()) { + auto writeStatus = persistRecoveryInfo(opCtx, *(params.commandInfo)); + if (!writeStatus.isOK()) { + return CommandSubmissionResult(params.id, distLockTaken, writeStatus); } + } - auto swRemoteCommandHandle = - (*_executor)->scheduleRemoteCommand(remoteCommand, onRemoteResponseReceived); - return CommandSubmissionResult(params.id, distLockTaken, swRemoteCommandHandle.getStatus()); - } catch (const DBException& e) { - return CommandSubmissionResult(params.id, distLockTaken, e.toStatus()); + const executor::RemoteCommandRequest remoteCommand = + executor::RemoteCommandRequest(shardHostWithStatus.getValue(), + params.commandInfo->getTargetDb(), + params.commandInfo->serialise(), + opCtx); + auto onRemoteResponseReceived = + [this, + requestId = params.id](const executor::TaskExecutor::RemoteCommandCallbackArgs& args) { + _applyCommandResponse(requestId, args.response); + }; + + if (params.commandInfo->requiresDistributedLock()) { + Status lockAcquisitionResponse = + _distributedLocks.acquireFor(opCtx, params.commandInfo->getNameSpace()); + if (!lockAcquisitionResponse.isOK()) { + return CommandSubmissionResult(params.id, distLockTaken, lockAcquisitionResponse); + } + distLockTaken = true; } + + auto swRemoteCommandHandle = + (*_executor)->scheduleRemoteCommand(remoteCommand, onRemoteResponseReceived); + return CommandSubmissionResult(params.id, distLockTaken, swRemoteCommandHandle.getStatus()); } void BalancerCommandsSchedulerImpl::_applySubmissionResult( @@ -545,11 +536,6 @@ void BalancerCommandsSchedulerImpl::_workerThread() { }); Client::initThread("BalancerCommandsScheduler"); - { - stdx::lock_guard<Client> lk(cc()); - cc().setSystemOperationKillableByStepdown(lk); - } - bool stopWorkerRequested = false; LOGV2(5847205, "Balancer scheduler thread started"); diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h index 5803a880029..92f9f074441 100644 --- a/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h +++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_impl.h @@ -336,14 +336,12 @@ public: const BSONObj& lowerBoundKey, const BSONObj& upperBoundKey, bool estimatedValue, - int64_t maxSize, const ChunkVersion& version) : CommandInfo(shardId, nss, boost::none), _shardKeyPattern(shardKeyPattern), _lowerBoundKey(lowerBoundKey), _upperBoundKey(upperBoundKey), _estimatedValue(estimatedValue), - _maxSize(maxSize), _version(version) {} BSONObj serialise() const override { @@ -352,8 +350,7 @@ public: .append(kKeyPattern, _shardKeyPattern) .append(kMinValue, _lowerBoundKey) .append(kMaxValue, _upperBoundKey) - .append(kEstimatedValue, _estimatedValue) - .append(kMaxSizeValue, _maxSize); + .append(kEstimatedValue, _estimatedValue); _version.serializeToBSON(ChunkVersion::kShardVersionField, &commandBuilder); @@ -365,7 +362,6 @@ private: BSONObj _lowerBoundKey; BSONObj _upperBoundKey; bool _estimatedValue; - int64_t _maxSize; ChunkVersion _version; static const std::string kCommandName; @@ -373,7 +369,6 @@ private: static const std::string kMinValue; static const std::string kMaxValue; static const std::string kEstimatedValue; - static const std::string kMaxSizeValue; }; class SplitChunkCommandInfo : public CommandInfo { @@ -594,8 +589,7 @@ public: const ChunkRange& chunkRange, const ChunkVersion& version, const KeyPattern& keyPattern, - bool estimatedValue, - int64_t maxSize) override; + bool estimatedValue) override; private: enum class SchedulerState { Recovering, Running, Stopping, Stopped }; diff --git a/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp b/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp index fdf01b9eebd..aab01274138 100644 --- a/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp +++ b/src/mongo/db/s/balancer/balancer_commands_scheduler_test.cpp @@ -319,15 +319,13 @@ TEST_F(BalancerCommandsSchedulerTest, SuccessfulRequestChunkDataSizeCommand) { _scheduler.start(operationContext(), getMigrationRecoveryDefaultValues()); ChunkType chunk = makeChunk(0, kShardId0); - auto futureResponse = - _scheduler.requestDataSize(operationContext(), - kNss, - chunk.getShard(), - chunk.getRange(), - chunk.getVersion(), - KeyPattern(BSON("x" << 1)), - false /* issuedByRemoteUser */, - (kDefaultMaxChunkSizeBytes / 100) * 25 /* maxSize */); + auto futureResponse = _scheduler.requestDataSize(operationContext(), + kNss, + chunk.getShard(), + chunk.getRange(), + chunk.getVersion(), + KeyPattern(BSON("x" << 1)), + false /* issuedByRemoteUser */); auto swReceivedDataSize = futureResponse.getNoThrow(); ASSERT_OK(swReceivedDataSize.getStatus()); auto receivedDataSize = swReceivedDataSize.getValue(); @@ -436,12 +434,11 @@ TEST_F(BalancerCommandsSchedulerTest, MoveChunkCommandGetsPersistedOnDiskWhenReq ASSERT_EQ(kNss, recoveredCommand->getNameSpace()); ASSERT_EQ(migrateInfo.from, recoveredCommand->getTarget()); ASSERT_TRUE(recoveredCommand->requiresDistributedLock()); - MoveChunkCommandInfo originalCommandInfo(migrateInfo.nss, migrateInfo.from, migrateInfo.to, migrateInfo.minKey, - *migrateInfo.maxKey, + migrateInfo.maxKey, requestSettings.maxChunkSizeBytes, requestSettings.secondaryThrottle, requestSettings.waitForDelete, @@ -467,7 +464,7 @@ TEST_F(BalancerCommandsSchedulerTest, PersistedCommandsAreReissuedWhenRecovering auto requestSettings = getMoveChunkSettings(kCustomizedMaxChunkSizeBytes); MigrationType recoveryInfo(migrateInfo.nss, migrateInfo.minKey, - *migrateInfo.maxKey, + migrateInfo.maxKey, migrateInfo.from, migrateInfo.to, migrateInfo.version, diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy.h b/src/mongo/db/s/balancer/balancer_defragmentation_policy.h index 0ccdf2f929a..3358c6c79df 100644 --- a/src/mongo/db/s/balancer/balancer_defragmentation_policy.h +++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy.h @@ -45,9 +45,12 @@ public: virtual ~BalancerDefragmentationPolicy() {} /** - * Requests the execution of the defragmentation algorithm on the required collections. + * Requests the execution of the defragmentation algorithm on the specified collection. + * Returns true if the request is accepted, false if ignored (meaning, the specified collection + * is already being processed) */ - virtual void startCollectionDefragmentations(OperationContext* opCtx) = 0; + virtual void startCollectionDefragmentation(OperationContext* opCtx, + const CollectionType& coll) = 0; /** * Checks if the collection is currently being defragmented, and signals the defragmentation @@ -73,11 +76,11 @@ public: /** * Pulls the next batch of actionable chunk migration requests, given the current internal state - * and the passed in list of available shards. + * and the passed in list of unavaible shards. * Every chunk migration request is then expected to be acknowledged by the balancer by issuing * a call to applyActionResult() (declared in ActionsStreamPolicy) */ virtual MigrateInfoVector selectChunksToMove(OperationContext* opCtx, - stdx::unordered_set<ShardId>* availableShards) = 0; + stdx::unordered_set<ShardId>* usedShards) = 0; }; } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp index 25087171256..0f1063d6d8c 100644 --- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp +++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.cpp @@ -58,8 +58,6 @@ const std::string kProgress("progress"); const std::string kNoPhase("none"); const std::string kRemainingChunksToProcess("remainingChunksToProcess"); -static constexpr int64_t kBigChunkMarker = std::numeric_limits<int64_t>::max(); - ChunkVersion getShardVersion(OperationContext* opCtx, const ShardId& shardId, const NamespaceString& nss) { @@ -87,7 +85,10 @@ uint64_t getCollectionMaxChunkSizeBytes(OperationContext* opCtx, const Collectio } ZoneInfo getCollectionZones(OperationContext* opCtx, const CollectionType& coll) { - return uassertStatusOK(createCollectionZoneInfo(opCtx, coll.getNss(), coll.getKeyPattern())); + ZoneInfo zones; + uassertStatusOK( + ZoneInfo::addTagsFromCatalog(opCtx, coll.getNss(), coll.getKeyPattern(), zones)); + return zones; } bool isRetriableForDefragmentation(const Status& status) { @@ -165,11 +166,6 @@ public: auto collectionChunks = getCollectionChunks(opCtx, coll); const auto collectionZones = getCollectionZones(opCtx, coll); - // Calculate small chunk threshold to limit dataSize commands - const auto maxChunkSizeBytes = getCollectionMaxChunkSizeBytes(opCtx, coll); - const int64_t smallChunkSizeThreshold = - (maxChunkSizeBytes / 100) * kSmallChunkSizeThresholdPctg; - stdx::unordered_map<ShardId, PendingActions> pendingActionsByShards; // Find ranges of chunks; for single-chunk ranges, request DataSize; for multi-range, issue // merge @@ -196,7 +192,6 @@ public: new MergeAndMeasureChunksPhase(coll.getNss(), coll.getUuid(), coll.getKeyPattern().toBSON(), - smallChunkSizeThreshold, std::move(pendingActionsByShards))); } @@ -222,15 +217,8 @@ public: if (pendingActions.rangesWithoutDataSize.size() > pendingActions.rangesToMerge.size()) { const auto& rangeToMeasure = pendingActions.rangesWithoutDataSize.back(); - nextAction = boost::optional<DefragmentationAction>( - DataSizeInfo(shardId, - _nss, - _uuid, - rangeToMeasure, - shardVersion, - _shardKey, - true /* estimate */, - _smallChunkSizeThresholdBytes /* maxSize */)); + nextAction = boost::optional<DefragmentationAction>(DataSizeInfo( + shardId, _nss, _uuid, rangeToMeasure, shardVersion, _shardKey, false)); pendingActions.rangesWithoutDataSize.pop_back(); } else if (!pendingActions.rangesToMerge.empty()) { const auto& rangeToMerge = pendingActions.rangesToMerge.back(); @@ -257,7 +245,7 @@ public: } boost::optional<MigrateInfo> popNextMigration( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) override { + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) override { return boost::none; } @@ -303,17 +291,10 @@ public: dataSizeAction.version, dataSizeAction.shardId); auto catalogManager = ShardingCatalogManager::get(opCtx); - // Max out the chunk size if it has has been estimated as - // bigger than _smallChunkSizeThresholdBytes; this will exlude - // the chunk from the list of candidates considered by - // MoveAndMergeChunksPhase - auto estimatedSize = dataSizeResponse.getValue().maxSizeReached - ? kBigChunkMarker - : dataSizeResponse.getValue().sizeBytes; catalogManager->setChunkEstimatedSize( opCtx, chunk, - estimatedSize, + dataSizeResponse.getValue().sizeBytes, ShardingCatalogClient::kMajorityWriteConcern); }, [&]() { @@ -366,12 +347,10 @@ private: const NamespaceString& nss, const UUID& uuid, const BSONObj& shardKey, - const int64_t smallChunkSizeThresholdBytes, stdx::unordered_map<ShardId, PendingActions>&& pendingActionsByShards) : _nss(nss), _uuid(uuid), _shardKey(shardKey), - _smallChunkSizeThresholdBytes(smallChunkSizeThresholdBytes), _pendingActionsByShards(std::move(pendingActionsByShards)) {} void _abort(const DefragmentationPhaseEnum nextPhase) { @@ -383,7 +362,6 @@ private: const NamespaceString _nss; const UUID _uuid; const BSONObj _shardKey; - const int64_t _smallChunkSizeThresholdBytes; stdx::unordered_map<ShardId, PendingActions> _pendingActionsByShards; boost::optional<ShardId> _shardToProcess; size_t _outstandingActions{0}; @@ -418,8 +396,7 @@ public: std::move(collectionChunks), std::move(shardInfos), std::move(collectionZones), - smallChunkSizeThresholdBytes, - maxChunkSizeBytes)); + smallChunkSizeThresholdBytes)); } DefragmentationPhaseEnum getType() const override { @@ -445,9 +422,9 @@ public: } boost::optional<MigrateInfo> popNextMigration( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) override { + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) override { for (const auto& shardId : _shardProcessingOrder) { - if (availableShards->count(shardId) == 0) { + if (usedShards->count(shardId) != 0) { // the shard is already busy in a migration continue; } @@ -455,7 +432,7 @@ public: ChunkRangeInfoIterator nextSmallChunk; std::list<ChunkRangeInfoIterator> candidateSiblings; if (!_findNextSmallChunkInShard( - shardId, *availableShards, &nextSmallChunk, &candidateSiblings)) { + shardId, *usedShards, &nextSmallChunk, &candidateSiblings)) { // there isn't a chunk in this shard that can currently be moved and merged with one // of its siblings. continue; @@ -478,12 +455,11 @@ public: // ... then build up the migration request, marking the needed resources as busy. nextSmallChunk->busyInOperation = true; targetSibling->busyInOperation = true; - availableShards->erase(nextSmallChunk->shard); - availableShards->erase(targetSibling->shard); + usedShards->insert(nextSmallChunk->shard); + usedShards->insert(targetSibling->shard); auto smallChunkVersion = getShardVersion(opCtx, nextSmallChunk->shard, _nss); _outstandingMigrations.emplace_back(nextSmallChunk, targetSibling); - return _outstandingMigrations.back().asMigrateInfo( - _uuid, _nss, smallChunkVersion, _maxChunkSizeBytes); + return _outstandingMigrations.back().asMigrateInfo(_uuid, _nss, smallChunkVersion); } return boost::none; @@ -518,7 +494,6 @@ public: _nss, boost::none, moveRequest.getDestinationShard()); auto transferredAmount = moveRequest.getMovedDataSizeBytes(); - invariant(transferredAmount <= _smallChunkSizeThresholdBytes); _shardInfos.at(moveRequest.getSourceShard()).currentSizeBytes -= transferredAmount; _shardInfos.at(moveRequest.getDestinationShard()).currentSizeBytes += @@ -542,14 +517,6 @@ public: moveRequest.chunkToMove->busyInOperation = false; moveRequest.chunkToMergeWith->busyInOperation = false; - if (migrationResponse.code() == ErrorCodes::ChunkTooBig || - migrationResponse.code() == ErrorCodes::ExceededMemoryLimit) { - // Never try moving this chunk again, it isn't actually small - _removeIteratorFromSmallChunks(moveRequest.chunkToMove, - moveRequest.chunkToMove->shard); - return; - } - if (isRetriableForDefragmentation(migrationResponse)) { // The migration will be eventually retried return; @@ -607,12 +574,7 @@ public: auto& chunkToDelete = mergeRequest.chunkToMove; mergedChunk->range = mergeRequest.asMergedRange(); - if (mergedChunk->estimatedSizeBytes != kBigChunkMarker && - chunkToDelete->estimatedSizeBytes != kBigChunkMarker) { - mergedChunk->estimatedSizeBytes += chunkToDelete->estimatedSizeBytes; - } else { - mergedChunk->estimatedSizeBytes = kBigChunkMarker; - } + mergedChunk->estimatedSizeBytes += chunkToDelete->estimatedSizeBytes; mergedChunk->busyInOperation = false; auto deletedChunkShard = chunkToDelete->shard; // the lookup data structures... @@ -741,8 +703,7 @@ private: MigrateInfo asMigrateInfo(const UUID& collUuid, const NamespaceString& nss, - const ChunkVersion& version, - uint64_t maxChunkSizeBytes) const { + const ChunkVersion& version) const { return MigrateInfo(chunkToMergeWith->shard, chunkToMove->shard, nss, @@ -750,8 +711,7 @@ private: chunkToMove->range.getMin(), chunkToMove->range.getMax(), version, - MoveChunkRequest::ForceJumbo::kDoNotForce, - maxChunkSizeBytes); + MoveChunkRequest::ForceJumbo::kForceBalancer); } ChunkRange asMergedRange() const { @@ -779,7 +739,7 @@ private: return chunkToMove->range.getMin(); } - int64_t getMovedDataSizeBytes() const { + uint64_t getMovedDataSizeBytes() const { return chunkToMove->estimatedSizeBytes; } @@ -790,6 +750,8 @@ private: bool _isChunkToMergeLeftSibling; }; + static constexpr uint64_t kSmallChunkSizeThresholdPctg = 25; + const NamespaceString _nss; const UUID _uuid; @@ -814,8 +776,6 @@ private: const int64_t _smallChunkSizeThresholdBytes; - const uint64_t _maxChunkSizeBytes; - bool _aborted{false}; DefragmentationPhaseEnum _nextPhase{DefragmentationPhaseEnum::kMergeChunks}; @@ -825,8 +785,7 @@ private: std::vector<ChunkType>&& collectionChunks, stdx::unordered_map<ShardId, ShardInfo>&& shardInfos, ZoneInfo&& collectionZones, - uint64_t smallChunkSizeThresholdBytes, - uint64_t maxChunkSizeBytes) + uint64_t smallChunkSizeThresholdBytes) : _nss(nss), _uuid(uuid), _collectionChunks(), @@ -837,8 +796,7 @@ private: _actionableMerges(), _outstandingMerges(), _zoneInfo(std::move(collectionZones)), - _smallChunkSizeThresholdBytes(smallChunkSizeThresholdBytes), - _maxChunkSizeBytes(maxChunkSizeBytes) { + _smallChunkSizeThresholdBytes(smallChunkSizeThresholdBytes) { // Load the collection routing table in a std::list to ease later manipulation for (auto&& chunk : collectionChunks) { @@ -922,7 +880,7 @@ private: // Returns true on success (storing the related info in nextSmallChunk + smallChunkSiblings), // false otherwise. bool _findNextSmallChunkInShard(const ShardId& shard, - const stdx::unordered_set<ShardId>& availableShards, + const stdx::unordered_set<ShardId>& usedShards, ChunkRangeInfoIterator* nextSmallChunk, std::list<ChunkRangeInfoIterator>* smallChunkSiblings) { auto matchingShardInfo = _smallChunksByShard.find(shard); @@ -948,7 +906,7 @@ private: size_t siblingsDiscardedDueToRangeDeletion = 0; for (const auto& sibling : candidateSiblings) { - if (sibling->busyInOperation || !availableShards.count(sibling->shard)) { + if (sibling->busyInOperation || usedShards.count(sibling->shard)) { continue; } if ((*candidateIt)->shardsToAvoid.count(sibling->shard)) { @@ -1020,10 +978,8 @@ private: mergeableSibling.estimatedSizeBytes) { ranking += kConvenientMove; } - auto estimatedMergedSize = (chunkTobeMovedAndMerged.estimatedSizeBytes == kBigChunkMarker || - mergeableSibling.estimatedSizeBytes == kBigChunkMarker) - ? kBigChunkMarker - : chunkTobeMovedAndMerged.estimatedSizeBytes + mergeableSibling.estimatedSizeBytes; + auto estimatedMergedSize = + chunkTobeMovedAndMerged.estimatedSizeBytes + mergeableSibling.estimatedSizeBytes; if (estimatedMergedSize > _smallChunkSizeThresholdBytes) { ranking += mergeableSibling.estimatedSizeBytes < _smallChunkSizeThresholdBytes ? kMergeSolvesTwoPendingChunks @@ -1124,7 +1080,7 @@ public: } boost::optional<MigrateInfo> popNextMigration( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) override { + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) override { return boost::none; } @@ -1309,7 +1265,7 @@ public: } boost::optional<MigrateInfo> popNextMigration( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) override { + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) override { return boost::none; } @@ -1436,26 +1392,13 @@ private: } // namespace -void BalancerDefragmentationPolicyImpl::startCollectionDefragmentations(OperationContext* opCtx) { - stdx::lock_guard<Latch> lk(_stateMutex); - - // Fetch all collections with `defragmentCollection` flag enabled - static const auto query = BSON(CollectionType::kDefragmentCollectionFieldName << true); - auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - const auto& collDocs = uassertStatusOK(configShard->exhaustiveFindOnConfig( - opCtx, - ReadPreferenceSetting(ReadPreference::Nearest), - repl::ReadConcernLevel::kMajorityReadConcern, - NamespaceString::kConfigsvrCollectionsNamespace, - query, - BSONObj(), - boost::none)) - .docs; - - for (const BSONObj& obj : collDocs) { - const CollectionType coll{obj}; - if (_defragmentationStates.contains(coll.getUuid())) { - continue; +void BalancerDefragmentationPolicyImpl::startCollectionDefragmentation(OperationContext* opCtx, + const CollectionType& coll) { + { + stdx::lock_guard<Latch> lk(_stateMutex); + const auto& uuid = coll.getUuid(); + if (!coll.getDefragmentCollection() || _defragmentationStates.contains(uuid)) { + return; } _initializeCollectionState(lk, opCtx, coll); } @@ -1500,8 +1443,7 @@ BSONObj BalancerDefragmentationPolicyImpl::reportProgressOn(const UUID& uuid) { } MigrateInfoVector BalancerDefragmentationPolicyImpl::selectChunksToMove( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) { - + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) { MigrateInfoVector chunksToMove; { stdx::lock_guard<Latch> lk(_stateMutex); @@ -1528,10 +1470,6 @@ MigrateInfoVector BalancerDefragmentationPolicyImpl::selectChunksToMove( for (auto it = collectionUUIDs.begin(); it != collectionUUIDs.end();) { const auto& collUUID = *it; - if (availableShards->size() == 0) { - return chunksToMove; - } - try { auto defragStateIt = _defragmentationStates.find(collUUID); if (defragStateIt == _defragmentationStates.end()) { @@ -1546,7 +1484,7 @@ MigrateInfoVector BalancerDefragmentationPolicyImpl::selectChunksToMove( continue; } auto actionableMigration = - collDefragmentationPhase->popNextMigration(opCtx, availableShards); + collDefragmentationPhase->popNextMigration(opCtx, usedShards); if (!actionableMigration.has_value()) { it = popCollectionUUID(it); continue; @@ -1567,7 +1505,7 @@ MigrateInfoVector BalancerDefragmentationPolicyImpl::selectChunksToMove( } } - if (chunksToMove.empty()) { + if (chunksToMove.empty() && usedShards->empty()) { // If the policy cannot produce new migrations even in absence of temporary constraints, it // is possible that some streaming actions must be processed first. Notify an update of the // internal state to make it happen. diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h index 6e45853eac1..bc41346ca7f 100644 --- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h +++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_impl.h @@ -53,7 +53,7 @@ public: OperationContext* opCtx) = 0; virtual boost::optional<MigrateInfo> popNextMigration( - OperationContext* opCtx, stdx::unordered_set<ShardId>* availableShards) = 0; + OperationContext* opCtx, stdx::unordered_set<ShardId>* usedShards) = 0; virtual void applyActionResult(OperationContext* opCtx, const DefragmentationAction& action, @@ -64,9 +64,6 @@ public: virtual bool isComplete() const = 0; virtual void userAbort() = 0; - -protected: - static constexpr uint64_t kSmallChunkSizeThresholdPctg = 25; }; class BalancerDefragmentationPolicyImpl : public BalancerDefragmentationPolicy { @@ -88,7 +85,7 @@ public: virtual BSONObj reportProgressOn(const UUID& uuid) override; MigrateInfoVector selectChunksToMove(OperationContext* opCtx, - stdx::unordered_set<ShardId>* availableShards) override; + stdx::unordered_set<ShardId>* usedShards) override; StringData getName() const override; @@ -98,7 +95,8 @@ public: const DefragmentationAction& action, const DefragmentationActionResponse& response) override; - void startCollectionDefragmentations(OperationContext* opCtx) override; + void startCollectionDefragmentation(OperationContext* opCtx, + const CollectionType& coll) override; void abortCollectionDefragmentation(OperationContext* opCtx, const NamespaceString& nss) override; diff --git a/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp b/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp index 98e2b921329..f47c899cee7 100644 --- a/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp +++ b/src/mongo/db/s/balancer/balancer_defragmentation_policy_test.cpp @@ -32,7 +32,6 @@ #include "mongo/db/s/balancer/balancer_random.h" #include "mongo/db/s/balancer/cluster_statistics_mock.h" #include "mongo/db/s/config/config_server_test_fixture.h" -#include "mongo/idl/server_parameter_test_util.h" namespace mongo { namespace { @@ -41,13 +40,8 @@ using ShardStatistics = ClusterStatistics::ShardStatistics; class BalancerDefragmentationPolicyTest : public ConfigServerTestFixture { protected: - inline static const NamespaceString kNss1{"testDb.testColl1"}; - inline static const NamespaceString kNss2{"testDb.testColl2"}; - inline static const NamespaceString kNss3{"testDb.testColl3"}; - const UUID kUuid1 = UUID::gen(); - const UUID kUuid2 = UUID::gen(); - const UUID kUuid3 = UUID::gen(); - + const NamespaceString kNss{"testDb.testColl"}; + const UUID kUuid = UUID::gen(); const ShardId kShardId0 = ShardId("shard0"); const ShardId kShardId1 = ShardId("shard1"); const ShardId kShardId2 = ShardId("shard2"); @@ -85,12 +79,12 @@ protected: _defragmentationPolicy(&_clusterStats, _random, onDefragmentationStateUpdated) {} CollectionType setupCollectionWithPhase( - const NamespaceString& nss, const std::vector<ChunkType>& chunkList, boost::optional<DefragmentationPhaseEnum> startingPhase = boost::none, boost::optional<int64_t> maxChunkSizeBytes = boost::none) { - setupCollection(nss, kShardKeyPattern, chunkList); + setupShards(kShardList); + setupCollection(kNss, kShardKeyPattern, chunkList); const auto updateClause = [&] { BSONObjBuilder builder; @@ -109,33 +103,31 @@ protected: return builder.obj(); }(); - const UUID& uuid = chunkList.at(0).getCollectionUUID(); ASSERT_OK(updateToConfigCollection(operationContext(), CollectionType::ConfigNS, - BSON(CollectionType::kUuidFieldName << uuid), + BSON(CollectionType::kUuidFieldName << kUuid), updateClause, false)); return Grid::get(operationContext()) ->catalogClient() - ->getCollection(operationContext(), uuid); + ->getCollection(operationContext(), kUuid); } - ChunkType makeConfigChunkEntry(const UUID& uuid, - const boost::optional<int64_t>& estimatedSize = boost::none) { - ChunkType chunk(uuid, ChunkRange(kKeyAtMin, kKeyAtMax), kCollectionVersion, kShardId0); + ChunkType makeConfigChunkEntry(const boost::optional<int64_t>& estimatedSize = boost::none) { + ChunkType chunk(kUuid, ChunkRange(kKeyAtMin, kKeyAtMax), kCollectionVersion, kShardId0); chunk.setEstimatedSizeBytes(estimatedSize); return chunk; } - std::vector<ChunkType> makeMergeableConfigChunkEntries(const UUID& uuid) { - return {ChunkType(uuid, ChunkRange(kKeyAtMin, kKeyAtTen), kCollectionVersion, kShardId0), - ChunkType(uuid, ChunkRange(kKeyAtTen, kKeyAtMax), kCollectionVersion, kShardId0)}; + std::vector<ChunkType> makeMergeableConfigChunkEntries() { + return {ChunkType(kUuid, ChunkRange(kKeyAtMin, kKeyAtTen), kCollectionVersion, kShardId0), + ChunkType(kUuid, ChunkRange(kKeyAtTen, kKeyAtMax), kCollectionVersion, kShardId0)}; } - BSONObj getConfigCollectionEntry(const UUID& uuid) { + BSONObj getConfigCollectionEntry() { DBDirectClient client(operationContext()); FindCommandRequest findRequest{NamespaceStringOrUUID{CollectionType::ConfigNS}}; - findRequest.setFilter(BSON(CollectionType::kUuidFieldName << uuid)); + findRequest.setFilter(BSON(CollectionType::kUuidFieldName << kUuid)); auto cursor = client.find(std::move(findRequest)); if (!cursor || !cursor->more()) return BSONObj(); @@ -152,10 +144,11 @@ protected: bool maxed = false, bool draining = false, std::set<std::string>&& zones = {}) { - return ShardStatistics(id, maxed ? currentSizeBytes : 0, currentSizeBytes, draining, zones); + return ShardStatistics( + id, maxed ? currentSizeBytes : 0, currentSizeBytes, draining, zones, ""); } - void setDefaultClusterStats(const std::vector<NamespaceString>& nssList = {kNss1}) { + void setDefaultClusterStats() { uint64_t oneKB = 1024 * 1024; auto shardInstance = 0; std::vector<ShardStatistics> stats; @@ -163,44 +156,26 @@ protected: for (const auto& shard : kShardList) { ++shardInstance; stats.push_back(buildShardStats(shard.getName(), oneKB * 1024 * shardInstance)); - - for (const auto& nss : nssList) { - collStats[nss].push_back(buildShardStats(shard.getName(), oneKB * shardInstance)); - } + collStats[kNss].push_back(buildShardStats(shard.getName(), oneKB * shardInstance)); } _clusterStats.setStats(std::move(stats), std::move(collStats)); } - void verifyExpectedDefragmentationStateOnDisk( - const UUID& uuid, boost::optional<DefragmentationPhaseEnum> expectedPhase) { + void verifyExpectedDefragmentationPhaseOndisk( + boost::optional<DefragmentationPhaseEnum> expectedPhase) { auto configDoc = findOneOnConfigCollection(operationContext(), CollectionType::ConfigNS, - BSON(CollectionType::kUuidFieldName << uuid)) + BSON(CollectionType::kUuidFieldName << kUuid)) .getValue(); if (expectedPhase.has_value()) { auto storedDefragmentationPhase = DefragmentationPhase_parse( IDLParserErrorContext("BalancerDefragmentationPolicyTest"), configDoc.getStringField(CollectionType::kDefragmentationPhaseFieldName)); ASSERT_TRUE(storedDefragmentationPhase == *expectedPhase); - ASSERT_TRUE(configDoc[CollectionType::kDefragmentCollectionFieldName].Bool()); } else { ASSERT_FALSE(configDoc.hasField(CollectionType::kDefragmentationPhaseFieldName)); - ASSERT_FALSE(configDoc.hasField(CollectionType::kDefragmentCollectionFieldName)); } }; - - stdx::unordered_set<ShardId> getAllShardIds(OperationContext* opCtx) { - std::vector<ShardStatistics> shardStats = _clusterStats.getStats(opCtx).getValue(); - stdx::unordered_set<ShardId> shards; - std::transform(shardStats.begin(), - shardStats.end(), - std::inserter(shards, shards.end()), - [](const ClusterStatistics::ShardStatistics& shardStatistics) -> ShardId { - return shardStatistics.shardId; - }); - - return shards; - } }; TEST_F(BalancerDefragmentationPolicyTest, TestGetNextActionIsNotReadyWhenNotDefragmenting) { @@ -209,35 +184,32 @@ TEST_F(BalancerDefragmentationPolicyTest, TestGetNextActionIsNotReadyWhenNotDefr } TEST_F(BalancerDefragmentationPolicyTest, TestAddCollectionWhenCollectionRemovedFailsGracefully) { - CollectionType coll( - kNss1, OID::gen(), Timestamp(1, 1), Date_t::now(), kUuid1, kShardKeyPattern); + CollectionType coll(kNss, OID::gen(), Timestamp(1, 1), Date_t::now(), kUuid, kShardKeyPattern); coll.setDefragmentCollection(true); // Collection entry is not persisted (to simulate collection dropped), defragmentation should // not begin. ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); auto configDoc = findOneOnConfigCollection(operationContext(), CollectionType::ConfigNS, - BSON(CollectionType::kUuidFieldName << kUuid1)); + BSON(CollectionType::kUuidFieldName << kUuid)); ASSERT_EQ(configDoc.getStatus(), Status(ErrorCodes::NoMatchingDocument, "No document found")); } // Phase 1 tests. TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAddSingleChunkCollectionTriggersDataSize) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // 1. The collection should be marked as undergoing through phase 1 of the algorithm... ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMergeAndMeasureChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMergeAndMeasureChunks); // 2. The action returned by the stream should be now an actionable DataSizeCommand... nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction); @@ -249,51 +221,47 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAddSingleChunkCollectionTr TEST_F(BalancerDefragmentationPolicyTest, AddSingleChunkCollectionWithKnownDataSizeCompletesDefragmentationWithNoOperationIssued) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1, 1024)}); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(1024)}); setDefaultClusterStats(); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMoveAndMergeChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMoveAndMergeChunks); // kMoveAndMergeChunks has no stream actions/migrations to offer, but the condition has to be // verified through a sequence of two action requests (the first being selectChunksToMove()) for // the phase to complete. - auto availableShards = getAllShardIds(operationContext()); + stdx::unordered_set<ShardId> usedShards; auto pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); + _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); ASSERT_TRUE(pendingMigrations.empty()); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMoveAndMergeChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMoveAndMergeChunks); nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), boost::none); + verifyExpectedDefragmentationPhaseOndisk(boost::none); ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); } TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAcknowledgeFinalDataSizeActionCompletesPhase) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); setDefaultClusterStats(); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction); - auto resp = StatusWith(DataSizeResponse(2000, 4, false)); + auto resp = StatusWith(DataSizeResponse(2000, 4)); _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp); // 1. The outcome of the data size has been stored in the expected document... auto chunkQuery = BSON(ChunkType::collectionUUID() - << kUuid1 << ChunkType::min(kKeyAtMin) << ChunkType::max(kKeyAtMax)); + << kUuid << ChunkType::min(kKeyAtMin) << ChunkType::max(kKeyAtMax)); auto configChunkDoc = findOneOnConfigCollection(operationContext(), ChunkType::ConfigNS, chunkQuery).getValue(); ASSERT_EQ(configChunkDoc.getIntField(ChunkType::estimatedSizeBytes.name()), 2000); @@ -302,42 +270,12 @@ TEST_F(BalancerDefragmentationPolicyTest, nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMoveAndMergeChunks); -} - -TEST_F(BalancerDefragmentationPolicyTest, - TestPhaseOneDataSizeResponsesWithMaxSizeReachedCausesChunkToBeSkippedByPhaseTwo) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - setDefaultClusterStats(); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); - auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); - ASSERT_TRUE(nextAction.has_value()); - DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction); - - auto resp = StatusWith(DataSizeResponse(2000, 4, true)); - _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp); - - // 1. The outcome of the data size has been stored in the expected document... - auto chunkQuery = BSON(ChunkType::collectionUUID() - << kUuid1 << ChunkType::min(kKeyAtMin) << ChunkType::max(kKeyAtMax)); - auto configChunkDoc = - findOneOnConfigCollection(operationContext(), ChunkType::ConfigNS, chunkQuery).getValue(); - ASSERT_EQ(configChunkDoc.getField("estimatedDataSizeBytes").safeNumberLong(), - std::numeric_limits<int64_t>::max()); - - // No new action is expected - and the algorithm should converge - nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); - ASSERT_TRUE(nextAction == boost::none); - ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), boost::none); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMoveAndMergeChunks); } TEST_F(BalancerDefragmentationPolicyTest, TestRetriableFailedDataSizeActionGetsReissued) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); DataSizeInfo failingDataSizeAction = stdx::get<DataSizeInfo>(*nextAction); StatusWith<DataSizeResponse> response( @@ -364,13 +302,12 @@ TEST_F(BalancerDefragmentationPolicyTest, TestRetriableFailedDataSizeActionGetsR } TEST_F(BalancerDefragmentationPolicyTest, TestRemoveCollectionEndsDefragmentation) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction); - auto resp = StatusWith(DataSizeResponse(2000, 4, false)); + auto resp = StatusWith(DataSizeResponse(2000, 4)); _defragmentationPolicy.applyActionResult(operationContext(), dataSizeAction, resp); // Remove collection entry from config.collections @@ -384,51 +321,26 @@ TEST_F(BalancerDefragmentationPolicyTest, TestRemoveCollectionEndsDefragmentatio ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); } -TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneUserCancellationFinishesDefragmentation) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); - - // Collection should be in phase 1 - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMergeAndMeasureChunks); - - // User cancellation of defragmentation - _defragmentationPolicy.abortCollectionDefragmentation(operationContext(), kNss1); - - // Defragmentation should complete since the NoMoreAutoSplitter feature flag is enabled - auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); - ASSERT_TRUE(nextAction == boost::none); - ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), boost::none); -} - TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneUserCancellationBeginsPhase3) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // Collection should be in phase 1 - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMergeAndMeasureChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMergeAndMeasureChunks); // User cancellation of defragmentation - _defragmentationPolicy.abortCollectionDefragmentation(operationContext(), kNss1); + _defragmentationPolicy.abortCollectionDefragmentation(operationContext(), kNss); // Defragmentation should transition to phase 3 auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kSplitChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kSplitChunks); ASSERT_TRUE(nextAction.is_initialized()); auto splitVectorAction = stdx::get<AutoSplitVectorInfo>(*nextAction); } TEST_F(BalancerDefragmentationPolicyTest, TestNonRetriableErrorRebuildsCurrentPhase) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); DataSizeInfo failingDataSizeAction = stdx::get<DataSizeInfo>(*nextAction); StatusWith<DataSizeResponse> response( @@ -439,8 +351,7 @@ TEST_F(BalancerDefragmentationPolicyTest, TestNonRetriableErrorRebuildsCurrentPh // 1. The collection should be marked as undergoing through phase 1 of the algorithm... ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMergeAndMeasureChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMergeAndMeasureChunks); // 2. The action returned by the stream should be now an actionable DataSizeCommand... ASSERT_TRUE(nextAction.is_initialized()); DataSizeInfo dataSizeAction = stdx::get<DataSizeInfo>(*nextAction); @@ -452,12 +363,10 @@ TEST_F(BalancerDefragmentationPolicyTest, TestNonRetriableErrorRebuildsCurrentPh TEST_F(BalancerDefragmentationPolicyTest, TestNonRetriableErrorWaitsForAllOutstandingActionsToComplete) { - setupShards(kShardList); auto coll = setupCollectionWithPhase( - kNss1, - {ChunkType{kUuid1, ChunkRange(kKeyAtMin, kKeyAtTen), kCollectionVersion, kShardId0}, - ChunkType{kUuid1, ChunkRange(kKeyAtTen, kKeyAtMax), kCollectionVersion, kShardId1}}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + {ChunkType{kUuid, ChunkRange(kKeyAtMin, kKeyAtTen), kCollectionVersion, kShardId0}, + ChunkType{kUuid, ChunkRange(BSON("x" << 11), kKeyAtMax), kCollectionVersion, kShardId0}}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); DataSizeInfo failingDataSizeAction = stdx::get<DataSizeInfo>(*nextAction); auto nextAction2 = _defragmentationPolicy.getNextStreamingAction(operationContext()); @@ -488,9 +397,8 @@ TEST_F(BalancerDefragmentationPolicyTest, TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAcknowledgeMergeChunkActionsTriggersDataSizeOnResultingRange) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeMergeableConfigChunkEntries(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase({makeMergeableConfigChunkEntries()}); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto mergeChunksAction = stdx::get<MergeInfo>(*nextAction); @@ -511,9 +419,8 @@ TEST_F(BalancerDefragmentationPolicyTest, } TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneFailedMergeChunksActionGetsReissued) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeMergeableConfigChunkEntries(kUuid1)}); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase(makeMergeableConfigChunkEntries()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto failingMergeChunksAction = stdx::get<MergeInfo>(*nextAction); @@ -538,11 +445,10 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneFailedMergeChunksActionGet } TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAcknowledgeSuccessfulMergeAction) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, {makeMergeableConfigChunkEntries(kUuid1)}); + auto coll = setupCollectionWithPhase(makeMergeableConfigChunkEntries()); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); MergeInfo mergeInfoAction = stdx::get<MergeInfo>(*nextAction); @@ -566,7 +472,7 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAllConsecutive) { const auto minKey = (i == 0) ? kKeyAtMin : BSON("x" << i); const auto maxKey = BSON("x" << i + 1); ChunkType chunk( - kUuid1, + kUuid, ChunkRange(minKey, maxKey), ChunkVersion(1, i, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId0); @@ -576,15 +482,14 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseOneAllConsecutive) { const auto minKey = BSON("x" << i); const auto maxKey = (i == 9) ? kKeyAtMax : BSON("x" << i + 1); ChunkType chunk( - kUuid1, + kUuid, ChunkRange(minKey, maxKey), ChunkVersion(1, i, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId1); chunkList.push_back(chunk); } - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, chunkList, boost::none, boost::none); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase(chunkList, boost::none, boost::none); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // Test auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); @@ -616,15 +521,14 @@ TEST_F(BalancerDefragmentationPolicyTest, PhaseOneNotConsecutive) { const auto maxKey = (i == 9) ? kKeyAtMax : BSON("x" << i + 1); ShardId chosenShard = (i == 5) ? kShardId1 : kShardId0; ChunkType chunk( - kUuid1, + kUuid, ChunkRange(minKey, maxKey), ChunkVersion(1, i, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), chosenShard); chunkList.push_back(chunk); } - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, chunkList, boost::none, boost::none); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + auto coll = setupCollectionWithPhase(chunkList, boost::none, boost::none); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // Three actions (in an unspecified order) should be immediately available. auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); @@ -674,20 +578,18 @@ TEST_F(BalancerDefragmentationPolicyTest, PhaseOneNotConsecutive) { // Phase 2 tests. TEST_F(BalancerDefragmentationPolicyTest, TestPhaseTwoMissingDataSizeRestartsPhase1) { - setupShards(kShardList); - auto coll = setupCollectionWithPhase( - kNss1, {makeConfigChunkEntry(kUuid1)}, DefragmentationPhaseEnum::kMoveAndMergeChunks); + auto coll = setupCollectionWithPhase({makeConfigChunkEntry()}, + DefragmentationPhaseEnum::kMoveAndMergeChunks); setDefaultClusterStats(); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // Should be in phase 1 ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kMergeAndMeasureChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kMergeAndMeasureChunks); // There should be a datasize entry and no migrations - auto availableShards = getAllShardIds(operationContext()); + stdx::unordered_set<ShardId> usedShards; auto pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); + _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); ASSERT_EQ(0, pendingMigrations.size()); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); @@ -696,45 +598,39 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseTwoMissingDataSizeRestartsPha TEST_F(BalancerDefragmentationPolicyTest, TestPhaseTwoChunkCanBeMovedAndMergedWithSibling) { ChunkType biggestChunk( - kUuid1, + kUuid, ChunkRange(kKeyAtMin, kKeyAtZero), ChunkVersion(1, 0, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId0); biggestChunk.setEstimatedSizeBytes(2048); ChunkType smallestChunk( - kUuid1, + kUuid, ChunkRange(kKeyAtZero, kKeyAtMax), ChunkVersion(1, 1, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId1); smallestChunk.setEstimatedSizeBytes(1024); - setupShards(kShardList); - auto coll = setupCollectionWithPhase( - kNss1, {smallestChunk, biggestChunk}, DefragmentationPhaseEnum::kMoveAndMergeChunks); + auto coll = setupCollectionWithPhase({smallestChunk, biggestChunk}, + DefragmentationPhaseEnum::kMoveAndMergeChunks); std::vector<ShardStatistics> clusterStats{buildShardStats(kShardId0, 4), buildShardStats(kShardId1, 2)}; std::map<NamespaceString, std::vector<ShardStatistics>> collectionStats{ - {kNss1, {buildShardStats(kShardId0, 4), buildShardStats(kShardId1, 2)}}}; + {kNss, {buildShardStats(kShardId0, 4), buildShardStats(kShardId1, 2)}}}; _clusterStats.setStats(std::move(clusterStats), std::move(collectionStats)); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - - - auto availableShards = getAllShardIds(operationContext()); - auto numOfShards = availableShards.size(); + stdx::unordered_set<ShardId> usedShards; auto pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); - auto numOfUsedShards = numOfShards - availableShards.size(); + _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); ASSERT_EQ(1, pendingMigrations.size()); - ASSERT_EQ(2, numOfUsedShards); - + ASSERT_EQ(2, usedShards.size()); auto moveAction = pendingMigrations.back(); // The chunk belonging to the "fullest" shard is expected to be moved - even though it is bigger // than its sibling. ASSERT_EQ(biggestChunk.getShard(), moveAction.from); ASSERT_EQ(smallestChunk.getShard(), moveAction.to); ASSERT_BSONOBJ_EQ(biggestChunk.getMin(), moveAction.minKey); - ASSERT_BSONOBJ_EQ(biggestChunk.getMax(), *moveAction.maxKey); + ASSERT_BSONOBJ_EQ(biggestChunk.getMax(), moveAction.maxKey); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); @@ -742,14 +638,10 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseTwoChunkCanBeMovedAndMergedWi _defragmentationPolicy.applyActionResult(operationContext(), moveAction, Status::OK()); nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction.is_initialized()); - - availableShards = getAllShardIds(operationContext()); - numOfShards = availableShards.size(); - pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); - numOfUsedShards = numOfShards - availableShards.size(); + usedShards.clear(); + pendingMigrations = _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); ASSERT_TRUE(pendingMigrations.empty()); - ASSERT_EQ(0, numOfUsedShards); + ASSERT_EQ(0, usedShards.size()); auto mergeAction = stdx::get<MergeInfo>(*nextAction); ASSERT_EQ(smallestChunk.getShard(), mergeAction.shardId); @@ -759,8 +651,7 @@ TEST_F(BalancerDefragmentationPolicyTest, TestPhaseTwoChunkCanBeMovedAndMergedWi _defragmentationPolicy.applyActionResult(operationContext(), mergeAction, Status::OK()); nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); - pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); + pendingMigrations = _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); ASSERT_TRUE(pendingMigrations.empty()); } @@ -769,50 +660,48 @@ TEST_F(BalancerDefragmentationPolicyTest, // Define a single collection, distributing 6 chunks across the 4 shards so that there cannot be // a merge without migrations ChunkType firstChunkOnShard0( - kUuid1, + kUuid, ChunkRange(kKeyAtMin, kKeyAtZero), ChunkVersion(1, 0, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId0); firstChunkOnShard0.setEstimatedSizeBytes(1); ChunkType firstChunkOnShard1( - kUuid1, + kUuid, ChunkRange(kKeyAtZero, kKeyAtTen), ChunkVersion(1, 1, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId1); firstChunkOnShard1.setEstimatedSizeBytes(1); ChunkType chunkOnShard2( - kUuid1, + kUuid, ChunkRange(kKeyAtTen, kKeyAtTwenty), ChunkVersion(1, 2, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId2); chunkOnShard2.setEstimatedSizeBytes(1); ChunkType chunkOnShard3( - kUuid1, + kUuid, ChunkRange(kKeyAtTwenty, kKeyAtThirty), ChunkVersion(1, 3, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId3); chunkOnShard3.setEstimatedSizeBytes(1); ChunkType secondChunkOnShard0( - kUuid1, + kUuid, ChunkRange(kKeyAtThirty, kKeyAtForty), ChunkVersion(1, 4, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId0); secondChunkOnShard0.setEstimatedSizeBytes(1); ChunkType secondChunkOnShard1( - kUuid1, + kUuid, ChunkRange(kKeyAtForty, kKeyAtMax), ChunkVersion(1, 5, kCollectionVersion.epoch(), kCollectionVersion.getTimestamp()), kShardId1); secondChunkOnShard1.setEstimatedSizeBytes(1); - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {firstChunkOnShard0, + auto coll = setupCollectionWithPhase({firstChunkOnShard0, firstChunkOnShard1, chunkOnShard2, chunkOnShard3, @@ -821,16 +710,14 @@ TEST_F(BalancerDefragmentationPolicyTest, DefragmentationPhaseEnum::kMoveAndMergeChunks, boost::none); setDefaultClusterStats(); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // Two move operation should be returned within a single invocation, using all the possible // shards - auto availableShards = getAllShardIds(operationContext()); - auto numOfShards = availableShards.size(); + stdx::unordered_set<ShardId> usedShards; auto pendingMigrations = - _defragmentationPolicy.selectChunksToMove(operationContext(), &availableShards); - auto numOfUsedShards = numOfShards - availableShards.size(); - ASSERT_EQ(4, numOfUsedShards); + _defragmentationPolicy.selectChunksToMove(operationContext(), &usedShards); + ASSERT_EQ(4, usedShards.size()); ASSERT_EQ(2, pendingMigrations.size()); } @@ -841,34 +728,25 @@ TEST_F(BalancerDefragmentationPolicyTest, */ TEST_F(BalancerDefragmentationPolicyTest, DefragmentationBeginsWithPhase3FromPersistedSetting) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); // Defragmentation does not start until startCollectionDefragmentation is called auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); ASSERT_TRUE(nextAction == boost::none); ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), - DefragmentationPhaseEnum::kSplitChunks); + verifyExpectedDefragmentationPhaseOndisk(DefragmentationPhaseEnum::kSplitChunks); } TEST_F(BalancerDefragmentationPolicyTest, SingleLargeChunkCausesAutoSplitAndSplitActions) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); // The new action returned by the stream should be an actionable AutoSplitVector command... nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); @@ -881,16 +759,11 @@ TEST_F(BalancerDefragmentationPolicyTest, SingleLargeChunkCausesAutoSplitAndSpli } TEST_F(BalancerDefragmentationPolicyTest, CollectionMaxChunkSizeIsUsedForPhase3) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; // One chunk > 1KB should trigger AutoSplitVector - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, 2 * 1024)}, - DefragmentationPhaseEnum::kSplitChunks, - 1024); + auto coll = setupCollectionWithPhase( + {makeConfigChunkEntry(2 * 1024)}, DefragmentationPhaseEnum::kSplitChunks, 1024); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); @@ -904,13 +777,9 @@ TEST_F(BalancerDefragmentationPolicyTest, CollectionMaxChunkSizeIsUsedForPhase3) } TEST_F(BalancerDefragmentationPolicyTest, TestRetryableFailedAutoSplitActionGetsReissued) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); AutoSplitVectorInfo failingAutoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); StatusWith<AutoSplitVectorResponse> response( @@ -936,13 +805,9 @@ TEST_F(BalancerDefragmentationPolicyTest, TestRetryableFailedAutoSplitActionGets TEST_F(BalancerDefragmentationPolicyTest, TestAcknowledgeAutoSplitActionTriggersSplitOnResultingRange) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto autoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); @@ -968,13 +833,9 @@ TEST_F(BalancerDefragmentationPolicyTest, } TEST_F(BalancerDefragmentationPolicyTest, TestAutoSplitWithNoSplitPointsDoesNotTriggerSplit) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto autoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); @@ -989,13 +850,9 @@ TEST_F(BalancerDefragmentationPolicyTest, TestAutoSplitWithNoSplitPointsDoesNotT } TEST_F(BalancerDefragmentationPolicyTest, TestMoreThan16MBSplitPointsTriggersSplitAndAutoSplit) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto autoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); @@ -1023,13 +880,9 @@ TEST_F(BalancerDefragmentationPolicyTest, TestMoreThan16MBSplitPointsTriggersSpl } TEST_F(BalancerDefragmentationPolicyTest, TestFailedSplitChunkActionGetsReissued) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto autoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); @@ -1061,13 +914,9 @@ TEST_F(BalancerDefragmentationPolicyTest, TestFailedSplitChunkActionGetsReissued TEST_F(BalancerDefragmentationPolicyTest, TestAcknowledgeLastSuccessfulSplitActionEndsDefragmentation) { - RAIIServerParameterControllerForTest featureFlagNoMoreAutoSplitterOff{ - "featureFlagNoMoreAutoSplitter", false}; - setupShards(kShardList); - auto coll = setupCollectionWithPhase(kNss1, - {makeConfigChunkEntry(kUuid1, kPhase3DefaultChunkSize)}, + auto coll = setupCollectionWithPhase({makeConfigChunkEntry(kPhase3DefaultChunkSize)}, DefragmentationPhaseEnum::kSplitChunks); - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); + _defragmentationPolicy.startCollectionDefragmentation(operationContext(), coll); auto nextAction = _defragmentationPolicy.getNextStreamingAction(operationContext()); auto autoSplitAction = stdx::get<AutoSplitVectorInfo>(*nextAction); @@ -1087,48 +936,7 @@ TEST_F(BalancerDefragmentationPolicyTest, // With phase 3 complete, defragmentation should be completed. ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll.getUuid())); - verifyExpectedDefragmentationStateOnDisk(coll.getUuid(), boost::none); -} - -TEST_F(BalancerDefragmentationPolicyTest, DontStartDefragmentationOnAnyCollection) { - - // Init a collection with defragmentation flag unset - setupShards(kShardList); - setupCollection(kNss1, kShardKeyPattern, {makeConfigChunkEntry(kUuid1)}); - - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); - - verifyExpectedDefragmentationStateOnDisk(kUuid1, boost::none); - ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(kUuid1)); -} - -TEST_F(BalancerDefragmentationPolicyTest, StartDefragmentationOnMultipleCollections) { - - // Setup 3 collections: - // coll1 -> DEFRAGMENTING - // coll2 -> NOT DEFRAGMENTING - // coll3 -> DEFRAGMENTING - - setupShards(kShardList); - auto coll1 = setupCollectionWithPhase(kNss1, {makeConfigChunkEntry(kUuid1)}); - - setupCollection(kNss2, kShardKeyPattern, {makeConfigChunkEntry(kUuid2)}); - auto coll2 = - Grid::get(operationContext())->catalogClient()->getCollection(operationContext(), kUuid2); - - auto coll3 = setupCollectionWithPhase(kNss3, {makeConfigChunkEntry(kUuid3)}); - - _defragmentationPolicy.startCollectionDefragmentations(operationContext()); - - ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll1.getUuid())); - ASSERT_FALSE(_defragmentationPolicy.isDefragmentingCollection(coll2.getUuid())); - ASSERT_TRUE(_defragmentationPolicy.isDefragmentingCollection(coll3.getUuid())); - - verifyExpectedDefragmentationStateOnDisk(kUuid1, - DefragmentationPhaseEnum::kMergeAndMeasureChunks); - verifyExpectedDefragmentationStateOnDisk(kUuid2, boost::none); - verifyExpectedDefragmentationStateOnDisk(kUuid3, - DefragmentationPhaseEnum::kMergeAndMeasureChunks); + verifyExpectedDefragmentationPhaseOndisk(boost::none); } } // namespace diff --git a/src/mongo/db/s/balancer/balancer_policy.cpp b/src/mongo/db/s/balancer/balancer_policy.cpp index d798aa7612e..96619b52c80 100644 --- a/src/mongo/db/s/balancer/balancer_policy.cpp +++ b/src/mongo/db/s/balancer/balancer_policy.cpp @@ -37,7 +37,6 @@ #include "mongo/db/s/balancer/type_migration.h" #include "mongo/logv2/log.h" -#include "mongo/s/balancer_configuration.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/s/grid.h" @@ -61,143 +60,65 @@ namespace { // optimal average across all shards for a zone for a rebalancing migration to be initiated. const size_t kDefaultImbalanceThreshold = 1; -ChunkType makeChunkType(const UUID& collUUID, const Chunk& chunk) { - ChunkType ct{collUUID, chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - ct.setJumbo(chunk.isJumbo()); - return ct; -} - -/** - * Return a vector of zones after they have been normalized according to the given chunk - * configuration. - * - * If a zone covers only partially a chunk, boundaries of that zone will be shrank so that the - * normalized zone won't overlap with that chunk. The boundaries of a normalized zone will never - * fall in the middle of a chunk. - * - * Additionally the vector will contain also zones for the "NoZone", - */ -std::vector<ZoneRange> normalizeZones(const ChunkManager& cm, const ZoneInfo& zoneInfo) { - std::vector<ZoneRange> normalizedRanges; - - auto lastMax = cm.getShardKeyPattern().getKeyPattern().globalMin(); - - for (const auto& [max, zoneRange] : zoneInfo.zoneRanges()) { - const auto& minChunk = cm.findIntersectingChunkWithSimpleCollation(zoneRange.min); - const auto gtMin = - SimpleBSONObjComparator::kInstance.evaluate(zoneRange.min > minChunk.getMin()); - const auto& normalizedMin = gtMin ? minChunk.getMax() : zoneRange.min; - - - const auto& maxChunk = cm.findIntersectingChunkWithSimpleCollation(zoneRange.max); - const auto gtMax = - SimpleBSONObjComparator::kInstance.evaluate(zoneRange.max > maxChunk.getMin()) && - SimpleBSONObjComparator::kInstance.evaluate( - zoneRange.max != cm.getShardKeyPattern().getKeyPattern().globalMax()); - const auto& normalizedMax = gtMax ? maxChunk.getMin() : zoneRange.max; - - - if (SimpleBSONObjComparator::kInstance.evaluate(normalizedMin == normalizedMax)) { - // This normalised zone has a length of zero, therefore can't contain any chunks so we - // can ignore it - continue; - } - - if (SimpleBSONObjComparator::kInstance.evaluate(normalizedMin != lastMax)) { - // The zone is not contiguous with the previous one so we add a kNoZoneRange - // does not fully contain any chunk so we will ignore it - normalizedRanges.emplace_back(lastMax, normalizedMin, ZoneInfo::kNoZoneName); - } - - normalizedRanges.emplace_back(normalizedMin, normalizedMax, zoneRange.zone); - lastMax = normalizedMax; - } - - const auto& globalMaxKey = cm.getShardKeyPattern().getKeyPattern().globalMax(); - if (SimpleBSONObjComparator::kInstance.evaluate(lastMax != globalMaxKey)) { - normalizedRanges.emplace_back(lastMax, globalMaxKey, ZoneInfo::kNoZoneName); - } - return normalizedRanges; -} - } // namespace -DistributionStatus::DistributionStatus(NamespaceString nss, - ZoneInfo zoneInfo, - const ChunkManager& chunkMngr) - : _nss(std::move(nss)), _zoneInfo(std::move(zoneInfo)), _chunkMngr(chunkMngr) { +DistributionStatus::DistributionStatus(NamespaceString nss, ShardToChunksMap shardToChunksMap) + : _nss(std::move(nss)), _shardChunks(std::move(shardToChunksMap)) {} - _normalizedZones = normalizeZones(_chunkMngr, _zoneInfo); - - for (size_t zoneRangeIdx = 0; zoneRangeIdx < _normalizedZones.size(); zoneRangeIdx++) { - const auto& zoneRange = _normalizedZones[zoneRangeIdx]; - chunkMngr.forEachOverlappingChunk( - zoneRange.min, zoneRange.max, false /* isMaxInclusive */, [&](const auto& chunkInfo) { - auto [zoneIt, created] = - _shardZoneInfoMap[chunkInfo.getShardId().toString()].try_emplace( - zoneRange.zone, 1 /* numChunks */, zoneRangeIdx, chunkInfo.getMin()); +size_t DistributionStatus::totalChunks() const { + size_t total = 0; - if (!created) { - ++(zoneIt->second.numChunks); - } - return true; - }); + for (const auto& shardChunk : _shardChunks) { + total += shardChunk.second.size(); } + + return total; } size_t DistributionStatus::totalChunksWithTag(const std::string& tag) const { size_t total = 0; - for (const auto& [_, shardZoneInfo] : _shardZoneInfoMap) { - const auto& zoneIt = shardZoneInfo.find(tag); - if (zoneIt != shardZoneInfo.end()) { - total += zoneIt->second.numChunks; - } + + for (const auto& shardChunk : _shardChunks) { + total += numberOfChunksInShardWithTag(shardChunk.first, tag); } + return total; } size_t DistributionStatus::numberOfChunksInShard(const ShardId& shardId) const { - const auto shardZonesIt = _shardZoneInfoMap.find(shardId.toString()); - if (shardZonesIt == _shardZoneInfoMap.end()) { - return 0; - } - size_t total = 0; - for (const auto& [_, shardZoneInfo] : shardZonesIt->second) { - total += shardZoneInfo.numChunks; - } - return total; + const auto& shardChunks = getChunks(shardId); + return shardChunks.size(); } size_t DistributionStatus::numberOfChunksInShardWithTag(const ShardId& shardId, const string& tag) const { - const auto shardZonesIt = _shardZoneInfoMap.find(shardId.toString()); - if (shardZonesIt == _shardZoneInfoMap.end()) { - return 0; - } - const auto& shardTags = shardZonesIt->second; + const auto& shardChunks = getChunks(shardId); + + size_t total = 0; - const auto& zoneIt = shardTags.find(tag); - if (zoneIt == shardTags.end()) { - return 0; + for (const auto& chunk : shardChunks) { + if (tag == getTagForChunk(chunk)) { + total++; + } } - return zoneIt->second.numChunks; + + return total; } -string DistributionStatus::getTagForRange(const ChunkRange& range) const { - return _zoneInfo.getZoneForChunk(range); +const vector<ChunkType>& DistributionStatus::getChunks(const ShardId& shardId) const { + ShardToChunksMap::const_iterator i = _shardChunks.find(shardId); + invariant(i != _shardChunks.end()); + + return i->second; } -const StringMap<ShardZoneInfo>& DistributionStatus::getZoneInfoForShard( - const ShardId& shardId) const { - static const StringMap<ShardZoneInfo> emptyMap; - const auto shardZonesIt = _shardZoneInfoMap.find(shardId.toString()); - if (shardZonesIt == _shardZoneInfoMap.end()) { - return emptyMap; - } - return shardZonesIt->second; +Status DistributionStatus::addRangeToZone(const ZoneRange& range) { + return _zoneInfo.addRangeToZone(range); } -const string ZoneInfo::kNoZoneName = ""; +string DistributionStatus::getTagForChunk(const ChunkType& chunk) const { + return _zoneInfo.getZoneForChunk(chunk.getRange()); +} ZoneInfo::ZoneInfo() : _zoneRanges(SimpleBSONObjComparator::kInstance.makeBSONObjIndexedMap<ZoneRange>()) {} @@ -249,11 +170,11 @@ string ZoneInfo::getZoneForChunk(const ChunkRange& chunk) const { // We should never have a partial overlap with a chunk range. If it happens, treat it as if this // chunk doesn't belong to a tag if (minIntersect != maxIntersect) { - return ZoneInfo::kNoZoneName; + return ""; } if (minIntersect == _zoneRanges.end()) { - return ZoneInfo::kNoZoneName; + return ""; } const ZoneRange& intersectRange = minIntersect->second; @@ -264,16 +185,17 @@ string ZoneInfo::getZoneForChunk(const ChunkRange& chunk) const { return intersectRange.zone; } - return ZoneInfo::kNoZoneName; + return ""; } + /** * read all tags for collection via the catalog client and add to the zoneInfo */ -StatusWith<ZoneInfo> createCollectionZoneInfo(OperationContext* opCtx, - const NamespaceString& nss, - const KeyPattern& keyPattern) { - ZoneInfo zoneInfo; +Status ZoneInfo::addTagsFromCatalog(OperationContext* opCtx, + const NamespaceString& nss, + const KeyPattern& keyPattern, + ZoneInfo& chunkMgr) { const auto swCollectionTags = Grid::get(opCtx)->catalogClient()->getTagsForCollection(opCtx, nss); if (!swCollectionTags.isOK()) { @@ -284,14 +206,16 @@ StatusWith<ZoneInfo> createCollectionZoneInfo(OperationContext* opCtx, for (const auto& tag : collectionTags) { auto status = - zoneInfo.addRangeToZone(ZoneRange(keyPattern.extendRangeBound(tag.getMinKey(), false), + chunkMgr.addRangeToZone(ZoneRange(keyPattern.extendRangeBound(tag.getMinKey(), false), keyPattern.extendRangeBound(tag.getMaxKey(), false), tag.getTag())); + if (!status.isOK()) { return status; } } - return {std::move(zoneInfo)}; + + return Status::OK(); } void DistributionStatus::report(BSONObjBuilder* builder) const { @@ -299,15 +223,15 @@ void DistributionStatus::report(BSONObjBuilder* builder) const { // Report all shards BSONArrayBuilder shardArr(builder->subarrayStart("shards")); - for (const auto& [shardId, zoneInfoMap] : _shardZoneInfoMap) { + for (const auto& shardChunk : _shardChunks) { BSONObjBuilder shardEntry(shardArr.subobjStart()); - shardEntry.append("name", shardId); + shardEntry.append("name", shardChunk.first.toString()); - BSONObjBuilder tagsObj(shardEntry.subobjStart("tags")); - for (const auto& [tagName, shardZoneInfo] : zoneInfoMap) { - tagsObj.appendNumber(tagName, static_cast<long long>(shardZoneInfo.numChunks)); + BSONArrayBuilder chunkArr(shardEntry.subarrayStart("chunks")); + for (const auto& chunk : shardChunk.second) { + chunkArr.append(chunk.toConfigBSON()); } - tagsObj.doneFast(); + chunkArr.doneFast(); shardEntry.doneFast(); } @@ -350,7 +274,7 @@ Status BalancerPolicy::isShardSuitableReceiver(const ClusterStatistics::ShardSta str::stream() << stat.shardId << " is currently draining."}; } - if (chunkTag != ZoneInfo::kNoZoneName && !stat.shardTags.count(chunkTag)) { + if (!chunkTag.empty() && !stat.shardTags.count(chunkTag)) { return {ErrorCodes::IllegalOperation, str::stream() << stat.shardId << " is not in the correct zone " << chunkTag}; } @@ -358,19 +282,16 @@ Status BalancerPolicy::isShardSuitableReceiver(const ClusterStatistics::ShardSta return Status::OK(); } -std::tuple<ShardId, int64_t> BalancerPolicy::_getLeastLoadedReceiverShard( +ShardId BalancerPolicy::_getLeastLoadedReceiverShard( const ShardStatisticsVector& shardStats, const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, const string& tag, - const stdx::unordered_set<ShardId>& availableShards) { + const stdx::unordered_set<ShardId>& excludedShards) { ShardId best; - int64_t currentMin = numeric_limits<int64_t>::max(); - - const auto shouldBalanceAccordingToDataSize = collDataSizeInfo.has_value(); + unsigned minChunks = numeric_limits<unsigned>::max(); for (const auto& stat : shardStats) { - if (!availableShards.count(stat.shardId)) + if (excludedShards.count(stat.shardId)) continue; auto status = isShardSuitableReceiver(stat, tag); @@ -378,68 +299,40 @@ std::tuple<ShardId, int64_t> BalancerPolicy::_getLeastLoadedReceiverShard( continue; } - if (shouldBalanceAccordingToDataSize) { - const auto& shardSizeIt = collDataSizeInfo->shardToDataSizeMap.find(stat.shardId); - if (shardSizeIt == collDataSizeInfo->shardToDataSizeMap.end()) { - // Skip if stats not available (may happen if add|remove shard during a round) - continue; - } - - int64_t shardSize = shardSizeIt->second; - if (shardSize < currentMin) { - best = stat.shardId; - currentMin = shardSize; - } - } else { - int64_t myChunks = distribution.numberOfChunksInShard(stat.shardId); - if (myChunks < currentMin) { - best = stat.shardId; - currentMin = myChunks; - } + unsigned myChunks = distribution.numberOfChunksInShard(stat.shardId); + if (myChunks >= minChunks) { + continue; } + + best = stat.shardId; + minChunks = myChunks; } - return {best, currentMin}; + return best; } -std::tuple<ShardId, int64_t> BalancerPolicy::_getMostOverloadedShard( +ShardId BalancerPolicy::_getMostOverloadedShard( const ShardStatisticsVector& shardStats, const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, const string& chunkTag, - const stdx::unordered_set<ShardId>& availableShards) { + const stdx::unordered_set<ShardId>& excludedShards) { ShardId worst; - long long currentMax = numeric_limits<long long>::min(); - - const auto shouldBalanceAccordingToDataSize = collDataSizeInfo.has_value(); + unsigned maxChunks = 0; for (const auto& stat : shardStats) { - if (!availableShards.count(stat.shardId)) + if (excludedShards.count(stat.shardId)) continue; - if (shouldBalanceAccordingToDataSize) { - const auto& shardSizeIt = collDataSizeInfo->shardToDataSizeMap.find(stat.shardId); - if (shardSizeIt == collDataSizeInfo->shardToDataSizeMap.end()) { - // Skip if stats not available (may happen if add|remove shard during a round) - continue; - } + const unsigned shardChunkCount = + distribution.numberOfChunksInShardWithTag(stat.shardId, chunkTag); + if (shardChunkCount <= maxChunks) + continue; - const auto shardSize = shardSizeIt->second; - if (shardSize > currentMax) { - worst = stat.shardId; - currentMax = shardSize; - } - } else { - const unsigned shardChunkCount = - distribution.numberOfChunksInShardWithTag(stat.shardId, chunkTag); - if (shardChunkCount > currentMax) { - worst = stat.shardId; - currentMax = shardChunkCount; - } - } + worst = stat.shardId; + maxChunks = shardChunkCount; } - return {worst, currentMax}; + return worst; } // Returns a random integer in [0, max) using a uniform random distribution. @@ -499,34 +392,18 @@ MigrateInfo chooseRandomMigration(const ShardStatisticsVector& shardStats, "fromShardId"_attr = sourceShardId, "toShardId"_attr = destShardId); - const auto& randomChunk = [&] { - const auto numChunksOnSourceShard = distribution.numberOfChunksInShard(sourceShardId); - const auto rndChunkIdx = getRandomIndex(numChunksOnSourceShard); - ChunkType rndChunk; - - int idx{0}; - distribution.getChunkManager().forEachChunk([&](const auto& chunk) { - if (chunk.getShardId() == sourceShardId && idx++ == rndChunkIdx) { - rndChunk = makeChunkType(distribution.getChunkManager().getUUID(), chunk); - return false; - } - return true; - }); - - invariant(rndChunk.getShard().isValid()); - return rndChunk; - }(); + const auto& chunks = distribution.getChunks(sourceShardId); - return { - destShardId, distribution.nss(), randomChunk, MoveChunkRequest::ForceJumbo::kDoNotForce}; + return {destShardId, + distribution.nss(), + chunks[getRandomIndex(chunks.size())], + MoveChunkRequest::ForceJumbo::kDoNotForce}; } -MigrateInfosWithReason BalancerPolicy::balance( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - stdx::unordered_set<ShardId>* availableShards, - bool forceJumbo) { +MigrateInfosWithReason BalancerPolicy::balance(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + stdx::unordered_set<ShardId>* usedShards, + bool forceJumbo) { vector<MigrateInfo> migrations; MigrationReason firstReason = MigrationReason::none; @@ -549,84 +426,50 @@ MigrateInfosWithReason BalancerPolicy::balance( if (!stat.isDraining) continue; - if (!availableShards->count(stat.shardId)) + if (usedShards->count(stat.shardId)) continue; - // Now we know we need to move chunks off this shard, but only if permitted by the + const vector<ChunkType>& chunks = distribution.getChunks(stat.shardId); + + if (chunks.empty()) + continue; + + // Now we know we need to move to chunks off this shard, but only if permitted by the // tags policy unsigned numJumboChunks = 0; - const auto& shardZones = distribution.getZoneInfoForShard(stat.shardId); - for (const auto& shardZone : shardZones) { - const auto& zoneName = shardZone.first; - - const auto chunkFoundForShard = !distribution.forEachChunkOnShardInZone( - stat.shardId, zoneName, [&](const auto& chunk) { - if (chunk.isJumbo()) { - numJumboChunks++; - return true; // continue - } - - const auto [to, _] = _getLeastLoadedReceiverShard( - shardStats, distribution, collDataSizeInfo, zoneName, *availableShards); - if (!to.isValid()) { - if (migrations.empty()) { - LOGV2_DEBUG( - 21889, - 3, - "Chunk {chunk} is on a draining shard, but no appropriate " - "recipient found", - "Chunk is on a draining shard, but no appropriate " - "recipient found", - "chunk"_attr = - redact(makeChunkType( - distribution.getChunkManager().getUUID(), chunk) - .toString())); - } - return true; // continue - } - invariant(to != stat.shardId); - - auto maxChunkSizeBytes = [&]() -> boost::optional<int64_t> { - if (collDataSizeInfo.has_value()) { - return collDataSizeInfo->maxChunkSizeBytes; - } - return boost::none; - }(); - - if (collDataSizeInfo.has_value()) { - migrations.emplace_back( - to, - chunk.getShardId(), - distribution.nss(), - distribution.getChunkManager().getUUID(), - chunk.getMin(), - boost::none /* max */, - chunk.getLastmod(), - // Always force jumbo chunks to be migrated off draining shards - MoveChunkRequest::ForceJumbo::kForceBalancer, - maxChunkSizeBytes); - } else { - migrations.emplace_back( - to, - distribution.nss(), - makeChunkType(distribution.getChunkManager().getUUID(), chunk), - MoveChunkRequest::ForceJumbo::kForceBalancer, - maxChunkSizeBytes); - } - - if (firstReason == MigrationReason::none) { - firstReason = MigrationReason::drain; - } - - invariant(availableShards->erase(stat.shardId)); - invariant(availableShards->erase(to)); - return false; // break - }); - - if (chunkFoundForShard) { - break; + // Since we have to move all chunks, lets just do in order + for (const auto& chunk : chunks) { + if (chunk.getJumbo()) { + numJumboChunks++; + continue; } + + const string tag = distribution.getTagForChunk(chunk); + + const ShardId to = + _getLeastLoadedReceiverShard(shardStats, distribution, tag, *usedShards); + if (!to.isValid()) { + if (migrations.empty()) { + LOGV2_WARNING(21889, + "Chunk {chunk} is on a draining shard, but no appropriate " + "recipient found", + "Chunk is on a draining shard, but no appropriate " + "recipient found", + "chunk"_attr = redact(chunk.toString())); + } + continue; + } + + invariant(to != stat.shardId); + migrations.emplace_back( + to, distribution.nss(), chunk, MoveChunkRequest::ForceJumbo::kForceBalancer); + if (firstReason == MigrationReason::none) { + firstReason = MigrationReason::drain; + } + invariant(usedShards->insert(stat.shardId).second); + invariant(usedShards->insert(to).second); + break; } if (migrations.empty()) { @@ -637,110 +480,62 @@ MigrateInfosWithReason BalancerPolicy::balance( "shardId"_attr = stat.shardId, "numJumboChunks"_attr = numJumboChunks); } - - if (availableShards->size() < 2) { - return std::make_pair(std::move(migrations), firstReason); - } } } // 2) Check for chunks, which are on the wrong shard and must be moved off of it if (!distribution.tags().empty()) { for (const auto& stat : shardStats) { - - if (!availableShards->count(stat.shardId)) + if (usedShards->count(stat.shardId)) continue; - const auto& shardZones = distribution.getZoneInfoForShard(stat.shardId); - for (const auto& shardZone : shardZones) { - const auto& zoneName = shardZone.first; + const vector<ChunkType>& chunks = distribution.getChunks(stat.shardId); + + for (const auto& chunk : chunks) { + const string tag = distribution.getTagForChunk(chunk); + + if (tag.empty()) + continue; - if (zoneName == ZoneInfo::kNoZoneName) + if (stat.shardTags.count(tag)) continue; - if (stat.shardTags.count(zoneName)) + if (chunk.getJumbo()) { + LOGV2_WARNING( + 21891, + "Chunk {chunk} violates zone {zone}, but it is jumbo and cannot be moved", + "Chunk violates zone, but it is jumbo and cannot be moved", + "chunk"_attr = redact(chunk.toString()), + "zone"_attr = redact(tag)); continue; + } - const auto chunkFoundForShard = !distribution.forEachChunkOnShardInZone( - stat.shardId, zoneName, [&](const auto& chunk) { - if (chunk.isJumbo()) { - LOGV2_WARNING( - 21891, - "Chunk {chunk} violates zone {zone}, but it is jumbo and " - "cannot be " - "moved", - "Chunk violates zone, but it is jumbo and cannot be moved", - "chunk"_attr = redact( - makeChunkType(distribution.getChunkManager().getUUID(), chunk) - .toString()), - "zone"_attr = redact(zoneName)); - return true; // continue - } - - const auto [to, _] = _getLeastLoadedReceiverShard( - shardStats, distribution, collDataSizeInfo, zoneName, *availableShards); - if (!to.isValid()) { - if (migrations.empty()) { - LOGV2_DEBUG( - 21892, - 3, - "Chunk {chunk} violates zone {zone}, but no appropriate " - "recipient found", - "Chunk violates zone, but no appropriate recipient found", - "chunk"_attr = - redact(makeChunkType( - distribution.getChunkManager().getUUID(), chunk) - .toString()), - "zone"_attr = redact(zoneName)); - } - return true; // continue - } - invariant(to != stat.shardId); - - auto maxChunkSizeBytes = [&]() -> boost::optional<int64_t> { - if (collDataSizeInfo.has_value()) { - return collDataSizeInfo->maxChunkSizeBytes; - } - return boost::none; - }(); - - if (collDataSizeInfo.has_value()) { - migrations.emplace_back( - to, - chunk.getShardId(), - distribution.nss(), - distribution.getChunkManager().getUUID(), - chunk.getMin(), - boost::none /* max */, - chunk.getLastmod(), - forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer - : MoveChunkRequest::ForceJumbo::kDoNotForce, - maxChunkSizeBytes); - } else { - migrations.emplace_back( - to, - distribution.nss(), - makeChunkType(distribution.getChunkManager().getUUID(), chunk), - forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer - : MoveChunkRequest::ForceJumbo::kDoNotForce, - maxChunkSizeBytes); - } - - if (firstReason == MigrationReason::none) { - firstReason = MigrationReason::zoneViolation; - } - - invariant(availableShards->erase(stat.shardId)); - invariant(availableShards->erase(to)); - return false; // break - }); - - if (chunkFoundForShard) { - break; + const ShardId to = + _getLeastLoadedReceiverShard(shardStats, distribution, tag, *usedShards); + if (!to.isValid()) { + if (migrations.empty()) { + LOGV2_WARNING(21892, + "Chunk {chunk} violates zone {zone}, but no appropriate " + "recipient found", + "Chunk violates zone, but no appropriate recipient found", + "chunk"_attr = redact(chunk.toString()), + "zone"_attr = redact(tag)); + } + continue; } - } - if (availableShards->size() < 2) { - return std::make_pair(std::move(migrations), firstReason); + + invariant(to != stat.shardId); + migrations.emplace_back(to, + distribution.nss(), + chunk, + forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer + : MoveChunkRequest::ForceJumbo::kDoNotForce); + if (firstReason == MigrationReason::none) { + firstReason = MigrationReason::zoneViolation; + } + invariant(usedShards->insert(stat.shardId).second); + invariant(usedShards->insert(to).second); + break; } } } @@ -748,32 +543,24 @@ MigrateInfosWithReason BalancerPolicy::balance( // 3) for each tag balance vector<string> tagsPlusEmpty(distribution.tags().begin(), distribution.tags().end()); - tagsPlusEmpty.push_back(ZoneInfo::kNoZoneName); + tagsPlusEmpty.push_back(""); for (const auto& tag : tagsPlusEmpty) { + const size_t totalNumberOfChunksWithTag = + (tag.empty() ? distribution.totalChunks() : distribution.totalChunksWithTag(tag)); + size_t totalNumberOfShardsWithTag = 0; - int64_t totalDataSizeOfShardsWithZone = 0; for (const auto& stat : shardStats) { - if (tag == ZoneInfo::kNoZoneName || stat.shardTags.count(tag)) { + if (tag.empty() || stat.shardTags.count(tag)) { totalNumberOfShardsWithTag++; - if (collDataSizeInfo.has_value()) { - const auto& shardSizeIt = - collDataSizeInfo->shardToDataSizeMap.find(stat.shardId); - if (shardSizeIt == collDataSizeInfo->shardToDataSizeMap.end()) { - // Skip if stats not available (may happen if add|remove shard during a - // round) - continue; - } - totalDataSizeOfShardsWithZone += shardSizeIt->second; - } } } // Skip zones which have no shards assigned to them. This situation is not harmful, but // should not be possible so warn the operator to correct it. if (totalNumberOfShardsWithTag == 0) { - if (tag != ZoneInfo::kNoZoneName) { + if (!tag.empty()) { LOGV2_WARNING( 21893, "Zone {zone} in collection {namespace} has no assigned shards and chunks " @@ -788,47 +575,18 @@ MigrateInfosWithReason BalancerPolicy::balance( continue; } - const int64_t idealDataSizePerShardForZone = - totalDataSizeOfShardsWithZone / totalNumberOfShardsWithTag; - - auto singleZoneBalance = [&]() { - if (collDataSizeInfo.has_value()) { - tassert(ErrorCodes::BadValue, - str::stream() - << "Total data size for shards in zone " << tag << " and collection " - << distribution.nss() << " must be greater or equal than zero but is " - << totalDataSizeOfShardsWithZone, - totalDataSizeOfShardsWithZone >= 0); - - if (totalDataSizeOfShardsWithZone == 0) { - // No data to balance within this zone - return false; - } - - return _singleZoneBalanceBasedOnDataSize( - shardStats, - distribution, - *collDataSizeInfo, - tag, - idealDataSizePerShardForZone, - &migrations, - availableShards, - forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer - : MoveChunkRequest::ForceJumbo::kDoNotForce); - } - - return _singleZoneBalanceBasedOnChunks( - shardStats, - distribution, - tag, - totalNumberOfShardsWithTag, - &migrations, - availableShards, - forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer - : MoveChunkRequest::ForceJumbo::kDoNotForce); - }; - - while (singleZoneBalance()) { + // Calculate the rounded optimal number of chunks per shard + const size_t idealNumberOfChunksPerShardForTag = + (size_t)std::roundf(totalNumberOfChunksWithTag / (float)totalNumberOfShardsWithTag); + + while (_singleZoneBalance(shardStats, + distribution, + tag, + idealNumberOfChunksPerShardForTag, + &migrations, + usedShards, + forceJumbo ? MoveChunkRequest::ForceJumbo::kForceBalancer + : MoveChunkRequest::ForceJumbo::kDoNotForce)) { if (firstReason == MigrationReason::none) { firstReason = MigrationReason::chunksImbalance; } @@ -842,18 +600,10 @@ boost::optional<MigrateInfo> BalancerPolicy::balanceSingleChunk( const ChunkType& chunk, const ShardStatisticsVector& shardStats, const DistributionStatus& distribution) { - const string tag = distribution.getTagForRange(chunk.getRange()); - - stdx::unordered_set<ShardId> availableShards; - std::transform(shardStats.begin(), - shardStats.end(), - std::inserter(availableShards, availableShards.end()), - [](const ClusterStatistics::ShardStatistics& shardStatistics) -> ShardId { - return shardStatistics.shardId; - }); - - const auto [newShardId, _] = _getLeastLoadedReceiverShard( - shardStats, distribution, boost::none /* collDataSizeInfo */, tag, availableShards); + const string tag = distribution.getTagForChunk(chunk); + + ShardId newShardId = + _getLeastLoadedReceiverShard(shardStats, distribution, tag, stdx::unordered_set<ShardId>()); if (!newShardId.isValid() || newShardId == chunk.getShard()) { return boost::optional<MigrateInfo>(); } @@ -862,25 +612,14 @@ boost::optional<MigrateInfo> BalancerPolicy::balanceSingleChunk( newShardId, distribution.nss(), chunk, MoveChunkRequest::ForceJumbo::kDoNotForce); } -bool BalancerPolicy::_singleZoneBalanceBasedOnChunks(const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const string& tag, - size_t totalNumberOfShardsWithTag, - vector<MigrateInfo>* migrations, - stdx::unordered_set<ShardId>* availableShards, - MoveChunkRequest::ForceJumbo forceJumbo) { - const auto totalNumberOfChunksWithTag = [&] { - if (tag == ZoneInfo::kNoZoneName) { - return static_cast<size_t>(distribution.getChunkManager().numChunks()); - } - return distribution.totalChunksWithTag(tag); - }(); - - const size_t idealNumberOfChunksPerShardForTag = - (size_t)std::roundf(totalNumberOfChunksWithTag / (float)totalNumberOfShardsWithTag); - - const auto [from, fromSize] = - _getMostOverloadedShard(shardStats, distribution, boost::none, tag, *availableShards); +bool BalancerPolicy::_singleZoneBalance(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + const string& tag, + size_t idealNumberOfChunksPerShardForTag, + vector<MigrateInfo>* migrations, + stdx::unordered_set<ShardId>* usedShards, + MoveChunkRequest::ForceJumbo forceJumbo) { + const ShardId from = _getMostOverloadedShard(shardStats, distribution, tag, *usedShards); if (!from.isValid()) return false; @@ -890,11 +629,13 @@ bool BalancerPolicy::_singleZoneBalanceBasedOnChunks(const ShardStatisticsVector if (max <= idealNumberOfChunksPerShardForTag) return false; - const auto [to, toSize] = - _getLeastLoadedReceiverShard(shardStats, distribution, boost::none, tag, *availableShards); + const ShardId to = _getLeastLoadedReceiverShard(shardStats, distribution, tag, *usedShards); if (!to.isValid()) { if (migrations->empty()) { - LOGV2(21882, "No available shards to take chunks for zone", "zone"_attr = tag); + LOGV2(21882, + "No available shards to take chunks for zone {zone}", + "No available shards to take chunks for zone", + "zone"_attr = tag); } return false; } @@ -927,34 +668,26 @@ bool BalancerPolicy::_singleZoneBalanceBasedOnChunks(const ShardStatisticsVector if (imbalance < kDefaultImbalanceThreshold) return false; - - const auto& fromShardId = from; - const auto& toShardId = to; + const vector<ChunkType>& chunks = distribution.getChunks(from); unsigned numJumboChunks = 0; - const auto chunkFound = - !distribution.forEachChunkOnShardInZone(fromShardId, tag, [&](const auto& chunk) { - if (chunk.isJumbo()) { - numJumboChunks++; - return true; // continue - } + for (const auto& chunk : chunks) { + if (distribution.getTagForChunk(chunk) != tag) + continue; - migrations->emplace_back(toShardId, - distribution.nss(), - makeChunkType(distribution.getChunkManager().getUUID(), chunk), - forceJumbo); - invariant(availableShards->erase(chunk.getShardId())); - invariant(availableShards->erase(toShardId)); - return false; // break - }); - - tassert(8236500, - "Expected to find at least one chunk for shard '{}' in zone '{}'"_format( - fromShardId.toString(), tag), - chunkFound || numJumboChunks); - - if (!chunkFound && numJumboChunks) { + if (chunk.getJumbo()) { + numJumboChunks++; + continue; + } + + migrations->emplace_back(to, distribution.nss(), chunk, forceJumbo); + invariant(usedShards->insert(chunk.getShard()).second); + invariant(usedShards->insert(to).second); + return true; + } + + if (numJumboChunks) { LOGV2_WARNING( 21894, "Shard: {shardId}, collection: {namespace} has only jumbo chunks for " @@ -966,99 +699,7 @@ bool BalancerPolicy::_singleZoneBalanceBasedOnChunks(const ShardStatisticsVector "numJumboChunks"_attr = numJumboChunks); } - return chunkFound; -} - -bool BalancerPolicy::_singleZoneBalanceBasedOnDataSize( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const CollectionDataSizeInfoForBalancing& collDataSizeInfo, - const string& tag, - const int64_t idealDataSizePerShardForZone, - vector<MigrateInfo>* migrations, - stdx::unordered_set<ShardId>* availableShards, - MoveChunkRequest::ForceJumbo forceJumbo) { - const auto [from, fromSize] = - _getMostOverloadedShard(shardStats, distribution, collDataSizeInfo, tag, *availableShards); - if (!from.isValid()) - return false; - - const auto [to, toSize] = _getLeastLoadedReceiverShard( - shardStats, distribution, collDataSizeInfo, tag, *availableShards); - if (!to.isValid()) { - if (migrations->empty()) { - LOGV2(6581600, "No available shards to take chunks for zone", "zone"_attr = tag); - } - return false; - } - - if (from == to) { - return false; - } - - LOGV2_DEBUG(7548100, - 1, - "Balancing single zone", - "namespace"_attr = distribution.nss().ns(), - "zone"_attr = tag, - "idealDataSizePerShardForZone"_attr = idealDataSizePerShardForZone, - "fromShardId"_attr = from, - "fromShardDataSize"_attr = fromSize, - "toShardId"_attr = to, - "toShardDataSize"_attr = toSize, - "maxChunkSizeBytes"_attr = collDataSizeInfo.maxChunkSizeBytes); - - if (fromSize <= idealDataSizePerShardForZone) { - return false; - } - - if (toSize >= idealDataSizePerShardForZone) { - // Do not use a shard if it already has more data than the ideal per-shard size - return false; - } - - if (fromSize - toSize < 3 * collDataSizeInfo.maxChunkSizeBytes) { - // Do not balance if the collection's size differs too few between the chosen shards - return false; - } - - - const auto& fromShardId = from; - const auto& toShardId = to; - - unsigned numJumboChunks = 0; - - const auto chunkFound = - !distribution.forEachChunkOnShardInZone(fromShardId, tag, [&](const auto& chunk) { - if (chunk.isJumbo()) { - numJumboChunks++; - return true; // continue - } - - migrations->emplace_back(toShardId, - chunk.getShardId(), - distribution.nss(), - distribution.getChunkManager().getUUID(), - chunk.getMin(), - boost::none /* max */, - chunk.getLastmod(), - forceJumbo, - collDataSizeInfo.maxChunkSizeBytes); - invariant(availableShards->erase(chunk.getShardId())); - invariant(availableShards->erase(toShardId)); - return false; // break - }); - - if (!chunkFound && numJumboChunks) { - LOGV2_WARNING(6581602, - "Shard has only jumbo chunks for this collection and cannot be balanced", - "namespace"_attr = distribution.nss().ns(), - "shardId"_attr = from, - "zone"_attr = tag, - "numJumboChunks"_attr = numJumboChunks); - } - - return chunkFound; + return false; } ZoneRange::ZoneRange(const BSONObj& a_min, const BSONObj& a_max, const std::string& _zone) @@ -1071,8 +712,7 @@ string ZoneRange::toString() const { MigrateInfo::MigrateInfo(const ShardId& a_to, const NamespaceString& a_nss, const ChunkType& a_chunk, - const MoveChunkRequest::ForceJumbo a_forceJumbo, - boost::optional<int64_t> maxChunkSizeBytes) + const MoveChunkRequest::ForceJumbo a_forceJumbo) : nss(a_nss), uuid(a_chunk.getCollectionUUID()) { invariant(a_to.isValid()); @@ -1083,7 +723,6 @@ MigrateInfo::MigrateInfo(const ShardId& a_to, maxKey = a_chunk.getMax(); version = a_chunk.getVersion(); forceJumbo = a_forceJumbo; - optMaxChunkSizeBytes = maxChunkSizeBytes; } MigrateInfo::MigrateInfo(const ShardId& a_to, @@ -1091,17 +730,15 @@ MigrateInfo::MigrateInfo(const ShardId& a_to, const NamespaceString& a_nss, const UUID& a_uuid, const BSONObj& a_min, - const boost::optional<BSONObj>& a_max, + const BSONObj& a_max, const ChunkVersion& a_version, - const MoveChunkRequest::ForceJumbo a_forceJumbo, - boost::optional<int64_t> maxChunkSizeBytes) + const MoveChunkRequest::ForceJumbo a_forceJumbo) : nss(a_nss), uuid(a_uuid), minKey(a_min), maxKey(a_max), version(a_version), - forceJumbo(a_forceJumbo), - optMaxChunkSizeBytes(maxChunkSizeBytes) { + forceJumbo(a_forceJumbo) { invariant(a_to.isValid()); invariant(a_from.isValid()); @@ -1135,10 +772,6 @@ string MigrateInfo::toString() const { << ", to " << to; } -boost::optional<int64_t> MigrateInfo::getMaxChunkSizeBytes() const { - return optMaxChunkSizeBytes; -} - SplitInfo::SplitInfo(const ShardId& inShardId, const NamespaceString& inNss, const ChunkVersion& inCollectionVersion, @@ -1222,15 +855,13 @@ DataSizeInfo::DataSizeInfo(const ShardId& shardId, const ChunkRange& chunkRange, const ChunkVersion& version, const KeyPattern& keyPattern, - bool estimatedValue, - int64_t maxSize) + bool estimatedValue) : shardId(shardId), nss(nss), uuid(uuid), chunkRange(chunkRange), version(version), keyPattern(keyPattern), - estimatedValue(estimatedValue), - maxSize(maxSize) {} + estimatedValue(estimatedValue) {} } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_policy.h b/src/mongo/db/s/balancer/balancer_policy.h index 51f079bf91c..008c1d93b7e 100644 --- a/src/mongo/db/s/balancer/balancer_policy.h +++ b/src/mongo/db/s/balancer/balancer_policy.h @@ -38,13 +38,13 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/s/balancer/cluster_statistics.h" #include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/chunk_manager.h" #include "mongo/s/request_types/auto_split_vector_gen.h" #include "mongo/s/request_types/move_chunk_request.h" #include "mongo/s/shard_id.h" namespace mongo { + struct ZoneRange { ZoneRange(const BSONObj& a_min, const BSONObj& a_max, const std::string& _zone); @@ -59,18 +59,16 @@ struct MigrateInfo { MigrateInfo(const ShardId& a_to, const NamespaceString& a_nss, const ChunkType& a_chunk, - MoveChunkRequest::ForceJumbo a_forceJumbo, - boost::optional<int64_t> maxChunkSizeBytes = boost::none); + MoveChunkRequest::ForceJumbo a_forceJumbo); MigrateInfo(const ShardId& a_to, const ShardId& a_from, const NamespaceString& a_nss, const UUID& a_uuid, const BSONObj& a_min, - const boost::optional<BSONObj>& a_max, + const BSONObj& a_max, const ChunkVersion& a_version, - MoveChunkRequest::ForceJumbo a_forceJumbo, - boost::optional<int64_t> maxChunkSizeBytes = boost::none); + MoveChunkRequest::ForceJumbo a_forceJumbo); std::string getName() const; @@ -78,22 +76,14 @@ struct MigrateInfo { std::string toString() const; - boost::optional<int64_t> getMaxChunkSizeBytes() const; - NamespaceString nss; UUID uuid; ShardId to; ShardId from; BSONObj minKey; - - // May be optional in case of moveRange - boost::optional<BSONObj> maxKey; + BSONObj maxKey; ChunkVersion version; MoveChunkRequest::ForceJumbo forceJumbo; - - // Set only in case of data-size aware balancing - // TODO SERVER-65322 make `optMaxChunkSizeBytes` non-optional - boost::optional<int64_t> optMaxChunkSizeBytes; }; enum MigrationReason { none, drain, zoneViolation, chunksImbalance }; @@ -186,8 +176,7 @@ struct DataSizeInfo { const ChunkRange& chunkRange, const ChunkVersion& version, const KeyPattern& keyPattern, - bool estimatedValue, - int64_t maxSize); + bool estimatedValue); ShardId shardId; NamespaceString nss; @@ -196,31 +185,14 @@ struct DataSizeInfo { ChunkVersion version; KeyPattern keyPattern; bool estimatedValue; - int64_t maxSize; }; struct DataSizeResponse { - DataSizeResponse(long long sizeBytes, long long numObjects, bool maxSizeReached) - : sizeBytes(sizeBytes), numObjects(numObjects), maxSizeReached(maxSizeReached) {} + DataSizeResponse(long long sizeBytes, long long numObjects) + : sizeBytes(sizeBytes), numObjects(numObjects) {} long long sizeBytes; long long numObjects; - bool maxSizeReached; -}; - -struct ShardZoneInfo { - ShardZoneInfo(size_t numChunks, size_t firstNormalizedZoneIdx, const BSONObj& firstChunkMinKey) - : numChunks(numChunks), - firstNormalizedZoneIdx(firstNormalizedZoneIdx), - firstChunkMinKey(firstChunkMinKey) {} - - // Total number of chunks this shard has for this zone - size_t numChunks; - // Index in the vector of normalised zones of the first zone range that contains the first chunk - // for this shard in this zone - size_t firstNormalizedZoneIdx; - // minKey of the first chunk this shard has in this zone - BSONObj firstChunkMinKey; }; typedef stdx:: @@ -231,29 +203,14 @@ typedef stdx::variant<Status, StatusWith<AutoSplitVectorResponse>, StatusWith<Da DefragmentationActionResponse; typedef std::vector<ClusterStatistics::ShardStatistics> ShardStatisticsVector; -typedef StringMap<StringMap<ShardZoneInfo>> ShardZoneInfoMap; - -/* - * Keeps track of info needed for data size aware balancing. - */ -struct CollectionDataSizeInfoForBalancing { - CollectionDataSizeInfoForBalancing(std::map<ShardId, int64_t>&& shardToDataSizeMap, - long maxChunkSizeBytes) - : shardToDataSizeMap(std::move(shardToDataSizeMap)), maxChunkSizeBytes(maxChunkSizeBytes) {} - - std::map<ShardId, int64_t> shardToDataSizeMap; - const int64_t maxChunkSizeBytes; -}; +typedef std::map<ShardId, std::vector<ChunkType>> ShardToChunksMap; /** * Keeps track of zones for a collection. */ class ZoneInfo { public: - static const std::string kNoZoneName; - ZoneInfo(); - ZoneInfo(ZoneInfo&&) = default; /** * Appends the specified range to the set of ranges tracked for this collection and checks if @@ -281,13 +238,13 @@ public: return _zoneRanges; } - const ZoneRange& getZoneRange(const std::string& zoneName) const { - for (const auto& [_, zoneRange] : _zoneRanges) { - if (zoneRange.zone == zoneName) - return zoneRange; - } - MONGO_UNREACHABLE; - } + /** + * read all tags for collection via the catalog client and add to the zoneInfo + */ + static Status addTagsFromCatalog(OperationContext* opCtx, + const NamespaceString& nss, + const KeyPattern& keyPattern, + ZoneInfo& zoneInfo); private: // Map of zone max key to the zone description @@ -297,12 +254,6 @@ private: std::set<std::string> _allZones; }; -/** - * read all tags for collection via the catalog client and add to the zoneInfo - */ -StatusWith<ZoneInfo> createCollectionZoneInfo(OperationContext* opCtx, - const NamespaceString& nss, - const KeyPattern& keyPattern); class ChunkManager; /** @@ -315,7 +266,7 @@ class DistributionStatus final { DistributionStatus& operator=(const DistributionStatus&) = delete; public: - DistributionStatus(NamespaceString nss, ZoneInfo zoneInfo, const ChunkManager& chunkMngr); + DistributionStatus(NamespaceString nss, ShardToChunksMap shardToChunksMap); DistributionStatus(DistributionStatus&&) = default; ~DistributionStatus() {} @@ -327,6 +278,17 @@ public: } /** + * Appends the specified range to the set of ranges tracked for this collection and checks if + * it overlaps with existing ranges. + */ + Status addRangeToZone(const ZoneRange& range); + + /** + * Returns total number of chunks across all shards. + */ + size_t totalChunks() const; + + /** * Returns the total number of chunks across all shards, which fall into the specified zone's * range. */ @@ -343,6 +305,18 @@ public: size_t numberOfChunksInShardWithTag(const ShardId& shardId, const std::string& tag) const; /** + * Returns all chunks for the specified shard. + */ + const std::vector<ChunkType>& getChunks(const ShardId& shardId) const; + + /** + * Returns all tag ranges defined for the collection. + */ + const BSONObjIndexedMap<ZoneRange>& tagRanges() const { + return _zoneInfo.zoneRanges(); + } + + /** * Returns all tags defined for the collection. */ const std::set<std::string>& tags() const { @@ -350,88 +324,17 @@ public: } /** - * Using the set of tags defined for the collection, returns what tag corresponds to the - * specified chunk. If the chunk doesn't fall into any tag returns the empty string. + * Direct access to zone info */ - std::string getTagForRange(const ChunkRange& range) const; - - const ChunkManager& getChunkManager() const { - return _chunkMngr; - } - - const ZoneInfo& getZoneInfo() const { + ZoneInfo& zoneInfo() { return _zoneInfo; } - const StringMap<ShardZoneInfo>& getZoneInfoForShard(const ShardId& shardId) const; - /** - * Loop through each chunk on this shard within the given zone invoking 'handler' for each one - * of them. - * - * The iteration stops either when all the chunks have been visited (the method - * will return 'true') or the first time 'handler' returns 'false' (in which case the method - * will return 'false'). - * - * Effectively, return of 'true' means all chunks were visited and none matched, and - * 'false' means the hanlder return 'false' before visiting all chunks. + * Using the set of tags defined for the collection, returns what tag corresponds to the + * specified chunk. If the chunk doesn't fall into any tag returns the empty string. */ - template <typename Callable> - bool forEachChunkOnShardInZone(const ShardId& shardId, - const std::string& zoneName, - Callable&& handler) const { - - bool shouldContinue = true; - - const auto& shardZoneInfoMap = getZoneInfoForShard(shardId); - auto shardZoneInfoIt = shardZoneInfoMap.find(zoneName); - if (shardZoneInfoIt == shardZoneInfoMap.end()) { - return shouldContinue; - } - const auto& shardZoneInfo = shardZoneInfoIt->second; - - // Start from the first normalized zone that contains chunks for this shard - const auto initialZoneIt = _normalizedZones.cbegin() + shardZoneInfo.firstNormalizedZoneIdx; - - for (auto normalizedZoneIt = initialZoneIt; normalizedZoneIt < _normalizedZones.cend(); - normalizedZoneIt++) { - const auto& zoneRange = *normalizedZoneIt; - - const auto isFirstRange = (normalizedZoneIt == initialZoneIt); - - if (isFirstRange) { - tassert( - 8236530, - "Unexpected first normalized zone for shard '{}'. Expected '{}' but found '{}'"_format( - shardId.toString(), zoneName, zoneRange.zone), - zoneRange.zone == zoneName); - } else if (zoneRange.zone != zoneName) { - continue; - } - - // For the first range in zone we have pre-cached the minKey of the first chunk, - // thus we can start iterating from that one. - // For the subsequent ranges in this zone we start iterating from the minKey of - // the range itself. - const auto firstKey = isFirstRange ? shardZoneInfo.firstChunkMinKey : zoneRange.min; - - getChunkManager().forEachOverlappingChunk( - firstKey, zoneRange.max, false /* isMaxInclusive */, [&](const auto& chunk) { - if (chunk.getShardId() != shardId) { - return true; // continue - } - if (!handler(chunk)) { - shouldContinue = false; - }; - return shouldContinue; - }); - - if (!shouldContinue) { - break; - } - } - return shouldContinue; - } + std::string getTagForChunk(const ChunkType& chunk) const; /** * Returns a BSON/string representation of this distribution status. @@ -443,27 +346,11 @@ private: // Namespace for which this distribution applies NamespaceString _nss; - // Map that tracks how many chunks every shard is owning in each zone - // shardId -> zoneName -> shardZoneInfo - ShardZoneInfoMap _shardZoneInfoMap; + // Map of what chunks are owned by each shard + ShardToChunksMap _shardChunks; // Info for zones. ZoneInfo _zoneInfo; - - // Normalized zone are calculated starting from the currently configured zone in `config.tags` - // and the chunks provided by @this._chunkManager. - // - // The normalization process is performed to guarantee the following properties: - // - **All zone ranges are contiguous.** If there was a gap between two zones ranges we fill it - // with a range associated to the special kNoZone. - // - // - **Range boundaries always align with chunk boundaries.** If a zone range covers only - // partially a chunk, boundaries of that zone will be shrunk so that the normalized zone won't - // overlap with that chunk. Boundaries of a normalized zone will never fall in the middle of a - // chunk. - std::vector<ZoneRange> _normalizedZones; - - ChunkManager _chunkMngr; }; class BalancerPolicy { @@ -477,26 +364,24 @@ public: const std::string& chunkTag); /** - * Returns a suggested set of chunks or ranges to move within a collection's shards, given the - * specified state of the shards (draining, max size reached, etc) and the number of chunks or - * data size for that collection. If the policy doesn't recommend anything to move, it returns - * an empty vector. The entries in the vector do are all for separate source/destination shards - * and as such do not need to be done serially and can be scheduled in parallel. + * Returns a suggested set of chunks to move whithin a collection's shards, given the specified + * state of the shards (draining, max size reached, etc) and the number of chunks for that + * collection. If the policy doesn't recommend anything to move, it returns an empty vector. The + * entries in the vector do are all for separate source/destination shards and as such do not + * need to be done serially and can be scheduled in parallel. * * The balancing logic calculates the optimum number of chunks per shard for each zone and if * any of the shards have chunks, which are sufficiently higher than this number, suggests * moving chunks to shards, which are under this number. * - * The availableShards parameter is in/out and it contains the set of shards, which haven't - * been used for migrations yet. Used so we don't return multiple conflicting migrations for the - * same shard. + * The usedShards parameter is in/out and it contains the set of shards, which have already been + * used for migrations. Used so we don't return multiple conflicting migrations for the same + * shard. */ - static MigrateInfosWithReason balance( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - stdx::unordered_set<ShardId>* availableShards, - bool forceJumbo); + static MigrateInfosWithReason balance(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + stdx::unordered_set<ShardId>* usedShards, + bool forceJumbo); /** * Using the specified distribution information, returns a suggested better location for the @@ -507,70 +392,43 @@ public: const DistributionStatus& distribution); private: - /* - * Only considers shards with the specified tag, all shards in case the tag is empty. - * - * Returns a tuple <ShardID, number of chunks> referring the shard with less chunks. - * - * If balancing based on collection size on shards: - * - Returns a tuple <ShardID, amount of data in bytes> referring the shard with less data. + /** + * Return the shard with the specified tag, which has the least number of chunks. If the tag is + * empty, considers all shards. */ - static std::tuple<ShardId, int64_t> _getLeastLoadedReceiverShard( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - const std::string& tag, - const stdx::unordered_set<ShardId>& availableShards); + static ShardId _getLeastLoadedReceiverShard(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + const std::string& tag, + const stdx::unordered_set<ShardId>& excludedShards); /** - * Only considers shards with the specified tag, all shards in case the tag is empty. - * - * If balancing based on number of chunks: - * - Returns a tuple <ShardID, number of chunks> referring the shard with more chunks. - * - * If balancing based on collection size on shards: - * - Returns a tuple <ShardID, amount of data in bytes> referring the shard with more data. + * Return the shard which has the least number of chunks with the specified tag. If the tag is + * empty, considers all chunks. */ - static std::tuple<ShardId, int64_t> _getMostOverloadedShard( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const boost::optional<CollectionDataSizeInfoForBalancing>& collDataSizeInfo, - const std::string& chunkTag, - const stdx::unordered_set<ShardId>& availableShards); + static ShardId _getMostOverloadedShard(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + const std::string& chunkTag, + const stdx::unordered_set<ShardId>& excludedShards); /** * Selects one chunk for the specified zone (if appropriate) to be moved in order to bring the * deviation of the shards chunk contents closer to even across all shards in the specified - * zone. Takes into account and updates the shards, which haven't been used for migrations yet. + * zone. Takes into account and updates the shards, which have already been used for migrations. * - * Returns true if a migration was suggested, false otherwise. This method is intented to be - * called multiple times until all posible migrations for a zone have been selected. - */ - static bool _singleZoneBalanceBasedOnChunks(const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const std::string& tag, - size_t totalNumberOfShardsWithTag, - std::vector<MigrateInfo>* migrations, - stdx::unordered_set<ShardId>* availableShards, - MoveChunkRequest::ForceJumbo forceJumbo); - - /** - * Selects one range for the specified zone (if appropriate) to be moved in order to bring the - * deviation of the collection data size closer to even across all shards in the specified - * zone. Takes into account and updates the shards, which haven't been used for migrations yet. + * The 'idealNumberOfChunksPerShardForTag' indicates what is the ideal number of chunks which + * each shard must have and is used to determine the imbalance and also to prevent chunks from + * moving when not necessary. * * Returns true if a migration was suggested, false otherwise. This method is intented to be * called multiple times until all posible migrations for a zone have been selected. */ - static bool _singleZoneBalanceBasedOnDataSize( - const ShardStatisticsVector& shardStats, - const DistributionStatus& distribution, - const CollectionDataSizeInfoForBalancing& collDataSizeInfo, - const std::string& tag, - int64_t idealDataSizePerShardForZone, - std::vector<MigrateInfo>* migrations, - stdx::unordered_set<ShardId>* availableShards, - MoveChunkRequest::ForceJumbo forceJumbo); + static bool _singleZoneBalance(const ShardStatisticsVector& shardStats, + const DistributionStatus& distribution, + const std::string& tag, + size_t idealNumberOfChunksPerShardForTag, + std::vector<MigrateInfo>* migrations, + stdx::unordered_set<ShardId>* usedShards, + MoveChunkRequest::ForceJumbo forceJumbo); }; } // namespace mongo diff --git a/src/mongo/db/s/balancer/balancer_policy_test.cpp b/src/mongo/db/s/balancer/balancer_policy_test.cpp index 37b6925a082..0a53200149e 100644 --- a/src/mongo/db/s/balancer/balancer_policy_test.cpp +++ b/src/mongo/db/s/balancer/balancer_policy_test.cpp @@ -35,7 +35,6 @@ #include "mongo/db/s/balancer/balancer_policy.h" #include "mongo/platform/random.h" #include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/chunks_test_util.h" #include "mongo/unittest/unittest.h" namespace mongo { @@ -47,12 +46,9 @@ using std::stringstream; using std::vector; using ShardStatistics = ClusterStatistics::ShardStatistics; -typedef std::map<ShardId, std::vector<ChunkType>> ShardToChunksMap; - -PseudoRandom _random{SecureRandom().nextInt64()}; const auto emptyTagSet = std::set<std::string>(); -const auto kConfigId = ShardId("config"); +const std::string emptyShardVersion = ""; const auto kShardId0 = ShardId("shard0"); const auto kShardId1 = ShardId("shard1"); const auto kShardId2 = ShardId("shard2"); @@ -61,51 +57,6 @@ const auto kShardId4 = ShardId("shard4"); const auto kShardId5 = ShardId("shard5"); const NamespaceString kNamespace("TestDB", "TestColl"); const uint64_t kNoMaxSize = 0; -const KeyPattern kSKeyPattern(BSON("x" << 1)); -const Timestamp kCollTimestamp{1, 1}; -const OID kCollEpoch; - -const UUID& collUUID() { - static const UUID kCollectionUUID{UUID::gen()}; - return kCollectionUUID; -} - -std::vector<ChunkType> makeChunks(const std::vector<std::pair<ShardId, ChunkRange>>& specs) { - ChunkVersion chunkVersion{1, 0, kCollEpoch, kCollTimestamp}; - std::vector<ChunkType> chunks; - for (const auto& [shardId, range] : specs) { - chunks.push_back(ChunkType(collUUID(), range, chunkVersion, shardId)); - chunkVersion.incMajor(); - } - return chunks; -} - -RoutingTableHistory makeRoutingTable(const std::vector<ChunkType>& chunks) { - - return RoutingTableHistory::makeNew(kNamespace, - collUUID(), - kSKeyPattern, - nullptr, - false /* unique */, - kCollEpoch, - kCollTimestamp, - boost::none /* timeseriesFields */, - boost::none /* reshardingFields */, - boost::none /* maxChunkSizeBytes */, - true /* allowMigrations */, - chunks); -} - -ChunkManager makeChunkManager(const std::vector<ChunkType>& chunks) { - DatabaseVersion dbVersion; - auto rt = std::make_shared<RoutingTableHistory>(makeRoutingTable(chunks)); - - return {kConfigId, std::move(dbVersion), {std::move(rt)}, boost::none /* atClusterTime */}; -} - -DistributionStatus makeDistStatus(const ChunkManager& cm, ZoneInfo zoneInfo = ZoneInfo()) { - return {kNamespace, std::move(zoneInfo), cm}; -} /** * Constructs a shard statistics vector and a consistent mapping of chunks to shards given the @@ -114,7 +65,7 @@ DistributionStatus makeDistStatus(const ChunkManager& cm, ZoneInfo zoneInfo = Zo * * [MinKey, 1), [1, 2), [2, 3) ... [N - 1, MaxKey) */ -std::pair<std::pair<ShardStatisticsVector, ShardToChunksMap>, ChunkManager> generateCluster( +std::pair<ShardStatisticsVector, ShardToChunksMap> generateCluster( const vector<std::pair<ShardStatistics, size_t>>& shardsAndNumChunks) { int64_t totalNumChunks = 0; for (const auto& entry : shardsAndNumChunks) { @@ -126,9 +77,10 @@ std::pair<std::pair<ShardStatisticsVector, ShardToChunksMap>, ChunkManager> gene int64_t currentChunk = 0; - ChunkVersion chunkVersion(1, 0, kCollEpoch, kCollTimestamp); + ChunkVersion chunkVersion(1, 0, OID::gen(), Timestamp(1, 1)); + const UUID uuid = UUID::gen(); - std::vector<ChunkType> chunks; + const KeyPattern shardKeyPattern(BSON("x" << 1)); for (auto it = shardsAndNumChunks.begin(); it != shardsAndNumChunks.end(); it++) { ShardStatistics shard = std::move(it->first); @@ -140,586 +92,406 @@ std::pair<std::pair<ShardStatisticsVector, ShardToChunksMap>, ChunkManager> gene for (size_t i = 0; i < numChunks; i++, currentChunk++) { ChunkType chunk; - chunk.setCollectionUUID(collUUID()); - chunk.setMin(currentChunk == 0 ? kSKeyPattern.globalMin() : BSON("x" << currentChunk)); - chunk.setMax(currentChunk == totalNumChunks - 1 ? kSKeyPattern.globalMax() + chunk.setCollectionUUID(uuid); + chunk.setMin(currentChunk == 0 ? shardKeyPattern.globalMin() + : BSON("x" << currentChunk)); + chunk.setMax(currentChunk == totalNumChunks - 1 ? shardKeyPattern.globalMax() : BSON("x" << currentChunk + 1)); chunk.setShard(shard.shardId); chunk.setVersion(chunkVersion); chunkVersion.incMajor(); - chunkMap[shard.shardId].push_back(chunk); - chunks.push_back(std::move(chunk)); + chunkMap[shard.shardId].push_back(std::move(chunk)); } shardStats.push_back(std::move(shard)); } - return std::make_pair(std::make_pair(std::move(shardStats), std::move(chunkMap)), - makeChunkManager(chunks)); -} - -stdx::unordered_set<ShardId> getAllShardIds(const ShardStatisticsVector& shardStats) { - stdx::unordered_set<ShardId> shards; - std::transform(shardStats.begin(), - shardStats.end(), - std::inserter(shards, shards.end()), - [](const ShardStatistics& shardStatistics) { return shardStatistics.shardId; }); - return shards; + return std::make_pair(std::move(shardStats), std::move(chunkMap)); } MigrateInfosWithReason balanceChunks(const ShardStatisticsVector& shardStats, const DistributionStatus& distribution, bool shouldAggressivelyBalance, bool forceJumbo) { - auto availableShards = getAllShardIds(shardStats); - return BalancerPolicy::balance( - shardStats, distribution, boost::none /* collDataSizeInfo */, &availableShards, forceJumbo); -} - -void checkChunksOnShardForTag(const DistributionStatus& dist, - const ShardId& shardId, - const std::string& zoneName, - const std::vector<ChunkType>& expectedChunks) { - auto expectedChunkIt = expectedChunks.cbegin(); - const auto completed = - dist.forEachChunkOnShardInZone(shardId, zoneName, [&](const auto& chunk) { - ASSERT(expectedChunkIt != expectedChunks.end()) - << "forEachChunkOnShardInZone loop found more chunks than expected"; - ChunkInfo expectedChunkInfo{*expectedChunkIt++}; - ASSERT_EQ(Chunk(expectedChunkInfo, boost::none /* atClusterTime */).toString(), - chunk.toString()); - return true; // continue - }); - ASSERT(completed) - << "forEachChunkOnShardInZone loop unexpectedly returned false (did not complete)"; - ASSERT(expectedChunkIt == expectedChunks.cend()) - << "forEachChunkOnShardInZone loop did not iterate over all the expected chunks"; + stdx::unordered_set<ShardId> usedShards; + return BalancerPolicy::balance(shardStats, distribution, &usedShards, forceJumbo); } TEST(BalancerPolicy, Basic) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId2, kNoMaxSize, 3, false, emptyTagSet), 3}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId2, kNoMaxSize, 3, false, emptyTagSet, emptyShardVersion), 3}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } TEST(BalancerPolicy, SmallClusterShouldBePerfectlyBalanced) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 1, false, emptyTagSet), 1}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, emptyTagSet), 2}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId1, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } TEST(BalancerPolicy, SingleChunkShouldNotMove) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 1, false, emptyTagSet), 1}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); { - auto [migrations, reason] = balanceChunks(cluster.first, makeDistStatus(cm), true, false); + auto [migrations, reason] = balanceChunks( + cluster.first, DistributionStatus(kNamespace, cluster.second), true, false); ASSERT(migrations.empty()); ASSERT_EQ(MigrationReason::none, reason); } { - auto [migrations, reason] = balanceChunks(cluster.first, makeDistStatus(cm), false, false); + auto [migrations, reason] = balanceChunks( + cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT(migrations.empty()); ASSERT_EQ(MigrationReason::none, reason); } } TEST(BalancerPolicy, BalanceThresholdObeyed) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, emptyTagSet), 2}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, false, emptyTagSet), 1}, - {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}}); { - auto [migrations, reason] = balanceChunks(cluster.first, makeDistStatus(cm), true, false); + auto [migrations, reason] = balanceChunks( + cluster.first, DistributionStatus(kNamespace, cluster.second), true, false); ASSERT(migrations.empty()); ASSERT_EQ(MigrationReason::none, reason); } { - auto [migrations, reason] = balanceChunks(cluster.first, makeDistStatus(cm), false, false); + auto [migrations, reason] = balanceChunks( + cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT(migrations.empty()); ASSERT_EQ(MigrationReason::none, reason); } } TEST(BalancerPolicy, ParallelBalancing) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); ASSERT_EQ(kShardId1, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[1].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), *migrations[1].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[1].maxKey); } TEST(BalancerPolicy, ParallelBalancingDoesNotPutChunksOnShardsAboveTheOptimal) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 100, false, emptyTagSet), 100}, - {ShardStatistics(kShardId1, kNoMaxSize, 90, false, emptyTagSet), 90}, - {ShardStatistics(kShardId2, kNoMaxSize, 90, false, emptyTagSet), 90}, - {ShardStatistics(kShardId3, kNoMaxSize, 80, false, emptyTagSet), 80}, - {ShardStatistics(kShardId4, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId5, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 100, false, emptyTagSet, emptyShardVersion), 100}, + {ShardStatistics(kShardId1, kNoMaxSize, 90, false, emptyTagSet, emptyShardVersion), 90}, + {ShardStatistics(kShardId2, kNoMaxSize, 90, false, emptyTagSet, emptyShardVersion), 90}, + {ShardStatistics(kShardId3, kNoMaxSize, 80, false, emptyTagSet, emptyShardVersion), 80}, + {ShardStatistics(kShardId4, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId5, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId4, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); ASSERT_EQ(kShardId1, migrations[1].from); ASSERT_EQ(kShardId5, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[1].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), *migrations[1].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[1].maxKey); } TEST(BalancerPolicy, ParallelBalancingDoesNotMoveChunksFromShardsBelowOptimal) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 100, false, emptyTagSet), 100}, - {ShardStatistics(kShardId1, kNoMaxSize, 30, false, emptyTagSet), 30}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 5}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 100, false, emptyTagSet, emptyShardVersion), 100}, + {ShardStatistics(kShardId1, kNoMaxSize, 30, false, emptyTagSet, emptyShardVersion), 30}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 5}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId3, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNecessary) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 8, false, emptyTagSet), 8}, - {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 8, false, emptyTagSet, emptyShardVersion), 8}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); // Here kShardId0 would have been selected as a donor - auto availableShards = getAllShardIds(cluster.first); - availableShards.erase(kShardId0); - const auto [migrations, reason] = BalancerPolicy::balance(cluster.first, - makeDistStatus(cm), - boost::none /* collDataSizeInfo */, - &availableShards, - false); + stdx::unordered_set<ShardId> usedShards{kShardId0}; + const auto [migrations, reason] = BalancerPolicy::balance( + cluster.first, DistributionStatus(kNamespace, cluster.second), &usedShards, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId1, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseSourceShardsWithMoveNotNecessary) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 12, false, emptyTagSet), 12}, - {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 12, false, emptyTagSet, emptyShardVersion), 12}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); // Here kShardId0 would have been selected as a donor - auto availableShards = getAllShardIds(cluster.first); - availableShards.erase(kShardId0); - const auto [migrations, reason] = BalancerPolicy::balance(cluster.first, - makeDistStatus(cm), - boost::none /* collDataSizeInfo */, - &availableShards, - false); + stdx::unordered_set<ShardId> usedShards{kShardId0}; + const auto [migrations, reason] = BalancerPolicy::balance( + cluster.first, DistributionStatus(kNamespace, cluster.second), &usedShards, false); ASSERT_EQ(0U, migrations.size()); } TEST(BalancerPolicy, ParallelBalancingNotSchedulingOnInUseDestinationShards) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId3, kNoMaxSize, 1, false, emptyTagSet, emptyShardVersion), 1}}); // Here kShardId2 would have been selected as a recipient - auto availableShards = getAllShardIds(cluster.first); - availableShards.erase(kShardId2); - const auto [migrations, reason] = BalancerPolicy::balance(cluster.first, - makeDistStatus(cm), - boost::none /* collDataSizeInfo */, - &availableShards, - false); + stdx::unordered_set<ShardId> usedShards{kShardId2}; + const auto [migrations, reason] = BalancerPolicy::balance( + cluster.first, DistributionStatus(kNamespace, cluster.second), &usedShards, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId3, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } -TEST(BalancerPolicy, JumboChunksNotMovedWhileEnforcingZones) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 3, false, emptyTagSet), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 3, false, {"a"}), 3}}); - - // construct a new chunk map where all the chunks are jumbo except this one - const auto& jumboChunk = cluster.second[kShardId0][1]; - - std::vector<ChunkType> chunks; - cm.forEachChunk([&](const auto& chunk) { - ChunkType ct{collUUID(), chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - if (chunk.getLastmod() == jumboChunk.getVersion()) - ct.setJumbo(false); - else - ct.setJumbo(true); - chunks.emplace_back(std::move(ct)); - return true; - }); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(makeChunkManager(chunks), std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(1U, migrations.size()); - ASSERT_EQ(kShardId0, migrations[0].from); - ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(jumboChunk.getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk.getMax(), *migrations[0].maxKey); - ASSERT_EQ(MigrationReason::zoneViolation, reason); -} - -TEST(BalancerPolicy, JumboChunksNotMovedWhileEnforcingZonesRandom) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 3, false, emptyTagSet), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 3, false, {"a"}), 3}}); - - // construct a new chunk map where all the chunks are jumbo except this one - const auto jumboChunkIdx = _random.nextInt64(cluster.second[kShardId0].size()); - const auto& jumboChunk = cluster.second[kShardId0][jumboChunkIdx]; - - std::vector<ChunkType> chunks; - cm.forEachChunk([&](const auto& chunk) { - ChunkType ct{collUUID(), chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - if (chunk.getLastmod() == jumboChunk.getVersion()) - ct.setJumbo(false); - else - ct.setJumbo(true); - chunks.emplace_back(std::move(ct)); - return true; - }); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(makeChunkManager(chunks), std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(1U, migrations.size()); - ASSERT_EQ(kShardId0, migrations[0].from); - ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(jumboChunk.getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk.getMax(), *migrations[0].maxKey); - ASSERT_EQ(MigrationReason::zoneViolation, reason); -} - TEST(BalancerPolicy, JumboChunksNotMoved) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}}); - - // construct a new chunk map where all the chunks are jumbo except this one - const auto& jumboChunk = cluster.second[kShardId0][1]; - - std::vector<ChunkType> chunks; - cm.forEachChunk([&](const auto& chunk) { - ChunkType ct{collUUID(), chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - if (chunk.getLastmod() == jumboChunk.getVersion()) - ct.setJumbo(false); - else - ct.setJumbo(true); - chunks.emplace_back(std::move(ct)); - return true; - }); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); - const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(makeChunkManager(chunks)), false, false); - ASSERT_EQ(1U, migrations.size()); - ASSERT_EQ(kShardId0, migrations[0].from); - ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(jumboChunk.getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk.getMax(), *migrations[0].maxKey); - ASSERT_EQ(MigrationReason::chunksImbalance, reason); -} - -TEST(BalancerPolicy, JumboChunksNotMovedRandom) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}}); - - // construct a new chunk map where all the chunks are jumbo except this one - const auto jumboChunkIdx = _random.nextInt64(cluster.second[kShardId0].size()); - const auto& jumboChunk = cluster.second[kShardId0][jumboChunkIdx]; - - std::vector<ChunkType> chunks; - cm.forEachChunk([&](const auto& chunk) { - ChunkType ct{collUUID(), chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - if (chunk.getLastmod() == jumboChunk.getVersion()) - ct.setJumbo(false); - else - ct.setJumbo(true); - chunks.emplace_back(std::move(ct)); - return true; - }); + cluster.second[kShardId0][0].setJumbo(true); + cluster.second[kShardId0][1].setJumbo(false); // Only chunk 1 is not jumbo + cluster.second[kShardId0][2].setJumbo(true); + cluster.second[kShardId0][3].setJumbo(true); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(makeChunkManager(chunks)), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(jumboChunk.getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk.getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMin(), migrations[0].minKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::chunksImbalance, reason); } TEST(BalancerPolicy, JumboChunksNotMovedParallel) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId2, kNoMaxSize, 2, false, emptyTagSet), 4}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 0}}); - - // construct a new chunk map where all the chunks are jumbo except the ones listed below - const auto& jumboChunk0 = cluster.second[kShardId0][1]; - const auto& jumboChunk1 = cluster.second[kShardId2][2]; - - std::vector<ChunkType> chunks; - cm.forEachChunk([&](const auto& chunk) { - ChunkType ct{collUUID(), chunk.getRange(), chunk.getLastmod(), chunk.getShardId()}; - if (chunk.getLastmod() == jumboChunk0.getVersion() || - chunk.getLastmod() == jumboChunk1.getVersion()) - ct.setJumbo(false); - else - ct.setJumbo(true); - chunks.emplace_back(std::move(ct)); - return true; - }); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId2, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 0}}); + + cluster.second[kShardId0][0].setJumbo(true); + cluster.second[kShardId0][1].setJumbo(false); // Only chunk 1 is not jumbo + cluster.second[kShardId0][2].setJumbo(true); + cluster.second[kShardId0][3].setJumbo(true); + + cluster.second[kShardId2][0].setJumbo(true); + cluster.second[kShardId2][1].setJumbo(true); + cluster.second[kShardId2][2].setJumbo(false); // Only chunk 1 is not jumbo + cluster.second[kShardId2][3].setJumbo(true); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(makeChunkManager(chunks)), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(jumboChunk0.getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk0.getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMin(), migrations[0].minKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), migrations[0].maxKey); + ASSERT_EQ(MigrationReason::chunksImbalance, reason); ASSERT_EQ(kShardId2, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); - ASSERT_BSONOBJ_EQ(jumboChunk1.getMin(), migrations[1].minKey); - ASSERT_BSONOBJ_EQ(jumboChunk1.getMax(), *migrations[1].maxKey); - - ASSERT_EQ(MigrationReason::chunksImbalance, reason); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][2].getMin(), migrations[1].minKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][2].getMax(), migrations[1].maxKey); } TEST(BalancerPolicy, DrainingSingleChunk) { // shard0 is draining and chunks will go to shard1, even though it has a lot more chunks - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet), 1}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 5}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); } TEST(BalancerPolicy, DrainingSingleChunkPerShard) { // shard0 and shard2 are draining and chunks will go to shard1 and shard3 in parallel - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet), 1}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 5}, - {ShardStatistics(kShardId2, kNoMaxSize, 2, true, emptyTagSet), 1}, - {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet), 5}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}, + {ShardStatistics(kShardId2, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId3, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(2U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); ASSERT_EQ(kShardId2, migrations[1].from); ASSERT_EQ(kShardId3, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[1].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[1].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[1].maxKey); } TEST(BalancerPolicy, DrainingWithTwoChunksFirstOneSelected) { // shard0 is draining and chunks will go to shard1, even though it has a lot more chunks - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 5}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet, emptyShardVersion), 5}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); } TEST(BalancerPolicy, DrainingMultipleShardsFirstOneSelected) { // shard0 and shard1 are both draining with very little chunks in them and chunks will go to // shard2, even though it has a lot more chunks that the other two - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, true, emptyTagSet), 1}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, true, emptyTagSet), 2}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 16}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, true, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, true, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 16}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); } TEST(BalancerPolicy, DrainingMultipleShardsWontAcceptChunks) { // shard0 has many chunks, but can't move them to shard1 or shard2 because they are draining - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, true, emptyTagSet), 0}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, true, emptyTagSet), 0}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 0, true, emptyTagSet, emptyShardVersion), 0}, + {ShardStatistics(kShardId2, kNoMaxSize, 0, true, emptyTagSet, emptyShardVersion), 0}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT(migrations.empty()); } TEST(BalancerPolicy, DrainingSingleAppropriateShardFoundDueToTag) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, {"NYC"}), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"LAX"}), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, true, {"LAX"}), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, {"NYC"}, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"LAX"}, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, true, {"LAX"}, emptyShardVersion), 1}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange( cluster.second[kShardId2][0].getMin(), cluster.second[kShardId2][0].getMax(), "LAX"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); } TEST(BalancerPolicy, DrainingNoAppropriateShardsFoundDueToTag) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, {"NYC"}), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"LAX"}), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, true, {"SEA"}), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, false, {"NYC"}, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"LAX"}, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, true, {"SEA"}, emptyShardVersion), 1}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange( cluster.second[kShardId2][0].getMin(), cluster.second[kShardId2][0].getMax(), "SEA"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT(migrations.empty()); } -TEST(BalancerPolicy, DrainingSingleAppropriateShardFoundMultipleTags) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, true, {}), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"Zone1", "Zone2"}), 2}, - {ShardStatistics(kShardId2, kNoMaxSize, 2, false, {"Zone1", "Zone2"}), 2}, - {ShardStatistics(kShardId3, kNoMaxSize, 2, false, {}), 2}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][0].getMin(), cluster.second[kShardId0][0].getMax(), "Zone1"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][1].getMin(), cluster.second[kShardId0][1].getMax(), "Zone2"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][2].getMin(), cluster.second[kShardId0][2].getMax(), "Zone1"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][3].getMin(), cluster.second[kShardId0][3].getMax(), "Zone2"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(1U, migrations.size()); - ASSERT_EQ(kShardId0, migrations[0].from); - const auto& recipientShard = migrations[0].to; - ASSERT(recipientShard == kShardId1 || recipientShard == kShardId2); - ASSERT_EQ(MigrationReason::drain, reason); -} - TEST(BalancerPolicy, NoBalancingDueToAllNodesEitherDrainingOrMaxedOut) { // shard0 and shard2 are draining, shard1 is maxed out - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet), 1}, - {ShardStatistics(kShardId1, 1, 1, false, emptyTagSet), 6}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, true, emptyTagSet), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 2, true, emptyTagSet, emptyShardVersion), 1}, + {ShardStatistics(kShardId1, 1, 1, false, emptyTagSet, emptyShardVersion), 6}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, true, emptyTagSet, emptyShardVersion), 1}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT(migrations.empty()); } @@ -727,188 +499,137 @@ TEST(BalancerPolicy, BalancerRespectsMaxShardSizeOnlyBalanceToNonMaxed) { // Note that maxSize of shard0 is 1, and it is therefore overloaded with currSize = 3. Other // shards have maxSize = 0 = unset. Even though the overloaded shard has the least number of // less chunks, we shouldn't move chunks to that shard. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, 1, 3, false, emptyTagSet), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, emptyTagSet), 5}, - {ShardStatistics(kShardId2, kNoMaxSize, 10, false, emptyTagSet), 10}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, 1, 3, false, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 5}, + {ShardStatistics(kShardId2, kNoMaxSize, 10, false, emptyTagSet, emptyShardVersion), 10}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); } TEST(BalancerPolicy, BalancerRespectsMaxShardSizeWhenAllBalanced) { // Note that maxSize of shard0 is 1, and it is therefore overloaded with currSize = 4. Other // shards have maxSize = 0 = unset. We check that being over the maxSize is NOT equivalent to // draining, we don't want to empty shards for no other reason than they are over this limit. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, 1, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet), 4}, - {ShardStatistics(kShardId2, kNoMaxSize, 4, false, emptyTagSet), 4}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, 1, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId1, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}, + {ShardStatistics(kShardId2, kNoMaxSize, 4, false, emptyTagSet, emptyShardVersion), 4}}); const auto [migrations, reason] = - balanceChunks(cluster.first, makeDistStatus(cm), false, false); + balanceChunks(cluster.first, DistributionStatus(kNamespace, cluster.second), false, false); ASSERT(migrations.empty()); } TEST(BalancerPolicy, BalancerRespectsTagsWhenDraining) { // shard1 drains the proper chunk to shard0, even though it is more loaded than shard2 - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 6}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, true, {"a", "b"}), 1}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, {"b"}), 2}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 6}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, true, {"a", "b"}, emptyShardVersion), 2}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, {"b"}, emptyShardVersion), 2}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 7), "a"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 8), kSKeyPattern.globalMax(), "b"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 7), "a"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 8), kMaxBSONKey, "b"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId1, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId1][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::drain, reason); } TEST(BalancerPolicy, BalancerRespectsTagPolicyBeforeImbalance) { // There is a large imbalance between shard0 and shard1, but the balancer must first fix the // chunks, which are on a wrong shard due to tag policy - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 6}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 2}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 6}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 2}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 100), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 100), "a"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::zoneViolation, reason); } -TEST(BalancerPolicy, OverloadedShardCannotDonateDueToTags) { - // There is a large imbalance between shard0 and the other shard, but shard0 can't donate chunks - // because it would violate tags. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 5}, - {ShardStatistics(kShardId1, kNoMaxSize, 0, false, emptyTagSet), 0}, - {ShardStatistics(kShardId2, kNoMaxSize, 0, false, emptyTagSet), 0}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(0U, migrations.size()); -} - TEST(BalancerPolicy, BalancerFixesIncorrectTagsWithCrossShardViolationOfTags) { // The zone policy dictates that the same shard must donate and also receive chunks. The test // validates that the same shard is not used as a donor and recipient as part of the same round. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, {"b"}), 3}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, {"b"}, emptyShardVersion), 3}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 1), "b"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 8), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 1), "b"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 8), kMaxBSONKey, "a"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::zoneViolation, reason); } -TEST(BalancerPolicy, BalancerFixesIncorrectTagInOtherwiseBalancedCluster) { +TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedCluster) { // Chunks are balanced across shards, but there are wrong tags, which need to be fixed - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 3}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 3}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 10), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 10), "a"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::zoneViolation, reason); } -TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedCluster) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 2, false, {}), 4}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"Zone1", "Zone2"}), 2}, - {ShardStatistics(kShardId2, kNoMaxSize, 2, false, {"Zone1", "Zone2"}), 2}, - {ShardStatistics(kShardId3, kNoMaxSize, 2, false, {}), 2}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][0].getMin(), cluster.second[kShardId0][0].getMax(), "Zone1"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][1].getMin(), cluster.second[kShardId0][1].getMax(), "Zone2"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][2].getMin(), cluster.second[kShardId0][2].getMax(), "Zone1"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange( - cluster.second[kShardId0][3].getMin(), cluster.second[kShardId0][3].getMax(), "Zone2"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(1U, migrations.size()); - ASSERT_EQ(kShardId0, migrations[0].from); - const auto& recipientShard = migrations[0].to; - ASSERT(recipientShard == kShardId1 || recipientShard == kShardId2); - ASSERT_EQ(MigrationReason::zoneViolation, reason); -} - - TEST(BalancerPolicy, BalancerTagAlreadyBalanced) { // Chunks are balanced across shards for the tag. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 3, false, {"a"}), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"a"}), 2}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 3, false, {"a"}, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 2, false, {"a"}, emptyShardVersion), 2}}); + + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, kMaxBSONKey, "a"))); ASSERT(balanceChunks(cluster.first, distribution, false, false).first.empty()); } TEST(BalancerPolicy, BalancerMostOverLoadShardHasMultipleTags) { // shard0 has chunks [MinKey, 1), [1, 2), [2, 3), [3, 4), [4, 5), so two chunks each // for tag "b" and "c". So [1, 2) is expected to be moved to shard1 in round 1. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a", "b", "c"}), 5}, - {ShardStatistics(kShardId1, kNoMaxSize, 1, false, {"b"}), 1}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, false, {"c"}), 1}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a", "b", "c"}, emptyShardVersion), 5}, + {ShardStatistics(kShardId1, kNoMaxSize, 1, false, {"b"}, emptyShardVersion), 1}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, false, {"c"}, emptyShardVersion), 1}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 1), "a"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 3), "b"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 3), BSON("x" << 5), "c"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 1), "a"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 3), "b"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 3), BSON("x" << 5), "c"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(1U, migrations.size()); @@ -916,47 +637,44 @@ TEST(BalancerPolicy, BalancerMostOverLoadShardHasMultipleTags) { ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId1, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][1].getMax(), migrations[0].maxKey); } TEST(BalancerPolicy, BalancerMostOverLoadShardHasMultipleTagsSkipTagWithShardInUse) { // shard0 has chunks [MinKey, 1), [1, 2), [2, 3), [3, 4), [4, 5), so two chunks each // for tag "b" and "c". So [3, 4) is expected to be moved to shard2 because shard1 is // in use. - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a", "b", "c"}), 5}, - {ShardStatistics(kShardId1, kNoMaxSize, 1, false, {"b"}), 1}, - {ShardStatistics(kShardId2, kNoMaxSize, 1, false, {"c"}), 1}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 1), "a"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 3), "b"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 3), BSON("x" << 5), "c"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - auto availableShards = getAllShardIds(cluster.first); - availableShards.erase(kShardId1); - const auto [migrations, reason] = BalancerPolicy::balance( - cluster.first, distribution, boost::none /* collDataSizeInfo */, &availableShards, false); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a", "b", "c"}, emptyShardVersion), 5}, + {ShardStatistics(kShardId1, kNoMaxSize, 1, false, {"b"}, emptyShardVersion), 1}, + {ShardStatistics(kShardId2, kNoMaxSize, 1, false, {"c"}, emptyShardVersion), 1}}); + + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 1), "a"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 3), "b"))); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(BSON("x" << 3), BSON("x" << 5), "c"))); + + stdx::unordered_set<ShardId> usedShards{kShardId1}; + const auto [migrations, reason] = + BalancerPolicy::balance(cluster.first, distribution, &usedShards, false); ASSERT_EQ(1U, migrations.size()); ASSERT_EQ(kShardId0, migrations[0].from); ASSERT_EQ(kShardId2, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][3].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][3].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][3].getMax(), migrations[0].maxKey); } TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedClusterParallel) { // Chunks are balanced across shards, but there are wrong tags, which need to be fixed - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 3}, - {ShardStatistics(kShardId3, kNoMaxSize, 5, false, emptyTagSet), 3}}); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}, emptyShardVersion), 3}, + {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 3}, + {ShardStatistics(kShardId3, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 3}}); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 20), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); + DistributionStatus distribution(kNamespace, cluster.second); + ASSERT_OK(distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 20), "a"))); const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); ASSERT_EQ(2U, migrations.size()); @@ -964,729 +682,177 @@ TEST(BalancerPolicy, BalancerFixesIncorrectTagsInOtherwiseBalancedClusterParalle ASSERT_EQ(kShardId2, migrations[0].from); ASSERT_EQ(kShardId0, migrations[0].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), *migrations[0].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId2][0].getMax(), migrations[0].maxKey); ASSERT_EQ(MigrationReason::zoneViolation, reason); ASSERT_EQ(kShardId3, migrations[1].from); ASSERT_EQ(kShardId1, migrations[1].to); ASSERT_BSONOBJ_EQ(cluster.second[kShardId3][0].getMin(), migrations[1].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId3][0].getMax(), *migrations[1].maxKey); + ASSERT_BSONOBJ_EQ(cluster.second[kShardId3][0].getMax(), migrations[1].maxKey); ASSERT_EQ(MigrationReason::zoneViolation, reason); } -TEST(BalancerPolicy, ChunksInNoZoneSpanOnAllShardsWithEmptyZones) { - // Balanacer is able to move chunks in the noZone to shards with tags - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, emptyTagSet), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 0}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 100), kSKeyPattern.globalMax(), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(1U, migrations.size()); - - ASSERT_EQ(kShardId0, migrations[0].from); - ASSERT_EQ(kShardId1, migrations[0].to); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMin(), migrations[0].minKey); - ASSERT_BSONOBJ_EQ(cluster.second[kShardId0][0].getMax(), *migrations[0].maxKey); - ASSERT_EQ(MigrationReason::chunksImbalance, reason); -} - -TEST(BalancerPolicy, BalancingNoZoneIgnoreTotalShardSize) { - // Shard1 is overloaded and contains: - // [min, 1) [1, 2) [2, 3] -> zone("a") - // [3, 4) [4, 5) [5, 6) -> NoZone - // - // But it won't donate any chunk since the - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, {"a"}), 3}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, {"a"}), 6}, - {ShardStatistics(kShardId2, kNoMaxSize, 5, false, emptyTagSet), 3}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 6), "a"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - const auto [migrations, reason] = balanceChunks(cluster.first, distribution, false, false); - ASSERT_EQ(0U, migrations.size()); -} - TEST(BalancerPolicy, BalancerHandlesNoShardsWithTag) { - auto [cluster, cm] = - generateCluster({{ShardStatistics(kShardId0, kNoMaxSize, 5, false, emptyTagSet), 2}, - {ShardStatistics(kShardId1, kNoMaxSize, 5, false, emptyTagSet), 2}}); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 7), "NonExistentZone"))); - const auto distribution = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT(balanceChunks(cluster.first, distribution, false, false).first.empty()); -} - -TEST(DistributionStatus, OneChunkNoZone) { - const auto chunks = - makeChunks({{kShardId0, {kSKeyPattern.globalMin(), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - const auto distStatus = makeDistStatus(cm); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(1, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& noZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, noZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), noZoneInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, chunks); -} - -TEST(DistributionStatus, OneChunkOneZone) { - const auto chunks = - makeChunks({{kShardId0, {kSKeyPattern.globalMin(), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "ZoneA"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(0, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - ASSERT_EQ(1, distStatus.totalChunksWithTag("ZoneA")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& shardZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneA"); - ASSERT_EQ(1, shardZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shardZoneInfo.firstChunkMinKey); + auto cluster = generateCluster( + {{ShardStatistics(kShardId0, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 2}, + {ShardStatistics(kShardId1, kNoMaxSize, 5, false, emptyTagSet, emptyShardVersion), 2}}); - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", chunks); -} - -TEST(DistributionStatus, OneChunkMultipleContiguosZones) { - const auto chunks = - makeChunks({{kShardId0, {kSKeyPattern.globalMin(), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 0), "ZoneA"))); - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 0), kSKeyPattern.globalMax(), "ZoneB"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(1, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("ZoneB")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneB")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneB")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& shardZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, shardZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shardZoneInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, chunks); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {}); -} - -TEST(DistributionStatus, OneChunkMultipleSparseZones) { - const auto chunks = - makeChunks({{kShardId0, {kSKeyPattern.globalMin(), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; + DistributionStatus distribution(kNamespace, cluster.second); ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 0), "ZoneA"))); - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 10), kSKeyPattern.globalMax(), "ZoneB"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(1, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("ZoneB")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneB")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneB")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& shardZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, shardZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shardZoneInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneB", {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, chunks); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneB", {}); -} - -TEST(DistributionStatus, MultipleChunksNoZone) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - {kShardId0, {BSON("x" << 0), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - const auto distStatus = makeDistStatus(cm); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(2, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& noZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(2, noZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), noZoneInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, chunks); -} - -TEST(DistributionStatus, MultipleChunksDistributedNoZone) { - const auto chunks = makeChunks({{kShardId1, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - {kShardId0, {BSON("x" << 0), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - const auto distStatus = makeDistStatus(cm); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(2, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(1, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& noZoneInfoS0 = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, noZoneInfoS0.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 0), noZoneInfoS0.firstChunkMinKey); - - const auto& noZoneInfoS1 = distStatus.getZoneInfoForShard(kShardId1).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, noZoneInfoS1.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), noZoneInfoS1.firstChunkMinKey); + distribution.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 7), "NonExistentZone"))); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {chunks[1]}); - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {chunks[0]}); -} - -TEST(DistributionStatus, MultipleChunksTwoShardsOneZone) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - {kShardId0, {BSON("x" << 0), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone( - ZoneRange(kSKeyPattern.globalMin(), kSKeyPattern.globalMax(), "ZoneA"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(0, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - ASSERT_EQ(2, distStatus.totalChunksWithTag("ZoneA")); - - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - - ASSERT_EQ(1, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - const auto& noZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneA"); - ASSERT_EQ(2, noZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), noZoneInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", chunks); - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); + ASSERT(balanceChunks(cluster.first, distribution, false, false).first.empty()); } -TEST(DistributionStatus, MultipleChunksTwoContiguosZones) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - {kShardId0, {BSON("x" << 0), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); +TEST(DistributionStatus, AddTagRangeOverlap) { + DistributionStatus d(kNamespace, ShardToChunksMap{}); - ZoneInfo zoneInfo; - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 0), "ZoneA"))); - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 0), kSKeyPattern.globalMax(), "ZoneB"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(0, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - ASSERT_EQ(1, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(1, distStatus.totalChunksWithTag("ZoneB")); - - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(0, distStatus.numberOfChunksInShard(kShardId1)); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneB")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneB")); - - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(0, distStatus.getZoneInfoForShard(kShardId1).size()); - - auto& shardZoneAInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneA"); - ASSERT_EQ(1, shardZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shardZoneAInfo.firstChunkMinKey); - - auto& shardZoneBInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneB"); - ASSERT_EQ(1, shardZoneBInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 0), shardZoneBInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneB", {}); - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {chunks[0]}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneB", {chunks[1]}); -} + // Note that there is gap between 10 and 20 for which there is no tag + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "a"))); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 20), BSON("x" << 30), "b"))); -TEST(DistributionStatus, MultipleChunksTwoZones) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - {kShardId1, {BSON("x" << 0), BSON("x" << 10)}}, - {kShardId2, {BSON("x" << 10), BSON("x" << 20)}}, - {kShardId2, {BSON("x" << 20), BSON("x" << 30)}}, - {kShardId0, {BSON("x" << 30), BSON("x" << 40)}}, - {kShardId1, {BSON("x" << 40), BSON("x" << 50)}}, - {kShardId2, {BSON("x" << 50), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 20), "ZoneA"))); - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 30), kSKeyPattern.globalMax(), "ZoneB"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(1, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(3, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(3, distStatus.totalChunksWithTag("ZoneB")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId1)); - ASSERT_EQ(3, distStatus.numberOfChunksInShard(kShardId2)); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneB")); - - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneB")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId2, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId2, "NotExistingZone")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId2, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId2, "ZoneB")); - - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId1).size()); - ASSERT_EQ(3, distStatus.getZoneInfoForShard(kShardId2).size()); - - auto& shard0ZoneAInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneA"); - ASSERT_EQ(1, shard0ZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shard0ZoneAInfo.firstChunkMinKey); - auto& shard0ZoneBInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneB"); - ASSERT_EQ(1, shard0ZoneBInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 30), shard0ZoneBInfo.firstChunkMinKey); - - auto& shard1ZoneAInfo = distStatus.getZoneInfoForShard(kShardId1).at("ZoneA"); - ASSERT_EQ(1, shard1ZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 0), shard1ZoneAInfo.firstChunkMinKey); - auto& shard1ZoneBInfo = distStatus.getZoneInfoForShard(kShardId1).at("ZoneB"); - ASSERT_EQ(1, shard1ZoneBInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 40), shard1ZoneBInfo.firstChunkMinKey); - - auto& shard2ZoneAInfo = distStatus.getZoneInfoForShard(kShardId2).at("ZoneA"); - ASSERT_EQ(1, shard2ZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 10), shard2ZoneAInfo.firstChunkMinKey); - auto& shard2NoZoneInfo = distStatus.getZoneInfoForShard(kShardId2).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(1, shard2NoZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 20), shard2NoZoneInfo.firstChunkMinKey); - auto& shard2ZoneBInfo = distStatus.getZoneInfoForShard(kShardId2).at("ZoneB"); - ASSERT_EQ(1, shard2ZoneBInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 50), shard2ZoneBInfo.firstChunkMinKey); - - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {chunks[0]}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneB", {chunks[4]}); - - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {chunks[1]}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneB", {chunks[5]}); - - checkChunksOnShardForTag(distStatus, kShardId2, ZoneInfo::kNoZoneName, {chunks[3]}); - checkChunksOnShardForTag(distStatus, kShardId2, "NotExistingZone", {}); - checkChunksOnShardForTag(distStatus, kShardId2, "ZoneA", {chunks[2]}); - checkChunksOnShardForTag(distStatus, kShardId2, "ZoneB", {chunks[6]}); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << 2), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << -1), BSON("x" << 5), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << 5), BSON("x" << 9), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << 5), BSON("x" << 25), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << -1), BSON("x" << 32), "d"))); + ASSERT_EQ(ErrorCodes::RangeOverlapConflict, + d.addRangeToZone(ZoneRange(BSON("x" << 25), kMaxBSONKey, "d"))); } -TEST(DistributionStatus, MultipleChunksMulitpleZoneRanges) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - - {kShardId1, {BSON("x" << 0), BSON("x" << 10)}}, // ZoneA - {kShardId0, {BSON("x" << 10), BSON("x" << 20)}}, // ZoneA - - {kShardId1, {BSON("x" << 20), BSON("x" << 30)}}, - {kShardId0, {BSON("x" << 30), BSON("x" << 40)}}, - - {kShardId1, {BSON("x" << 40), BSON("x" << 50)}}, // ZoneA - {kShardId0, {BSON("x" << 50), BSON("x" << 60)}}, // ZoneA +TEST(DistributionStatus, ChunkTagsSelectorWithRegularKeys) { + DistributionStatus d(kNamespace, ShardToChunksMap{}); - {kShardId1, {BSON("x" << 60), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "a"))); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 10), BSON("x" << 20), "b"))); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 20), BSON("x" << 30), "c"))); - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 0), BSON("x" << 20), "ZoneA"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 40), BSON("x" << 60), "ZoneA"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(4, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(4, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(4, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(4, distStatus.numberOfChunksInShard(kShardId1)); + { + ChunkType chunk; + chunk.setMin(kMinBSONKey); + chunk.setMax(BSON("x" << 1)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 0)); + chunk.setMax(BSON("x" << 1)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 1)); + chunk.setMax(BSON("x" << 5)); + ASSERT_EQUALS("a", d.getTagForChunk(chunk)); + } - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId1).size()); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 10)); + chunk.setMax(BSON("x" << 20)); + ASSERT_EQUALS("b", d.getTagForChunk(chunk)); + } - auto& shard0ZoneAInfo = distStatus.getZoneInfoForShard(kShardId0).at("ZoneA"); - ASSERT_EQ(2, shard0ZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 10), shard0ZoneAInfo.firstChunkMinKey); - auto& shard0NoZoneInfo = distStatus.getZoneInfoForShard(kShardId0).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(2, shard0NoZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(kSKeyPattern.globalMin(), shard0NoZoneInfo.firstChunkMinKey); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 15)); + chunk.setMax(BSON("x" << 20)); + ASSERT_EQUALS("b", d.getTagForChunk(chunk)); + } - auto& shard1ZoneAInfo = distStatus.getZoneInfoForShard(kShardId1).at("ZoneA"); - ASSERT_EQ(2, shard1ZoneAInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 0), shard1ZoneAInfo.firstChunkMinKey); - auto& shard1NoZoneInfo = distStatus.getZoneInfoForShard(kShardId1).at(ZoneInfo::kNoZoneName); - ASSERT_EQ(2, shard1NoZoneInfo.numChunks); - ASSERT_BSONOBJ_EQ(BSON("x" << 20), shard1NoZoneInfo.firstChunkMinKey); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 25)); + chunk.setMax(BSON("x" << 30)); + ASSERT_EQUALS("c", d.getTagForChunk(chunk)); + } - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {chunks[0], chunks[4]}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {chunks[2], chunks[6]}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 35)); + chunk.setMax(BSON("x" << 40)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {chunks[3], chunks[7]}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {chunks[1], chunks[5]}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); -} + { + ChunkType chunk; + chunk.setMin(BSON("x" << 30)); + chunk.setMax(kMaxBSONKey); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } -TEST(DistributionStatus, MultipleChunksMulitpleZoneRangesNotAligned) { - const auto chunks = - makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, // Zone A - {kShardId0, {BSON("x" << 0), BSON("x" << 10)}}, // Zone A - {kShardId2, {BSON("x" << 10), BSON("x" << 20)}}, // No Zone - {kShardId2, {BSON("x" << 20), BSON("x" << 30)}}, // Zone A - {kShardId1, {BSON("x" << 30), BSON("x" << 40)}}, // No Zone - {kShardId1, {BSON("x" << 40), BSON("x" << 50)}}, // Zone B - {kShardId0, {BSON("x" << 50), BSON("x" << 60)}}, // No Zone - {kShardId0, {BSON("x" << 60), BSON("x" << 70)}}, // Zone B - {kShardId2, {BSON("x" << 70), kSKeyPattern.globalMax()}}}); // No zone - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK( - zoneInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 15), "ZoneA"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 15), BSON("x" << 35), "ZoneA"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 40), BSON("x" << 55), "ZoneB"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 55), BSON("x" << 75), "ZoneB"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - ASSERT_EQ(kNamespace, distStatus.nss()); - - ASSERT_EQ(4, distStatus.totalChunksWithTag(ZoneInfo::kNoZoneName)); - ASSERT_EQ(3, distStatus.totalChunksWithTag("ZoneA")); - ASSERT_EQ(2, distStatus.totalChunksWithTag("ZoneB")); - ASSERT_EQ(0, distStatus.totalChunksWithTag("NotExistingZone")); - - ASSERT_EQ(4, distStatus.numberOfChunksInShard(kShardId0)); - ASSERT_EQ(2, distStatus.numberOfChunksInShard(kShardId1)); - ASSERT_EQ(3, distStatus.numberOfChunksInShard(kShardId2)); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, ZoneInfo::kNoZoneName)); - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId0, "ZoneB")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId0, "NotExistingZone")); - - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId1, ZoneInfo::kNoZoneName)); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneA")); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId1, "ZoneB")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId1, "NotExistingZone")); - - ASSERT_EQ(2, distStatus.numberOfChunksInShardWithTag(kShardId2, ZoneInfo::kNoZoneName)); - ASSERT_EQ(1, distStatus.numberOfChunksInShardWithTag(kShardId2, "ZoneA")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId2, "ZoneB")); - ASSERT_EQ(0, distStatus.numberOfChunksInShardWithTag(kShardId2, "NotExistingZone")); - - ASSERT_EQ(3, distStatus.getZoneInfoForShard(kShardId0).size()); - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId1).size()); - ASSERT_EQ(2, distStatus.getZoneInfoForShard(kShardId2).size()); - - checkChunksOnShardForTag(distStatus, kShardId0, ZoneInfo::kNoZoneName, {chunks[6]}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneA", {chunks[0], chunks[1]}); - checkChunksOnShardForTag(distStatus, kShardId0, "ZoneB", {chunks[7]}); - checkChunksOnShardForTag(distStatus, kShardId0, "NotExistingZone", {}); - - checkChunksOnShardForTag(distStatus, kShardId1, ZoneInfo::kNoZoneName, {chunks[4]}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneA", {}); - checkChunksOnShardForTag(distStatus, kShardId1, "ZoneB", {chunks[5]}); - checkChunksOnShardForTag(distStatus, kShardId1, "NotExistingZone", {}); - - checkChunksOnShardForTag(distStatus, kShardId2, ZoneInfo::kNoZoneName, {chunks[2], chunks[8]}); - checkChunksOnShardForTag(distStatus, kShardId2, "ZoneA", {chunks[3]}); - checkChunksOnShardForTag(distStatus, kShardId2, "ZoneB", {}); - checkChunksOnShardForTag(distStatus, kShardId2, "NotExistingZone", {}); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 40)); + chunk.setMax(kMaxBSONKey); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } } -TEST(DistributionStatus, forEachChunkOnShardInZoneExitCondition) { - const auto chunks = makeChunks({{kShardId0, {kSKeyPattern.globalMin(), BSON("x" << 0)}}, - - {kShardId1, {BSON("x" << 0), BSON("x" << 10)}}, // ZoneA - {kShardId0, {BSON("x" << 10), BSON("x" << 20)}}, // ZoneA - - {kShardId1, {BSON("x" << 20), BSON("x" << 30)}}, - {kShardId0, {BSON("x" << 30), BSON("x" << 40)}}, - - {kShardId1, {BSON("x" << 40), BSON("x" << 50)}}, // ZoneA - {kShardId0, {BSON("x" << 50), BSON("x" << 60)}}, // ZoneA - - {kShardId1, {BSON("x" << 60), kSKeyPattern.globalMax()}}}); - const auto cm = makeChunkManager(chunks); - - ZoneInfo zoneInfo; - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 0), BSON("x" << 20), "ZoneA"))); - ASSERT_OK(zoneInfo.addRangeToZone(ZoneRange(BSON("x" << 40), BSON("x" << 60), "ZoneA"))); - const auto distStatus = makeDistStatus(cm, std::move(zoneInfo)); - - const auto assertLoopStopAt = - [&](const ShardId& shardId, const std::string& zoneName, const size_t stopCount) { - size_t numChunkIterated{0}; +TEST(DistributionStatus, ChunkTagsSelectorWithMinMaxKeys) { + DistributionStatus d(kNamespace, ShardToChunksMap{}); - const auto completed = - distStatus.forEachChunkOnShardInZone(shardId, zoneName, [&](const auto& chunk) { - if (++numChunkIterated == stopCount) { - return false; // break - } - return true; // continue - }); + ASSERT_OK(d.addRangeToZone(ZoneRange(kMinBSONKey, BSON("x" << -100), "a"))); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << -10), BSON("x" << 10), "b"))); + ASSERT_OK(d.addRangeToZone(ZoneRange(BSON("x" << 100), kMaxBSONKey, "c"))); - ASSERT(!completed) << "forEachChunkOnShardInZone loop did not stop"; - ASSERT_EQ(stopCount, numChunkIterated); - }; - - assertLoopStopAt(kShardId0, ZoneInfo::kNoZoneName, 1); - assertLoopStopAt(kShardId0, ZoneInfo::kNoZoneName, 2); - - assertLoopStopAt(kShardId1, ZoneInfo::kNoZoneName, 1); - assertLoopStopAt(kShardId1, ZoneInfo::kNoZoneName, 2); - - assertLoopStopAt(kShardId0, "ZoneA", 1); - assertLoopStopAt(kShardId0, "ZoneA", 2); + { + ChunkType chunk; + chunk.setMin(kMinBSONKey); + chunk.setMax(BSON("x" << -100)); + ASSERT_EQUALS("a", d.getTagForChunk(chunk)); + } - assertLoopStopAt(kShardId1, "ZoneA", 1); - assertLoopStopAt(kShardId1, "ZoneA", 2); -} + { + ChunkType chunk; + chunk.setMin(BSON("x" << -100)); + chunk.setMax(BSON("x" << -11)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } -TEST(ZoneInfo, AddTagRangeOverlap) { - ZoneInfo zInfo; + { + ChunkType chunk; + chunk.setMin(BSON("x" << -10)); + chunk.setMax(BSON("x" << 0)); + ASSERT_EQUALS("b", d.getTagForChunk(chunk)); + } - // Note that there is gap between 10 and 20 for which there is no tag - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "a"))); - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 20), BSON("x" << 30), "b"))); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 0)); + chunk.setMax(BSON("x" << 10)); + ASSERT_EQUALS("b", d.getTagForChunk(chunk)); + } - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << 2), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << -1), BSON("x" << 5), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << 5), BSON("x" << 9), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << 5), BSON("x" << 25), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << -1), BSON("x" << 32), "d"))); - ASSERT_EQ(ErrorCodes::RangeOverlapConflict, - zInfo.addRangeToZone(ZoneRange(BSON("x" << 25), kSKeyPattern.globalMax(), "d"))); -} + { + ChunkType chunk; + chunk.setMin(BSON("x" << 10)); + chunk.setMax(BSON("x" << 20)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } -TEST(ZoneInfo, ChunkTagsSelectorWithRegularKeys) { - ZoneInfo zInfo; - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 1), BSON("x" << 10), "a"))); - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 10), BSON("x" << 20), "b"))); - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 20), BSON("x" << 30), "c"))); - - ASSERT_EQUALS(ZoneInfo::kNoZoneName, - zInfo.getZoneForChunk({kSKeyPattern.globalMin(), BSON("x" << 1)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, zInfo.getZoneForChunk({BSON("x" << 0), BSON("x" << 1)})); - ASSERT_EQUALS("a", zInfo.getZoneForChunk({BSON("x" << 1), BSON("x" << 5)})); - ASSERT_EQUALS("b", zInfo.getZoneForChunk({BSON("x" << 10), BSON("x" << 20)})); - ASSERT_EQUALS("b", zInfo.getZoneForChunk({BSON("x" << 15), BSON("x" << 20)})); - ASSERT_EQUALS("c", zInfo.getZoneForChunk({BSON("x" << 25), BSON("x" << 30)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, zInfo.getZoneForChunk({BSON("x" << 35), BSON("x" << 40)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, - zInfo.getZoneForChunk({BSON("x" << 30), kSKeyPattern.globalMax()})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, - zInfo.getZoneForChunk({BSON("x" << 40), kSKeyPattern.globalMax()})); -} + { + ChunkType chunk; + chunk.setMin(BSON("x" << 10)); + chunk.setMax(BSON("x" << 100)); + ASSERT_EQUALS("", d.getTagForChunk(chunk)); + } -TEST(ZoneInfo, ChunkTagsSelectorWithMinMaxKeys) { - - ZoneInfo zInfo; - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(kSKeyPattern.globalMin(), BSON("x" << -100), "a"))); - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << -10), BSON("x" << 10), "b"))); - ASSERT_OK(zInfo.addRangeToZone(ZoneRange(BSON("x" << 100), kSKeyPattern.globalMax(), "c"))); - - ASSERT_EQUALS("a", zInfo.getZoneForChunk({kSKeyPattern.globalMin(), BSON("x" << -100)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, - zInfo.getZoneForChunk({BSON("x" << -100), BSON("x" << -11)})); - ASSERT_EQUALS("b", zInfo.getZoneForChunk({BSON("x" << -10), BSON("x" << 0)})); - ASSERT_EQUALS("b", zInfo.getZoneForChunk({BSON("x" << 0), BSON("x" << 10)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, zInfo.getZoneForChunk({BSON("x" << 10), BSON("x" << 20)})); - ASSERT_EQUALS(ZoneInfo::kNoZoneName, - zInfo.getZoneForChunk({BSON("x" << 10), BSON("x" << 100)})); - ASSERT_EQUALS("c", zInfo.getZoneForChunk({BSON("x" << 200), kSKeyPattern.globalMax()})); + { + ChunkType chunk; + chunk.setMin(BSON("x" << 200)); + chunk.setMax(kMaxBSONKey); + ASSERT_EQUALS("c", d.getTagForChunk(chunk)); + } } } // namespace diff --git a/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.cpp b/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.cpp index f84c7125edb..e886b8b9e48 100644 --- a/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.cpp +++ b/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.cpp @@ -26,12 +26,11 @@ * exception statement from all source files in the program, then also delete * it in the license file. */ -#include "mongo/bson/bsonobj.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding +#include "mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/persistent_task_store.h" -#include "mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h" #include "mongo/logv2/log.h" #include "mongo/s/grid.h" @@ -213,10 +212,20 @@ SharedSemiFuture<void> ClusterChunksResizePolicyImpl::activate(OperationContext* stdx::lock_guard<Latch> lk(_stateMutex); if (!_activeRequestPromise.is_initialized()) { - invariant(_collectionsBeingProcessed.empty()); + invariant(!_unprocessedCollections && _collectionsBeingProcessed.empty()); _defaultMaxChunksSizeBytes = defaultMaxChunksSizeBytes; invariant(_defaultMaxChunksSizeBytes > 0); + DBDirectClient dbClient(opCtx); + FindCommandRequest findCollectionsRequest{CollectionType::ConfigNS}; + findCollectionsRequest.setFilter( + BSON(CollectionTypeBase::kChunksAlreadySplitForDowngradeFieldName + << BSON("$not" << BSON("$eq" << true)))); + _unprocessedCollections = dbClient.find(std::move(findCollectionsRequest)); + uassert(ErrorCodes::OperationFailed, + "Failed to establish a cursor for accessing config.collections", + _unprocessedCollections); + _activeRequestPromise.emplace(); } @@ -233,6 +242,7 @@ void ClusterChunksResizePolicyImpl::stop() { stdx::lock_guard<Latch> lk(_stateMutex); if (_activeRequestPromise.is_initialized()) { _collectionsBeingProcessed.clear(); + _unprocessedCollections = nullptr; _activeRequestPromise->setFrom( Status(ErrorCodes::Interrupted, "Chunk resizing task has been interrupted")); _activeRequestPromise = boost::none; @@ -254,17 +264,6 @@ boost::optional<DefragmentationAction> ClusterChunksResizePolicyImpl::getNextStr } bool stateInspectionCompleted = false; - - DBDirectClient dbClient(opCtx); - FindCommandRequest findCollectionsRequest{CollectionType::ConfigNS}; - findCollectionsRequest.setFilter( - BSON(CollectionTypeBase::kChunksAlreadySplitForDowngradeFieldName - << BSON("$not" << BSON("$eq" << true)))); - auto unprocessedCollections = dbClient.find(std::move(findCollectionsRequest)); - uassert(ErrorCodes::OperationFailed, - "Failed to establish a cursor for accessing config.collections", - unprocessedCollections); - while (!stateInspectionCompleted) { // Try to get the next action from the current subset of collections being processed. for (auto it = _collectionsBeingProcessed.begin(); @@ -308,9 +307,9 @@ boost::optional<DefragmentationAction> ClusterChunksResizePolicyImpl::getNextStr } if (_collectionsBeingProcessed.size() < kMaxCollectionsBeingProcessed && - unprocessedCollections->more()) { + _unprocessedCollections->more()) { // Start processing a new collection - auto nextDoc = unprocessedCollections->next(); + auto nextDoc = _unprocessedCollections->next(); CollectionType coll(nextDoc); auto initialCollState = _buildInitialStateFor(opCtx, coll); if (initialCollState) { @@ -323,35 +322,25 @@ boost::optional<DefragmentationAction> ClusterChunksResizePolicyImpl::getNextStr } } - if (_collectionsBeingProcessed.empty() && !unprocessedCollections->more()) { + if (_collectionsBeingProcessed.empty() && !_unprocessedCollections->more()) { LOGV2(6417104, "Cluster chunks resize process completed. Clearing up internal state"); + PersistentTaskStore<CollectionType> store(CollectionType::ConfigNS); try { - DBDirectClient dbClient(opCtx); - auto ignoreWriteResponse = write_ops::checkWriteErrors(dbClient.update([&] { - write_ops::UpdateCommandRequest updateOp(CollectionType::ConfigNS); - BSONObj allDocsQuery; - auto unsetResizeField = write_ops::UpdateModification::parseFromClassicUpdate( - BSON("$unset" << BSON(CollectionType::kChunksAlreadySplitForDowngradeFieldName - << ""))); - write_ops::UpdateOpEntry updateEntry(allDocsQuery, unsetResizeField); - updateEntry.setMulti(true); - updateEntry.setUpsert(false); - - updateOp.setUpdates({updateEntry}); - return updateOp; - }())); - - WriteConcernResult ignoreResult; - auto latestOpTime = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); - uassertStatusOK(waitForWriteConcern( - opCtx, latestOpTime, WriteConcerns::kMajorityWriteConcernNoTimeout, &ignoreResult)); + BSONObj allDocsQuery; + store.update(opCtx, + allDocsQuery, + BSON("$unset" << BSON( + CollectionType::kChunksAlreadySplitForDowngradeFieldName << "")), + WriteConcerns::kMajorityWriteConcernNoTimeout); + } catch (const ExceptionFor<ErrorCodes::NoMatchingDocument>&) { + // ignore } catch (const DBException& e) { LOGV2_WARNING( 6417105, "Failed to clear persisted state while ending cluster chunks resize process", "err"_attr = redact(e)); } - + _unprocessedCollections = nullptr; _activeRequestPromise->setFrom(Status::OK()); _activeRequestPromise = boost::none; } diff --git a/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h b/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h index 8aa08a48859..b53b7afbeda 100644 --- a/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h +++ b/src/mongo/db/s/balancer/cluster_chunks_resize_policy_impl.h @@ -29,6 +29,7 @@ #pragma once +#include "mongo/client/dbclient_cursor.h" #include "mongo/db/s/balancer/cluster_chunks_resize_policy.h" #include "mongo/s/catalog/type_collection.h" @@ -121,6 +122,7 @@ private: Mutex _stateMutex = MONGO_MAKE_LATCH("ClusterChunksResizePolicyImpl::_stateMutex"); boost::optional<SharedPromise<void>> _activeRequestPromise{boost::none}; + std::unique_ptr<mongo::DBClientCursor> _unprocessedCollections; stdx::unordered_map<UUID, CollectionState, UUID::Hash> _collectionsBeingProcessed; int64_t _defaultMaxChunksSizeBytes{-1}; diff --git a/src/mongo/db/s/balancer/cluster_statistics.cpp b/src/mongo/db/s/balancer/cluster_statistics.cpp index 73f202cbfbc..b9fffb0af51 100644 --- a/src/mongo/db/s/balancer/cluster_statistics.cpp +++ b/src/mongo/db/s/balancer/cluster_statistics.cpp @@ -54,23 +54,27 @@ ClusterStatistics::ShardStatistics::ShardStatistics(ShardId inShardId, uint64_t inCurrSizeBytes, bool inIsDraining, std::set<std::string> inShardTags, + std::string inMongoVersion, use_bytes_t t) : shardId(std::move(inShardId)), maxSizeBytes(inMaxSizeBytes), currSizeBytes(inCurrSizeBytes), isDraining(inIsDraining), - shardTags(std::move(inShardTags)) {} + shardTags(std::move(inShardTags)), + mongoVersion(std::move(inMongoVersion)) {} ClusterStatistics::ShardStatistics::ShardStatistics(ShardId inShardId, uint64_t inMaxSizeMB, uint64_t inCurrSizeMB, bool inIsDraining, - std::set<std::string> inShardTags) + std::set<std::string> inShardTags, + std::string inMongoVersion) : ShardStatistics(inShardId, convertMBToBytes(inMaxSizeMB), convertMBToBytes(inCurrSizeMB), inIsDraining, std::move(inShardTags), + std::move(inMongoVersion), use_bytes_t{}) {} bool ClusterStatistics::ShardStatistics::isSizeMaxed() const { @@ -91,6 +95,8 @@ BSONObj ClusterStatistics::ShardStatistics::toBSON() const { BSONArrayBuilder arrayBuilder(builder.subarrayStart("tags")); arrayBuilder.append(shardTags); } + + builder.append("version", mongoVersion); return builder.obj(); } diff --git a/src/mongo/db/s/balancer/cluster_statistics.h b/src/mongo/db/s/balancer/cluster_statistics.h index 973766cf116..868c59d774a 100644 --- a/src/mongo/db/s/balancer/cluster_statistics.h +++ b/src/mongo/db/s/balancer/cluster_statistics.h @@ -66,13 +66,15 @@ public: uint64_t currSizeBytes, bool isDraining, std::set<std::string> shardTags, + std::string mongoVersion, use_bytes_t t); ShardStatistics(ShardId shardId, uint64_t maxSizeMB, uint64_t currSizeMB, bool isDraining, - std::set<std::string> shardTags); + std::set<std::string> shardTags, + std::string mongoVersion); /** * Returns true if a shard is not allowed to receive any new chunks because it has reached @@ -99,6 +101,9 @@ public: // Set of tags for the shard std::set<std::string> shardTags; + + // Version of mongod, which runs on this shard's primary + std::string mongoVersion; }; virtual ~ClusterStatistics(); diff --git a/src/mongo/db/s/balancer/cluster_statistics_impl.cpp b/src/mongo/db/s/balancer/cluster_statistics_impl.cpp index 285dd749114..495e8a77774 100644 --- a/src/mongo/db/s/balancer/cluster_statistics_impl.cpp +++ b/src/mongo/db/s/balancer/cluster_statistics_impl.cpp @@ -46,6 +46,50 @@ #include "mongo/util/str.h" namespace mongo { +namespace { + +const char kVersionField[] = "version"; + +/** + * Executes the serverStatus command against the specified shard and obtains the version of the + * running MongoD service. + * + * Returns the MongoD version in strig format or an error. Known error codes are: + * ShardNotFound if shard by that id is not available on the registry + * NoSuchKey if the version could not be retrieved + */ +StatusWith<std::string> retrieveShardMongoDVersion(OperationContext* opCtx, ShardId shardId) { + auto shardRegistry = Grid::get(opCtx)->shardRegistry(); + auto shardStatus = shardRegistry->getShard(opCtx, shardId); + if (!shardStatus.isOK()) { + return shardStatus.getStatus(); + } + auto shard = shardStatus.getValue(); + + auto commandResponse = + shard->runCommandWithFixedRetryAttempts(opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + "admin", + BSON("serverStatus" << 1), + Shard::RetryPolicy::kIdempotent); + if (!commandResponse.isOK()) { + return commandResponse.getStatus(); + } + if (!commandResponse.getValue().commandStatus.isOK()) { + return commandResponse.getValue().commandStatus; + } + + BSONObj serverStatus = std::move(commandResponse.getValue().response); + + std::string version; + Status status = bsonExtractStringField(serverStatus, kVersionField, &version); + if (!status.isOK()) { + return status; + } + + return version; +} +} // namespace using ShardStatistics = ClusterStatistics::ShardStatistics; @@ -101,6 +145,21 @@ StatusWith<std::vector<ShardStatistics>> ClusterStatisticsImpl::_getStats( << shard.getName()); } + std::string mongoDVersion; + + auto mongoDVersionStatus = retrieveShardMongoDVersion(opCtx, shard.getName()); + if (mongoDVersionStatus.isOK()) { + mongoDVersion = std::move(mongoDVersionStatus.getValue()); + } else { + // Since the mongod version is only used for reporting, there is no need to fail the + // entire round if it cannot be retrieved, so just leave it empty + LOGV2(21895, + "Unable to obtain shard version for {shardId}: {error}", + "Unable to obtain shard version", + "shardId"_attr = shard.getName(), + "error"_attr = mongoDVersionStatus.getStatus()); + } + std::set<std::string> shardTags; for (const auto& shardTag : shard.getTags()) { @@ -112,6 +171,7 @@ StatusWith<std::vector<ShardStatistics>> ClusterStatisticsImpl::_getStats( shardSizeStatus.getValue(), shard.getDraining(), std::move(shardTags), + std::move(mongoDVersion), ShardStatistics::use_bytes_t{}); } diff --git a/src/mongo/db/s/balancer/cluster_statistics_test.cpp b/src/mongo/db/s/balancer/cluster_statistics_test.cpp index 905943551cb..25bd0bb804d 100644 --- a/src/mongo/db/s/balancer/cluster_statistics_test.cpp +++ b/src/mongo/db/s/balancer/cluster_statistics_test.cpp @@ -40,9 +40,12 @@ using ShardStatistics = ClusterStatistics::ShardStatistics; const auto emptyTagSet = std::set<std::string>(); TEST(ShardStatistics, SizeMaxedTest) { - ASSERT(!ShardStatistics(ShardId("TestShardId"), 0, 0, false, emptyTagSet).isSizeMaxed()); - ASSERT(!ShardStatistics(ShardId("TestShardId"), 100LL, 80LL, false, emptyTagSet).isSizeMaxed()); - ASSERT(ShardStatistics(ShardId("TestShardId"), 100LL, 110LL, false, emptyTagSet).isSizeMaxed()); + ASSERT( + !ShardStatistics(ShardId("TestShardId"), 0, 0, false, emptyTagSet, "3.2.0").isSizeMaxed()); + ASSERT(!ShardStatistics(ShardId("TestShardId"), 100LL, 80LL, false, emptyTagSet, "3.2.0") + .isSizeMaxed()); + ASSERT(ShardStatistics(ShardId("TestShardId"), 100LL, 110LL, false, emptyTagSet, "3.2.0") + .isSizeMaxed()); } } // namespace diff --git a/src/mongo/db/s/balancer/migration_test_fixture.cpp b/src/mongo/db/s/balancer/migration_test_fixture.cpp index 76cf548b860..fdc8b740979 100644 --- a/src/mongo/db/s/balancer/migration_test_fixture.cpp +++ b/src/mongo/db/s/balancer/migration_test_fixture.cpp @@ -162,18 +162,4 @@ void MigrationTestFixture::checkMigrationsCollectionIsEmptyAndLocksAreUnlocked() ASSERT_EQUALS(0U, locksQueryResponse.docs.size()); } -ShardId MigrationTestFixture::getShardIdByHost(HostAndPort host) { - if (host == kShardHost0) { - return kShardId0; - } else if (host == kShardHost1) { - return kShardId1; - } else if (host == kShardHost2) { - return kShardId2; - } else if (host == kShardHost3) { - return kShardId3; - } - - MONGO_UNREACHABLE; -} - } // namespace mongo diff --git a/src/mongo/db/s/balancer/migration_test_fixture.h b/src/mongo/db/s/balancer/migration_test_fixture.h index 3434bee663a..76d13c587d7 100644 --- a/src/mongo/db/s/balancer/migration_test_fixture.h +++ b/src/mongo/db/s/balancer/migration_test_fixture.h @@ -119,11 +119,6 @@ protected: */ void checkMigrationsCollectionIsEmptyAndLocksAreUnlocked(); - /** - * Returns the ShardId by its HostAndPort - */ - ShardId getShardIdByHost(HostAndPort host); - // Random static initialization order can result in X constructor running before Y constructor // if X and Y are defined in different source files. Defining variables here to enforce order. const ShardId kShardId0 = ShardId("shard0"); diff --git a/src/mongo/db/s/balancer_stats_registry.cpp b/src/mongo/db/s/balancer_stats_registry.cpp index 0d262b2c299..0d664c0762c 100644 --- a/src/mongo/db/s/balancer_stats_registry.cpp +++ b/src/mongo/db/s/balancer_stats_registry.cpp @@ -31,7 +31,6 @@ #include "mongo/db/s/balancer_stats_registry.h" -#include "mongo/db/catalog_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/pipeline/aggregate_command_gen.h" #include "mongo/db/repl/replication_coordinator.h" @@ -115,12 +114,9 @@ void BalancerStatsRegistry::initializeAsync(OperationContext* opCtx) { LOGV2_DEBUG(6419601, 2, "Initializing BalancerStatsRegistry"); try { - // Lock the range deleter to prevent concurrent modifications of orphans count - ScopedRangeDeleterLock rangeDeleterLock(opCtx, LockMode::MODE_S); - // The collection lock is needed to serialize with direct writes to - // config.rangeDeletions - AutoGetCollection rangeDeletionLock( - opCtx, NamespaceString::kRangeDeletionNamespace, MODE_S); + // Lock the range deleter to prevent + // concurrent modifications of orphans count + ScopedRangeDeleterLock rangeDeleterLock(opCtx); // Load current ophans count from disk _loadOrphansCount(opCtx); LOGV2_DEBUG(6419602, 2, "Completed BalancerStatsRegistry initialization"); @@ -180,41 +176,6 @@ long long BalancerStatsRegistry::getCollNumOrphanDocs(const UUID& collectionUUID return 0; } -long long BalancerStatsRegistry::getCollNumOrphanDocsFromDiskIfNeeded( - OperationContext* opCtx, const UUID& collectionUUID) const { - try { - return getCollNumOrphanDocs(collectionUUID); - } catch (const ExceptionFor<ErrorCodes::NotYetInitialized>&) { - // Since the registry is not initialized, run an aggregation to get the number of orphans - DBDirectClient client(opCtx); - std::vector<BSONObj> pipeline; - pipeline.push_back( - BSON("$match" << BSON(RangeDeletionTask::kCollectionUuidFieldName << collectionUUID))); - pipeline.push_back( - BSON("$group" << BSON("_id" - << "numOrphans" - << "count" - << BSON("$sum" - << "$" + RangeDeletionTask::kNumOrphanDocsFieldName)))); - AggregateCommandRequest aggRequest(NamespaceString::kRangeDeletionNamespace, pipeline); - auto swCursor = DBClientCursor::fromAggregationRequest( - &client, aggRequest, false /* secondaryOk */, true /* useExhaust */); - if (!swCursor.isOK()) { - return 0; - } - auto cursor = std::move(swCursor.getValue()); - if (!cursor->more()) { - return 0; - } - auto res = cursor->nextSafe(); - invariant(!cursor->more()); - auto numOrphans = res.getField("count"); - invariant(numOrphans); - return numOrphans.exactNumberLong(); - } -} - - void BalancerStatsRegistry::onRangeDeletionTaskInsertion(const UUID& collectionUUID, long long numOrphanDocs) { if (!_isInitialized()) @@ -234,8 +195,7 @@ void BalancerStatsRegistry::onRangeDeletionTaskDeletion(const UUID& collectionUU stdx::lock_guard lk{_mutex}; auto collStatsIt = _collStatsMap.find(collectionUUID); if (collStatsIt == _collStatsMap.end()) { - LOGV2_DEBUG(6419612, - 1, + LOGV2_ERROR(6419612, "Couldn't find cached range deletion tasks count during decrese attempt", "collectionUUID"_attr = collectionUUID, "numOrphanDocs"_attr = numOrphanDocs); @@ -248,8 +208,7 @@ void BalancerStatsRegistry::onRangeDeletionTaskDeletion(const UUID& collectionUU if (stats.numRangeDeletionTasks <= 0) { if (MONGO_unlikely(stats.numRangeDeletionTasks < 0)) { - LOGV2_DEBUG(6419613, - 1, + LOGV2_ERROR(6419613, "Cached count of range deletion tasks became negative. Resetting it to 0", "collectionUUID"_attr = collectionUUID, "numRangeDeletionTasks"_attr = stats.numRangeDeletionTasks, @@ -284,12 +243,9 @@ void BalancerStatsRegistry::updateOrphansCount(const UUID& collectionUUID, long stats.numOrphanDocs += delta; if (stats.numOrphanDocs < 0) { - // This could happen in case of direct manipulation of range deletion tasks documents or - // direct writes into orphaned ranges, but also in some other benign situations. - // numOrphanDocs is a best-effort counter, miscounting or even being negative in some - // scenarios is expected. - LOGV2_DEBUG(6419611, - 1, + // This should happen only in case of direct manipulation of range deletion tasks + // documents or direct writes into orphaned ranges + LOGV2_ERROR(6419611, "Cached orphan documents count became negative, resetting it to 0", "collectionUUID"_attr = collectionUUID, "numOrphanDocs"_attr = stats.numOrphanDocs, @@ -337,8 +293,7 @@ void BalancerStatsRegistry::_loadOrphansCount(OperationContext* opCtx) { auto numRangeDeletionTasks = collObj[kNumRangeDeletionTasksLabel].exactNumberLong(); invariant(numRangeDeletionTasks > 0); if (orphanCount < 0) { - LOGV2_DEBUG(6419621, - 1, + LOGV2_ERROR(6419621, "Found negative orphan count in range deletion task documents", "collectionUUID"_attr = collUUID, "numOrphanDocs"_attr = orphanCount, diff --git a/src/mongo/db/s/balancer_stats_registry.h b/src/mongo/db/s/balancer_stats_registry.h index e35ae419539..6dcfae68609 100644 --- a/src/mongo/db/s/balancer_stats_registry.h +++ b/src/mongo/db/s/balancer_stats_registry.h @@ -29,7 +29,6 @@ #pragma once -#include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/replica_set_aware_service.h" #include "mongo/util/concurrency/thread_pool.h" @@ -38,22 +37,6 @@ namespace mongo { /** - * Scoped lock to synchronize with the execution of range deletions. - * The range-deleter acquires a scoped lock in IX mode while orphans are being deleted. - * Acquiring the scoped lock in MODE_X ensures that no orphan counter in `config.rangeDeletions` - * entries is going to be updated concurrently. - */ -class ScopedRangeDeleterLock { -public: - ScopedRangeDeleterLock(OperationContext* opCtx, LockMode mode) - : _resourceLock(opCtx, opCtx->lockState(), _mutex.getRid(), mode) {} - -private: - const Lock::ResourceLock _resourceLock; - static inline const Lock::ResourceMutex _mutex{"ScopedRangeDeleterLock"}; -}; - -/** * The BalancerStatsRegistry is used to cache metadata on shards, such as the orphan documents * count. The blancer (on the config sever) periodically fetches this metadata through the * _shardsvrGetStatsForBalancing command and uses it to evaluate balancing status of collections. @@ -95,13 +78,6 @@ public: long long getCollNumOrphanDocs(const UUID& collectionUUID) const; - /** - * Retrieves the numOrphanDocs from the balancer stats registry if initialized or runs an - * aggregation on disk if not. - */ - long long getCollNumOrphanDocsFromDiskIfNeeded(OperationContext* opCtx, - const UUID& collectionUUID) const; - private: void onInitialDataAvailable(OperationContext* opCtx, bool isMajorityDataAvailable) override final {} diff --git a/src/mongo/db/s/check_sharding_index_command.cpp b/src/mongo/db/s/check_sharding_index_command.cpp index dd5331e33f1..16dbbf8a2fe 100644 --- a/src/mongo/db/s/check_sharding_index_command.cpp +++ b/src/mongo/db/s/check_sharding_index_command.cpp @@ -94,15 +94,15 @@ public: return false; } - std::string tmpErrMsg = "couldn't find valid index for shard key"; auto shardKeyIdx = findShardKeyPrefixedIndex(opCtx, *collection, collection->getIndexCatalog(), keyPattern, - /*requireSingleKey=*/true, - &tmpErrMsg); - - uassert(ErrorCodes::InvalidOptions, str::stream() << tmpErrMsg, shardKeyIdx); + /*requireSingleKey=*/true); + if (!shardKeyIdx) { + errmsg = "couldn't find valid index for shard key"; + return false; + } return true; } diff --git a/src/mongo/db/s/chunk_splitter.cpp b/src/mongo/db/s/chunk_splitter.cpp index cd7d5c897a7..f61abc59eaf 100644 --- a/src/mongo/db/s/chunk_splitter.cpp +++ b/src/mongo/db/s/chunk_splitter.cpp @@ -336,6 +336,7 @@ void ChunkSplitter::_runAutosplit(std::shared_ptr<ChunkSplitStateDriver> chunkSp } return balancerConfig->getMaxChunkSizeBytes(); }(); + invariant(ChunkSizeSettingsType::checkMaxChunkSizeValid(maxChunkSizeBytes)); LOGV2_DEBUG(21906, 1, diff --git a/src/mongo/db/s/collection_metadata.cpp b/src/mongo/db/s/collection_metadata.cpp index 6b0fb689356..ea9f6f0c7c1 100644 --- a/src/mongo/db/s/collection_metadata.cpp +++ b/src/mongo/db/s/collection_metadata.cpp @@ -107,13 +107,13 @@ void CollectionMetadata::throwIfReshardingInProgress(NamespaceString const& nss) } } -BSONObj CollectionMetadata::extractDocumentKey(const ShardKeyPattern* shardKeyPattern, - const BSONObj& doc) { +BSONObj CollectionMetadata::extractDocumentKey(const BSONObj& doc) const { BSONObj key; - if (shardKeyPattern) { - key = dotted_path_support::extractElementsBasedOnTemplate(doc, shardKeyPattern->toBSON()); - if (shardKeyPattern->hasId()) { + if (isSharded()) { + auto const& pattern = _cm->getShardKeyPattern(); + key = dotted_path_support::extractElementsBasedOnTemplate(doc, pattern.toBSON()); + if (pattern.hasId()) { return key; } // else, try to append an _id field from the document. @@ -127,10 +127,6 @@ BSONObj CollectionMetadata::extractDocumentKey(const ShardKeyPattern* shardKeyPa return doc; } -BSONObj CollectionMetadata::extractDocumentKey(const BSONObj& doc) const { - return extractDocumentKey(isSharded() ? &_cm->getShardKeyPattern() : nullptr, doc); -} - std::string CollectionMetadata::toStringBasic() const { if (isSharded()) { return str::stream() << "collection version: " << _cm->getVersion().toString() diff --git a/src/mongo/db/s/collection_metadata.h b/src/mongo/db/s/collection_metadata.h index c8f5f110e6a..b691f94ebe5 100644 --- a/src/mongo/db/s/collection_metadata.h +++ b/src/mongo/db/s/collection_metadata.h @@ -93,21 +93,7 @@ public: } /** - * Returns the current shard's latest placement timestamp or Timestamp(0, 0) if it is not - * sharded. This value indicates the commit time of the latest placement change that this shard - * participated in and is used to answer the question of "did any chunks move since some - * timestamp". - * - * Will throw ShardInvalidatedForTargeting if _thisShardId is marked as stale by - * the CollectionMetadata's current chunk manager. - */ - Timestamp getShardMaxValidAfter() const { - return (isSharded() ? _cm->getMaxValidAfter(_thisShardId) : Timestamp(0, 0)); - } - - /** - * Returns the current shard's placement version for the collection or UNSHARDED if it is not - * sharded. + * Returns the current shard version for the collection or UNSHARDED if it is not sharded. * * Will not throw an exception if _thisShardId is marked as stale by the CollectionMetadata's * current chunk manager. Only use this function when logging the returned ChunkVersion. If the @@ -176,12 +162,6 @@ public: BSONObj extractDocumentKey(const BSONObj& doc) const; /** - * Static version of the function above. Only use this for internal sharding operations where - * shard key pattern is fixed and cannot change. - */ - static BSONObj extractDocumentKey(const ShardKeyPattern* shardKeyPattern, const BSONObj& doc); - - /** * String output of the collection and shard versions. */ std::string toStringBasic() const; diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp index 72e89aa8804..8f789549796 100644 --- a/src/mongo/db/s/collection_metadata_test.cpp +++ b/src/mongo/db/s/collection_metadata_test.cpp @@ -296,7 +296,7 @@ TEST_F(SingleChunkMinMaxCompoundKeyFixture, KeyBelongsToMe) { ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << MINKEY << "b" << 10))); ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << 10 << "b" << 20))); - ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << MAXKEY << "b" << MAXKEY))); + ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSON("a" << MAXKEY << "b" << MAXKEY))); ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSONObj())); } @@ -378,10 +378,10 @@ TEST_F(ThreeChunkWithRangeGapFixture, KeyBelongsToMe) { ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << 10))); ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << 30))); ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << 40))); - ASSERT(makeCollectionMetadata().keyBelongsToMe(BSON("a" << MAXKEY))); ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSON("a" << 20))); ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSON("a" << 25))); + ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSON("a" << MAXKEY))); ASSERT(!makeCollectionMetadata().keyBelongsToMe(BSONObj())); } diff --git a/src/mongo/db/s/collection_sharding_runtime.cpp b/src/mongo/db/s/collection_sharding_runtime.cpp index 4c5950b4f75..91a8d3d6090 100644 --- a/src/mongo/db/s/collection_sharding_runtime.cpp +++ b/src/mongo/db/s/collection_sharding_runtime.cpp @@ -102,9 +102,8 @@ ScopedCollectionFilter CollectionShardingRuntime::getOwnershipFilter( if (!supportNonVersionedOperations) { optReceivedShardVersion = getOperationReceivedVersion(opCtx, _nss); // No operations should be calling getOwnershipFilter without a shard version - tassert(7032300, - "getOwnershipFilter called by operation that doesn't specify shard version", - optReceivedShardVersion); + invariant(optReceivedShardVersion, + "getOwnershipFilter called by operation that doesn't specify shard version"); } auto metadata = @@ -112,6 +111,13 @@ ScopedCollectionFilter CollectionShardingRuntime::getOwnershipFilter( repl::ReadConcernArgs::get(opCtx).getArgsAtClusterTime(), supportNonVersionedOperations); + if (!supportNonVersionedOperations) { + invariant(!ChunkVersion::isIgnoredVersion(*optReceivedShardVersion) || + !metadata->get().allowMigrations() || !metadata->get().isSharded(), + "For sharded collections getOwnershipFilter cannot be relied on without a valid " + "shard version"); + } + return {std::move(metadata)}; } @@ -157,10 +163,6 @@ void CollectionShardingRuntime::checkShardVersionOrThrow(OperationContext* opCtx void CollectionShardingRuntime::enterCriticalSectionCatchUpPhase(const CSRLock&, const BSONObj& reason) { _critSec.enterCriticalSectionCatchUpPhase(reason); - - if (_shardVersionInRecoverOrRefresh) { - _shardVersionInRecoverOrRefresh->cancellationSource.cancel(); - } } void CollectionShardingRuntime::enterCriticalSectionCommitPhase(const CSRLock&, @@ -196,9 +198,8 @@ void CollectionShardingRuntime::setFilteringMetadata(OperationContext* opCtx, void CollectionShardingRuntime::setFilteringMetadata_withLock(OperationContext* opCtx, CollectionMetadata newMetadata, const CSRLock& csrExclusiveLock) { - tassert(7032302, - str::stream() << "Namespace " << _nss.ns() << " must never be sharded.", - !newMetadata.isSharded() || !_nss.isNamespaceAlwaysUnsharded()); + invariant(!newMetadata.isSharded() || !_nss.isNamespaceAlwaysUnsharded(), + str::stream() << "Namespace " << _nss.ns() << " must never be sharded."); stdx::lock_guard lk(_metadataManagerLock); @@ -210,11 +211,9 @@ void CollectionShardingRuntime::setFilteringMetadata_withLock(OperationContext* _metadataType = MetadataType::kUnsharded; _metadataManager.reset(); ++_numMetadataManagerChanges; - return; - } - - _metadataType = MetadataType::kSharded; - if (!_metadataManager || !newMetadata.uuidMatches(_metadataManager->getCollectionUuid())) { + } else if (!_metadataManager || + !newMetadata.uuidMatches(_metadataManager->getCollectionUuid())) { + _metadataType = MetadataType::kSharded; _metadataManager = std::make_shared<MetadataManager>( opCtx->getServiceContext(), _nss, _rangeDeleterExecutor, newMetadata); ++_numMetadataManagerChanges; @@ -223,8 +222,7 @@ void CollectionShardingRuntime::setFilteringMetadata_withLock(OperationContext* } } -void CollectionShardingRuntime::_clearFilteringMetadata(OperationContext* opCtx, - bool clearMetadataManager) { +void CollectionShardingRuntime::clearFilteringMetadata(OperationContext* opCtx) { const auto csrLock = CSRLock::lockExclusive(opCtx, this); if (_shardVersionInRecoverOrRefresh) { _shardVersionInRecoverOrRefresh->cancellationSource.cancel(); @@ -236,23 +234,12 @@ void CollectionShardingRuntime::_clearFilteringMetadata(OperationContext* opCtx, 1, "Clearing metadata for collection {namespace}", "Clearing collection metadata", - "namespace"_attr = _nss, - "clearMetadataManager"_attr = clearMetadataManager); + "namespace"_attr = _nss); _metadataType = MetadataType::kUnknown; - if (clearMetadataManager) - _metadataManager.reset(); + _metadataManager.reset(); } } -void CollectionShardingRuntime::clearFilteringMetadata(OperationContext* opCtx) { - _clearFilteringMetadata(opCtx, /* clearMetadataManager */ false); -} - -void CollectionShardingRuntime::clearFilteringMetadataForDroppedCollection( - OperationContext* opCtx) { - _clearFilteringMetadata(opCtx, /* clearMetadataManager */ true); -} - SharedSemiFuture<void> CollectionShardingRuntime::cleanUpRange(ChunkRange const& range, const UUID& migrationId, CleanWhen when) { @@ -276,7 +263,7 @@ Status CollectionShardingRuntime::waitForClean(OperationContext* opCtx, // If the metadata was reset, or the collection was dropped and recreated since the // metadata manager was created, return an error. - if (self->_metadataType != MetadataType::kSharded || + if (!self->_metadataManager || (collectionUuid != self->_metadataManager->getCollectionUuid())) { return {ErrorCodes::ConflictingOperationInProgress, "Collection being migrated was dropped and created or otherwise had its " @@ -434,7 +421,7 @@ void CollectionShardingRuntime::appendShardVersion(BSONObjBuilder* builder) { size_t CollectionShardingRuntime::numberOfRangesScheduledForDeletion() const { stdx::lock_guard lk(_metadataManagerLock); - if (_metadataType == MetadataType::kSharded) { + if (_metadataManager) { return _metadataManager->numberOfRangesScheduledForDeletion(); } return 0; @@ -474,9 +461,7 @@ CollectionCriticalSection::CollectionCriticalSection(OperationContext* opCtx, Milliseconds(migrationLockAcquisitionMaxWaitMS.load())); auto* const csr = CollectionShardingRuntime::get(_opCtx, _nss); auto csrLock = CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr); - tassert(7032305, - "Collection metadata unknown when entering critical section", - csr->getCurrentMetadataIfKnown()); + invariant(csr->getCurrentMetadataIfKnown()); csr->enterCriticalSectionCatchUpPhase(csrLock, _reason); } @@ -497,9 +482,7 @@ void CollectionCriticalSection::enterCommitPhase() { Milliseconds(migrationLockAcquisitionMaxWaitMS.load())); auto* const csr = CollectionShardingRuntime::get(_opCtx, _nss); auto csrLock = CollectionShardingRuntime::CSRLock::lockExclusive(_opCtx, csr); - tassert(7032304, - "Collection metadata unknown when entering critical section commit phase", - csr->getCurrentMetadataIfKnown()); + invariant(csr->getCurrentMetadataIfKnown()); csr->enterCriticalSectionCommitPhase(csrLock, _reason); } diff --git a/src/mongo/db/s/collection_sharding_runtime.h b/src/mongo/db/s/collection_sharding_runtime.h index c356e3bd6b7..76954af1b61 100644 --- a/src/mongo/db/s/collection_sharding_runtime.h +++ b/src/mongo/db/s/collection_sharding_runtime.h @@ -113,25 +113,18 @@ public: * Marks the collection's filtering metadata as UNKNOWN, meaning that all attempts to check for * shard version match will fail with StaleConfig errors in order to trigger an update. * - * Interrupts any ongoing shard metadata refresh. - * * It is safe to call this method with only an intent lock on the collection (as opposed to - * setFilteringMetadata which requires exclusive). + * setFilteringMetadata which requires exclusive), however note that clearing a collection's + * filtering metadata will interrupt all in-progress orphan cleanups in which case orphaned data + * will remain behind on disk. */ void clearFilteringMetadata(OperationContext* opCtx); /** - * Calls to clearFilteringMetadata + clears the _metadataManager object. - */ - void clearFilteringMetadataForDroppedCollection(OperationContext* opCtx); - - /** * Methods to control the collection's critical section. Methods listed below must be called * with both the collection lock and CSRLock held in exclusive mode. * * In these methods, the CSRLock ensures concurrent access to the critical section. - * - * Entering into the Critical Section interrupts any ongoing filtering metadata refresh. */ void enterCriticalSectionCatchUpPhase(const CSRLock&, const BSONObj& reason); void enterCriticalSectionCommitPhase(const CSRLock&, const BSONObj& reason); @@ -255,11 +248,6 @@ private: const boost::optional<mongo::LogicalTime>& atClusterTime, bool supportNonVersionedOperations = false); - /** - * Auxiliary function used to implement the different flavours of clearFilteringMetadata. - */ - void _clearFilteringMetadata(OperationContext* opCtx, bool clearFilteringMetadata); - // The service context under which this instance runs ServiceContext* const _serviceContext; @@ -285,26 +273,10 @@ private: // Tracks whether the filtering metadata is unknown, unsharded, or sharded enum class MetadataType { kUnknown, kUnsharded, kSharded } _metadataType; - // If the collection state is known and is unsharded, this will be nullptr. - // - // If the collection state is known and is sharded, this will point to the metadata associated - // with this collection. - // - // If the collection state is unknown: - // - If the metadata had never been set yet, this will be nullptr. - // - If the collection state was known and was sharded, this contains the metadata that - // were known for the collection before the last invocation of clearFilteringMetadata(). + // If the collection is sharded, contains all the metadata associated with this collection. // - // The following matrix enumerates the valid (Y) and invalid (X) scenarios. - // _________________________________ - // | _metadataType (collection state)| - // |_________________________________| - // | UNKNOWN | UNSHARDED | SHARDED | - // _______________________|_________|___________|___________| - // |_metadataManager unset | Y | Y | X | - // |_______________________|_________|___________|___________| - // |_metadataManager set | Y | X | Y | - // |_______________________|_________|___________|___________| + // If the collection is unsharded, the metadata has not been set yet, or the metadata has been + // specifically reset by calling clearFilteringMetadata(), this will be nullptr; std::shared_ptr<MetadataManager> _metadataManager; // Used for testing to check the number of times a new MetadataManager has been installed. diff --git a/src/mongo/db/s/collection_sharding_runtime_test.cpp b/src/mongo/db/s/collection_sharding_runtime_test.cpp index d4cc012de28..dcee5b73ac0 100644 --- a/src/mongo/db/s/collection_sharding_runtime_test.cpp +++ b/src/mongo/db/s/collection_sharding_runtime_test.cpp @@ -289,16 +289,14 @@ public: StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { return repl::OpTimeWith<std::vector<ShardType>>(_shards); } - std::vector<CollectionType> getCollections(OperationContext* opCtx, - StringData dbName, - repl::ReadConcernLevel readConcernLevel, - const BSONObj& sort) override { + std::vector<CollectionType> getCollections( + OperationContext* opCtx, + StringData dbName, + repl::ReadConcernLevel readConcernLevel) override { return _colls; } @@ -541,43 +539,5 @@ TEST_F(CollectionShardingRuntimeWithRangeDeleterTest, ASSERT(cleanupComplete.isReady()); } -TEST_F(CollectionShardingRuntimeWithRangeDeleterTest, - WaitForCleanCorrectEvenAfterClearFollowedBySetFilteringMetadata) { - globalFailPointRegistry().find("suspendRangeDeletion")->setMode(FailPoint::alwaysOn); - ScopeGuard resetFailPoint( - [=] { globalFailPointRegistry().find("suspendRangeDeletion")->setMode(FailPoint::off); }); - - OperationContext* opCtx = operationContext(); - auto metadata = makeShardedMetadata(opCtx, uuid()); - csr().setFilteringMetadata(opCtx, metadata); - const ChunkRange range = ChunkRange(BSON(kShardKey << MINKEY), BSON(kShardKey << MAXKEY)); - const auto task = insertRangeDeletionTask(opCtx, kTestNss, uuid(), range, 0); - - // Schedule range deletion that will hang due to `suspendRangeDeletion` failpoint - auto cleanupComplete = - csr().cleanUpRange(range, task.getId(), CollectionShardingRuntime::CleanWhen::kNow); - - // Clear and set again filtering metadata - csr().clearFilteringMetadata(opCtx); - csr().setFilteringMetadata(opCtx, metadata); - - auto waitForCleanUp = [&](Date_t timeout) { - return CollectionShardingRuntime::waitForClean(opCtx, kTestNss, uuid(), range, timeout); - }; - - // Check that the hanging range deletion is still tracked even following a clear of the metadata - auto status = waitForCleanUp(Date_t::now() + Milliseconds(100)); - ASSERT_NOT_OK(status); - ASSERT(!cleanupComplete.isReady()); - - globalFailPointRegistry().find("suspendRangeDeletion")->setMode(FailPoint::off); - resetFailPoint.dismiss(); - - // Check that the range deletion is not tracked anymore after it succeeds - status = waitForCleanUp(Date_t::max()); - ASSERT_OK(status); - ASSERT(cleanupComplete.isReady()); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/collection_sharding_state.h b/src/mongo/db/s/collection_sharding_state.h index 7684847cfbc..575ccb85f2f 100644 --- a/src/mongo/db/s/collection_sharding_state.h +++ b/src/mongo/db/s/collection_sharding_state.h @@ -98,7 +98,7 @@ public: /** * If the shard currently doesn't know whether the collection is sharded or not, it will throw a - * StaleConfig error. + * StaleConfig exception. * * If the request doesn't have a shard version all collections will be treated as UNSHARDED. * @@ -112,7 +112,7 @@ public: * * If the shard currently doesn't know whether the collection is sharded or not, or if the * expected shard version doesn't match with the one in the OperationShardingState, it will - * throw a StaleConfig error. + * throw a StaleConfig exception. * * If the operation context contains an 'atClusterTime', the returned filtering object will be * tied to a specific point in time. Otherwise, it will reference the latest cluster time @@ -143,7 +143,7 @@ public: /** * Checks whether the shard version in the operation context is compatible with the shard - * version of the collection and if not, throws StaleConfig error populated with the received + * version of the collection and if not, throws StaleConfigException populated with the received * and wanted versions. * * If the request is not versioned all collections will be treated as UNSHARDED. diff --git a/src/mongo/db/s/collmod_coordinator.cpp b/src/mongo/db/s/collmod_coordinator.cpp index 2dbf5e176e3..9e75175964f 100644 --- a/src/mongo/db/s/collmod_coordinator.cpp +++ b/src/mongo/db/s/collmod_coordinator.cpp @@ -31,7 +31,6 @@ #include "mongo/db/s/collmod_coordinator.h" -#include "mongo/db/catalog/coll_mod.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/collection_uuid_mismatch.h" #include "mongo/db/catalog/database_holder.h" @@ -40,7 +39,6 @@ #include "mongo/db/ops/insert.h" #include "mongo/db/s/participant_block_gen.h" #include "mongo/db/s/sharded_collmod_gen.h" -#include "mongo/db/s/sharding_ddl_coordinator_gen.h" #include "mongo/db/s/sharding_ddl_util.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/timeseries/catalog_helper.h" @@ -48,6 +46,7 @@ #include "mongo/db/timeseries/timeseries_options.h" #include "mongo/idl/idl_parser.h" #include "mongo/logv2/log.h" +#include "mongo/s/async_requests_sender.h" #include "mongo/s/cluster_commands_helpers.h" #include "mongo/s/grid.h" #include "mongo/util/fail_point.h" @@ -135,13 +134,7 @@ void CollModCoordinator::_enterPhase(Phase newPhase) { if (_doc.getPhase() == Phase::kUnset) { newDoc = _insertStateDocument(std::move(newDoc)); } else { - ServiceContext::UniqueOperationContext uniqueOpCtx; - auto opCtx = cc().getOperationContext(); - if (!opCtx) { - uniqueOpCtx = cc().makeOperationContext(); - opCtx = uniqueOpCtx.get(); - } - newDoc = _updateStateDocument(opCtx, std::move(newDoc)); + newDoc = _updateStateDocument(cc().makeOperationContext().get(), std::move(newDoc)); } { @@ -180,85 +173,19 @@ void CollModCoordinator::_saveShardingInfoOnCoordinatorIfNecessary(OperationCont 6522700, "Sharding information must be gathered after collection information", _collInfo); if (!_shardingInfo && _collInfo->isSharded) { ShardingInfo info; - info.isPrimaryOwningChunks = false; const auto chunkManager = uassertStatusOK(Grid::get(opCtx)->catalogCache()->getCollectionRoutingInfoWithRefresh( opCtx, _collInfo->nsForTargeting)); - // Coordinator is guaranteed to be running on primary shard - info.primaryShard = ShardingState::get(opCtx)->shardId(); - + info.primaryShard = chunkManager.dbPrimary(); std::set<ShardId> shardIdsSet; chunkManager.getAllShardIds(&shardIdsSet); - std::vector<ShardId> participantsNotOwningChunks; - - std::vector<ShardId> shardIdsVec; - shardIdsVec.reserve(shardIdsSet.size()); - for (const auto& shard : shardIdsSet) { - if (shard != info.primaryShard) { - shardIdsVec.push_back(shard); - } else { - info.isPrimaryOwningChunks = true; - } - } - - auto allShards = Grid::get(opCtx)->shardRegistry()->getAllShardIds(opCtx); - for (const auto& shard : allShards) { - if (std::find(shardIdsVec.begin(), shardIdsVec.end(), shard) == shardIdsVec.end() && - shard != info.primaryShard) { - participantsNotOwningChunks.push_back(shard); - } - } - - info.participantsOwningChunks = std::move(shardIdsVec); - info.participantsNotOwningChunks = std::move(participantsNotOwningChunks); + std::vector<ShardId> shardIdsVec{shardIdsSet.begin(), shardIdsSet.end()}; + info.shardsOwningChunks = std::move(shardIdsVec); _shardingInfo = std::move(info); } } -std::vector<AsyncRequestsSender::Response> CollModCoordinator::_sendCollModToPrimaryShard( - OperationContext* opCtx, - ShardsvrCollModParticipant& request, - const std::shared_ptr<executor::ScopedTaskExecutor>& executor) { - // A view definition will only be present on the primary shard. So we pass an addition - // 'performViewChange' flag only to the primary shard. - request.setPerformViewChange(true); - - return sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, - nss().db(), - CommandHelpers::appendMajorityWriteConcern(request.toBSON({})), - {_shardingInfo->primaryShard}, - **executor, - _shardingInfo->isPrimaryOwningChunks); -} - -std::vector<AsyncRequestsSender::Response> CollModCoordinator::_sendCollModToParticipantShards( - OperationContext* opCtx, - ShardsvrCollModParticipant& request, - const std::shared_ptr<executor::ScopedTaskExecutor>& executor) { - request.setPerformViewChange(false); - - // The collMod command targets all shards, regardless of whether they have chunks. The shards - // that have no chunks for the collection will not throw nor will be included in the responses. - - sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, - nss().db(), - CommandHelpers::appendMajorityWriteConcern(request.toBSON({})), - _shardingInfo->participantsNotOwningChunks, - **executor, - false /* throwOnError */); - - return sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, - nss().db(), - CommandHelpers::appendMajorityWriteConcern(request.toBSON({})), - _shardingInfo->participantsOwningChunks, - **executor, - true /* throwOnError */); -} - ExecutorFuture<void> CollModCoordinator::_runImpl( std::shared_ptr<executor::ScopedTaskExecutor> executor, const CancellationToken& token) noexcept { @@ -305,37 +232,19 @@ ExecutorFuture<void> CollModCoordinator::_runImpl( _saveCollectionInfoOnCoordinatorIfNecessary(opCtx); if (_collInfo->isSharded) { - const auto migrationsAlreadyBlockedForBucketNss = - hasTimeSeriesGranularityUpdate(_request) && - _doc.getMigrationsAlreadyBlockedForBucketNss(); - - if (!migrationsAlreadyBlockedForBucketNss) { - _doc.setCollUUID(sharding_ddl_util::getCollectionUUID( - opCtx, _collInfo->nsForTargeting, true /* allowViews */)); - sharding_ddl_util::stopMigrations( - opCtx, _collInfo->nsForTargeting, _doc.getCollUUID()); - } + _doc.setCollUUID( + sharding_ddl_util::getCollectionUUID(opCtx, nss(), true /* allowViews */)); + sharding_ddl_util::stopMigrations(opCtx, nss(), _doc.getCollUUID()); } _saveShardingInfoOnCoordinatorIfNecessary(opCtx); if (_collInfo->isSharded && hasTimeSeriesGranularityUpdate(_request)) { - { - // Persist the migrationAlreadyBlocked flag on the coordinator document - auto newDoc = _doc; - newDoc.setMigrationsAlreadyBlockedForBucketNss(true); - _updateStateDocument(opCtx, std::move(newDoc)); - } - ShardsvrParticipantBlock blockCRUDOperationsRequest(_collInfo->nsForTargeting); const auto cmdObj = CommandHelpers::appendMajorityWriteConcern( blockCRUDOperationsRequest.toBSON({})); - std::vector<ShardId> shards = _shardingInfo->participantsOwningChunks; - if (_shardingInfo->isPrimaryOwningChunks) { - shards.push_back(_shardingInfo->primaryShard); - } sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, nss().db(), cmdObj, shards, **executor, true /* throwOnError */); + opCtx, nss().db(), cmdObj, _shardingInfo->shardsOwningChunks, **executor); } })) .then(_executePhase( @@ -380,92 +289,51 @@ ExecutorFuture<void> CollModCoordinator::_runImpl( _saveShardingInfoOnCoordinatorIfNecessary(opCtx); if (_collInfo->isSharded) { - try { - if (!_firstExecution) { - bool allowMigrations = sharding_ddl_util::checkAllowMigrations( - opCtx, _collInfo->nsForTargeting); - if (_result.is_initialized() && allowMigrations) { - // The command finished and we have the response. Return it. - return; - } else if (allowMigrations) { - // Previous run on a different node completed, but we lost the - // result in the stepdown. Restart from stage in which we disallow - // migrations. - _enterPhase(Phase::kBlockShards); - uasserted(ErrorCodes::Interrupted, - "Retriable error to move to previous stage"); - } - } - - ShardsvrCollModParticipant request(nss(), _request); - bool needsUnblock = _collInfo->timeSeriesOptions && - hasTimeSeriesGranularityUpdate(_request); - request.setNeedsUnblock(needsUnblock); - - // If trying to convert an index to unique, executes a dryRun first to find - // any duplicates without actually changing the indexes to avoid - // inconsistent index specs on different shards. Example: - // Shard0: {_id: 0, a: 1} - // Shard1: {_id: 1, a: 2}, {_id: 2, a: 2} - // When trying to convert index {a: 1} to unique, the dry run will return - // the duplicate errors to the user without converting the indexes. - if (isCollModIndexUniqueConversion(_request)) { - // The 'dryRun' option only works with 'unique' index option. We need to - // strip out other incompatible options. - auto dryRunRequest = ShardsvrCollModParticipant{ - nss(), makeCollModDryRunRequest(_request)}; - std::vector<ShardId> shards = _shardingInfo->participantsOwningChunks; - if (_shardingInfo->isPrimaryOwningChunks) { - shards.push_back(_shardingInfo->primaryShard); - } + ShardsvrCollModParticipant request(nss(), _request); + bool needsUnblock = + _collInfo->timeSeriesOptions && hasTimeSeriesGranularityUpdate(_request); + request.setNeedsUnblock(needsUnblock); + + std::vector<AsyncRequestsSender::Response> responses; + auto shardsOwningChunks = _shardingInfo->shardsOwningChunks; + auto primaryShardOwningChunk = std::find(shardsOwningChunks.begin(), + shardsOwningChunks.end(), + _shardingInfo->primaryShard); + // A view definition will only be present on the primary shard. So we pass an + // addition 'performViewChange' flag only to the primary shard. + if (primaryShardOwningChunk != shardsOwningChunks.end()) { + request.setPerformViewChange(true); + const auto& primaryResponse = sharding_ddl_util::sendAuthenticatedCommandToShards( opCtx, nss().db(), - CommandHelpers::appendMajorityWriteConcern( - dryRunRequest.toBSON({})), - shards, + CommandHelpers::appendMajorityWriteConcern(request.toBSON({})), + {_shardingInfo->primaryShard}, **executor); - } - - std::vector<AsyncRequestsSender::Response> responses; - - // In the case of the participants, we are broadcasting the collMod to all - // the shards. On one hand, if the shard contains chunks for the - // collections, we parse all the responses. On the other hand, if the shard - // does not contain chunks, we make a best effort to not process the - // returned responses or throw any errors. - - auto primaryResponse = _sendCollModToPrimaryShard(opCtx, request, executor); - if (_shardingInfo->isPrimaryOwningChunks) { - responses.insert(responses.end(), - std::make_move_iterator(primaryResponse.begin()), - std::make_move_iterator(primaryResponse.end())); - } - - auto participantsResponses = - _sendCollModToParticipantShards(opCtx, request, executor); - responses.insert(responses.end(), - std::make_move_iterator(participantsResponses.begin()), - std::make_move_iterator(participantsResponses.end())); - - - BSONObjBuilder builder; - std::string errmsg; - auto ok = - appendRawResponses(opCtx, &errmsg, &builder, responses).responseOK; - if (!errmsg.empty()) { - CommandHelpers::appendSimpleCommandStatus(builder, ok, errmsg); - } - _result = builder.obj(); - sharding_ddl_util::resumeMigrations( - opCtx, _collInfo->nsForTargeting, _doc.getCollUUID()); - } catch (DBException& ex) { - if (!_isRetriableErrorForDDLCoordinator(ex.toStatus())) { - sharding_ddl_util::resumeMigrations( - opCtx, _collInfo->nsForTargeting, _doc.getCollUUID()); - } - throw; + responses.insert( + responses.end(), primaryResponse.begin(), primaryResponse.end()); + shardsOwningChunks.erase(primaryShardOwningChunk); } + + request.setPerformViewChange(false); + const auto& secondaryResponses = + sharding_ddl_util::sendAuthenticatedCommandToShards( + opCtx, + nss().db(), + CommandHelpers::appendMajorityWriteConcern(request.toBSON({})), + shardsOwningChunks, + **executor); + responses.insert( + responses.end(), secondaryResponses.begin(), secondaryResponses.end()); + + BSONObjBuilder builder; + std::string errmsg; + auto ok = appendRawResponses(opCtx, &errmsg, &builder, responses).responseOK; + if (!errmsg.empty()) { + CommandHelpers::appendSimpleCommandStatus(builder, ok, errmsg); + } + _result = builder.obj(); + sharding_ddl_util::resumeMigrations(opCtx, nss(), _doc.getCollUUID()); } else { CollMod cmd(nss()); cmd.setCollModRequest(_request); @@ -493,6 +361,15 @@ ExecutorFuture<void> CollModCoordinator::_runImpl( "Error running collMod", "namespace"_attr = nss(), "error"_attr = redact(status)); + // If we have the collection UUID set, this error happened in a sharded collection, + // we should restore the migrations. + if (_doc.getCollUUID()) { + auto opCtxHolder = cc().makeOperationContext(); + auto* opCtx = opCtxHolder.get(); + getForwardableOpMetadata().setOn(opCtx); + + sharding_ddl_util::resumeMigrations(opCtx, nss(), _doc.getCollUUID()); + } } return status; }); diff --git a/src/mongo/db/s/collmod_coordinator.h b/src/mongo/db/s/collmod_coordinator.h index 941e4187001..b85b6b16d5a 100644 --- a/src/mongo/db/s/collmod_coordinator.h +++ b/src/mongo/db/s/collmod_coordinator.h @@ -30,9 +30,7 @@ #pragma once #include "mongo/db/s/collmod_coordinator_document_gen.h" -#include "mongo/db/s/sharded_collmod_gen.h" #include "mongo/db/s/sharding_ddl_coordinator.h" -#include "mongo/s/async_requests_sender.h" #include "mongo/s/request_types/sharded_ddl_commands_gen.h" namespace mongo { @@ -72,14 +70,8 @@ private: struct ShardingInfo { // The primary shard for the collection, only set if the collection is sharded. ShardId primaryShard; - // Flag that tells if the primary db shard has chunks for the collection. - bool isPrimaryOwningChunks; - // The participant shards owning chunks for the collection, only set if the collection is - // sharded. - std::vector<ShardId> participantsOwningChunks; - // The participant shards not owning chunks for the collection, only set if the collection - // is sharded. - std::vector<ShardId> participantsNotOwningChunks; + // The shards owning chunks for the collection, only set if the collection is sharded. + std::vector<ShardId> shardsOwningChunks; }; ShardingDDLCoordinatorMetadata const& metadata() const override { @@ -119,16 +111,6 @@ private: mutable Mutex _docMutex = MONGO_MAKE_LATCH("CollModCoordinator::_docMutex"); CollModCoordinatorDocument _doc; - std::vector<AsyncRequestsSender::Response> _sendCollModToPrimaryShard( - OperationContext* opCtx, - ShardsvrCollModParticipant& request, - const std::shared_ptr<executor::ScopedTaskExecutor>& executor); - - std::vector<AsyncRequestsSender::Response> _sendCollModToParticipantShards( - OperationContext* opCtx, - ShardsvrCollModParticipant& request, - const std::shared_ptr<executor::ScopedTaskExecutor>& executor); - const mongo::CollModRequest _request; boost::optional<BSONObj> _result; diff --git a/src/mongo/db/s/collmod_coordinator_document.idl b/src/mongo/db/s/collmod_coordinator_document.idl index afe91c133dc..1f2bc418e34 100644 --- a/src/mongo/db/s/collmod_coordinator_document.idl +++ b/src/mongo/db/s/collmod_coordinator_document.idl @@ -68,8 +68,3 @@ structs: type: uuid description: "Collection uuid." optional: true - # TODO SERVER-68008 remove once 7.0 becomes last LTS - migrationsAlreadyBlockedForBucketNss: - type: safeBool - description: "Flag set by the coordinator before acquiring critical section on bucket namespace" - optional: true diff --git a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp index 3a1e04c017b..5ce4678fc5d 100644 --- a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp +++ b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp @@ -46,8 +46,6 @@ namespace mongo { namespace { -MONGO_FAIL_POINT_DEFINE(fleCompactHangAfterDropTempCollection); - const auto kMajorityWriteConcern = BSON("writeConcern" << BSON("w" << "majority")); /** @@ -175,21 +173,10 @@ void doDropOperation(const CompactStructuredEncryptionDataState& state) { "Cannot drop temporary encrypted compaction collection due to missing collection UUID", state.getEcocRenameUuid().has_value()); - auto opCtx = cc().makeOperationContext(); - auto catalog = CollectionCatalog::get(opCtx.get()); auto ecocNss = state.getEcocRenameNss(); - auto ecocUuid = catalog->lookupUUIDByNSS(opCtx.get(), ecocNss); - - if (!ecocUuid) { - LOGV2_DEBUG( - 6790901, - 1, - "Skipping drop operation as temporary encrypted compaction collection does not exist"); - return; - } - Drop cmd(ecocNss); cmd.setCollectionUUID(state.getEcocRenameUuid().value()); + auto opCtx = cc().makeOperationContext(); doRunCommand(opCtx.get(), ecocNss.db(), cmd); } @@ -242,16 +229,6 @@ void CompactStructuredEncryptionDataCoordinator::_enterPhase(Phase newPhase) { StateDoc doc(_doc); doc.setPhase(newPhase); - // This coordinator persists the result of the doCompactOperation() - // by reusing the compactionTokens field to store the _response BSON. - // If newPhase is kDropTempCollection, the compactionTokens field is replaced - // on the temporary copy of the state document so that in the event that - // updating the persisted document fails, the compaction tokens remain - // in the in-memory state document (_doc). - if (newPhase == Phase::kDropTempCollection) { - doc.setCompactionTokens(_response->toBSON()); - } - LOGV2_DEBUG(6350490, 2, "Transitioning phase for CompactStructuredEncryptionDataCoordinator", @@ -295,29 +272,7 @@ ExecutorFuture<void> CompactStructuredEncryptionDataCoordinator::_runImpl( [this, anchor = shared_from_this()](const auto& state) { _response = doCompactOperation(state); })) - .then(_executePhase( - Phase::kDropTempCollection, [this, anchor = shared_from_this()](const auto& state) { - try { - // restore the response that was stored in the compactionTokens - // field - IDLParserErrorContext ctxt("response"); - _response = CompactStructuredEncryptionDataCommandReply::parse( - ctxt, state.getCompactionTokens()); - } catch (...) { - LOGV2_ERROR(6846101, - "Failed to parse response from " - "CompactStructuredEncryptionDataState document", - "response"_attr = state.getCompactionTokens()); - // ignore for compatibility with 6.0.0 - } - - doDropOperation(state); - if (MONGO_unlikely(fleCompactHangAfterDropTempCollection.shouldFail())) { - LOGV2(6790902, - "Hanging due to fleCompactHangAfterDropTempCollection fail point"); - fleCompactHangAfterDropTempCollection.pauseWhileSet(); - } - })); + .then(_executePhase(Phase::kDropTempCollection, doDropOperation)); } } // namespace mongo diff --git a/src/mongo/db/s/config/configsvr_abort_reshard_collection_command.cpp b/src/mongo/db/s/config/configsvr_abort_reshard_collection_command.cpp index 348e4bc2b40..9cb503a2979 100644 --- a/src/mongo/db/s/config/configsvr_abort_reshard_collection_command.cpp +++ b/src/mongo/db/s/config/configsvr_abort_reshard_collection_command.cpp @@ -76,7 +76,7 @@ void assertExistsReshardingDocument(OperationContext* opCtx, UUID reshardingUUID } auto assertGetReshardingMachine(OperationContext* opCtx, UUID reshardingUUID) { - auto machine = resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< + auto machine = resharding::tryGetReshardingStateMachine< ReshardingCoordinatorService, ReshardingCoordinatorService::ReshardingCoordinator, ReshardingCoordinatorDocument>(opCtx, reshardingUUID); diff --git a/src/mongo/db/s/config/configsvr_commit_move_primary_command.cpp b/src/mongo/db/s/config/configsvr_commit_move_primary_command.cpp deleted file mode 100644 index 76fc3f7b5c9..00000000000 --- a/src/mongo/db/s/config/configsvr_commit_move_primary_command.cpp +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - -#include "mongo/db/auth/authorization_session.h" -#include "mongo/db/commands.h" -#include "mongo/db/repl/read_concern_args.h" -#include "mongo/db/s/config/sharding_catalog_manager.h" -#include "mongo/s/request_types/move_primary_gen.h" - -namespace mongo { -namespace { - -class ConfigsvrCommitMovePrimaryCommand final - : public TypedCommand<ConfigsvrCommitMovePrimaryCommand> { -public: - using Request = ConfigsvrCommitMovePrimary; - - class Invocation final : public InvocationBase { - public: - using InvocationBase::InvocationBase; - - void typedRun(OperationContext* opCtx) { - uassert(ErrorCodes::IllegalOperation, - str::stream() << Request::kCommandName << " can only be run on config servers", - serverGlobalParams.clusterRole == ClusterRole::ConfigServer); - - // Set the operation context read concern level to local for reads into the config - // database. - repl::ReadConcernArgs::get(opCtx) = - repl::ReadConcernArgs(repl::ReadConcernLevel::kLocalReadConcern); - - ShardingCatalogManager::get(opCtx)->commitMovePrimary( - opCtx, - request().getCommandParameter(), - request().getExpectedDatabaseVersion(), - request().getTo()); - } - - private: - NamespaceString ns() const override { - return NamespaceString(request().getDbName()); - } - - bool supportsWriteConcern() const override { - return true; - } - - void doCheckAuthorization(OperationContext* opCtx) const override { - uassert(ErrorCodes::Unauthorized, - "Unauthorized", - AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::internal)); - } - }; - -private: - std::string help() const override { - return "Reassign a new primary shard for the given database on the config server. This is " - "an internal command only invokable on the config server, therefore do not call " - "directly."; - } - - bool adminOnly() const override { - return true; - } - - AllowedOnSecondary secondaryAllowed(ServiceContext* context) const override { - return AllowedOnSecondary::kNever; - } -} configsvrCommitMovePrimaryCommand; - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/s/config/configsvr_commit_reshard_collection_command.cpp b/src/mongo/db/s/config/configsvr_commit_reshard_collection_command.cpp index 0e5c9f803c9..fb67df820c4 100644 --- a/src/mongo/db/s/config/configsvr_commit_reshard_collection_command.cpp +++ b/src/mongo/db/s/config/configsvr_commit_reshard_collection_command.cpp @@ -84,7 +84,7 @@ public: UUID reshardingUUID = retrieveReshardingUUID(opCtx, ns()); - auto machine = resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< + auto machine = resharding::tryGetReshardingStateMachine< ReshardingCoordinatorService, ReshardingCoordinatorService::ReshardingCoordinator, ReshardingCoordinatorDocument>(opCtx, reshardingUUID); diff --git a/src/mongo/db/s/config/configsvr_configure_collection_balancing.cpp b/src/mongo/db/s/config/configsvr_configure_collection_balancing.cpp index c471c70437f..a9db28064b5 100644 --- a/src/mongo/db/s/config/configsvr_configure_collection_balancing.cpp +++ b/src/mongo/db/s/config/configsvr_configure_collection_balancing.cpp @@ -36,7 +36,6 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" -#include "mongo/db/commands/feature_compatibility_version.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/balancer/balancer.h" @@ -61,16 +60,6 @@ public: void typedRun(OperationContext* opCtx) { opCtx->setAlwaysInterruptAtStepDownOrUp(); - - // Hold the FCV region to serialize with the setFeatureCompatibilityVersion command - FixedFCVRegion fcvRegion(opCtx); - uassert(ErrorCodes::IllegalOperation, - "_configsvrConfigureCollectionBalancing can only be run when the cluster is in " - "feature " - "compatibility versions greater or equal than 5.3.", - serverGlobalParams.featureCompatibility.isGreaterThanOrEqualTo( - multiversion::FeatureCompatibilityVersion::kVersion_5_3)); - uassert(ErrorCodes::IllegalOperation, str::stream() << Request::kCommandName << " can only be run on config servers", serverGlobalParams.clusterRole == ClusterRole::ConfigServer); diff --git a/src/mongo/db/s/config/configsvr_merge_chunks_command.cpp b/src/mongo/db/s/config/configsvr_merge_chunks_command.cpp index 5df553b5bd3..ea2823dcdf0 100644 --- a/src/mongo/db/s/config/configsvr_merge_chunks_command.cpp +++ b/src/mongo/db/s/config/configsvr_merge_chunks_command.cpp @@ -94,7 +94,8 @@ public: request().getTimestamp(), request().getCollectionUUID(), request().getChunkRange(), - request().getShard())); + request().getShard(), + request().getValidAfter())); return ConfigSvrMergeResponse{ChunkVersion::fromBSONPositionalOrNewerFormat( shardAndCollVers[ChunkVersion::kShardVersionField])}; } diff --git a/src/mongo/db/s/config/configsvr_reshard_collection_cmd.cpp b/src/mongo/db/s/config/configsvr_reshard_collection_cmd.cpp index 07f1248a8c5..f8a12c57f90 100644 --- a/src/mongo/db/s/config/configsvr_reshard_collection_cmd.cpp +++ b/src/mongo/db/s/config/configsvr_reshard_collection_cmd.cpp @@ -193,12 +193,6 @@ public: if (auto zones = request().getZones()) { checkForOverlappingZones(*zones); - for (const auto& zone : *zones) { - uassertStatusOK( - ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(zone.getMin())); - uassertStatusOK( - ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(zone.getMax())); - } } auto coordinatorDoc = diff --git a/src/mongo/db/s/config/configsvr_run_restore_command.cpp b/src/mongo/db/s/config/configsvr_run_restore_command.cpp index 22e3f7b2985..166c10a4e70 100644 --- a/src/mongo/db/s/config/configsvr_run_restore_command.cpp +++ b/src/mongo/db/s/config/configsvr_run_restore_command.cpp @@ -73,25 +73,6 @@ ShouldRestoreDocument shouldRestoreDocument(OperationContext* opCtx, : ShouldRestoreDocument::kNo; } -std::set<std::string> getDatabasesToRestore(OperationContext* opCtx) { - auto findRequest = FindCommandRequest(NamespaceString::kConfigsvrRestoreNamespace); - - std::set<std::string> databasesToRestore; - DBDirectClient client(opCtx); - auto it = client.find(findRequest); - while (it->more()) { - const auto doc = it->next(); - if (!doc.hasField("ns")) { - continue; - } - - NamespaceString nss(doc.getStringField("ns")); - databasesToRestore.emplace(nss.db()); - } - - return databasesToRestore; -} - // Modifications to this map should add new testing in 'sharded_backup_restore.js'. // { config collection namespace -> ( optional nss field name, optional UUID field name ) } const stdx::unordered_map<NamespaceString, @@ -165,7 +146,7 @@ public: // Keeps track of database names for collections restored. Databases with no collections // restored will have their entries removed in the config collections. - std::set<std::string> databasesRestored = getDatabasesToRestore(opCtx); + std::set<std::string> databasesRestored; for (const auto& collectionEntry : kCollectionEntries) { const NamespaceString& nss = collectionEntry.first; @@ -219,6 +200,10 @@ public: "doc"_attr = doc, "shouldRestore"_attr = shouldRestore); + if (shouldRestore == ShouldRestoreDocument::kYes && docNss) { + databasesRestored.insert(docNss->db().toString()); + } + if (shouldRestore == ShouldRestoreDocument::kYes || shouldRestore == ShouldRestoreDocument::kMaybe) { continue; diff --git a/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp b/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp index 24030e5fe57..da76faa6593 100644 --- a/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp +++ b/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp @@ -61,11 +61,6 @@ public: const auto coordinatorCompletionFuture = [&]() -> SharedSemiFuture<void> { FixedFCVRegion fcvRegion(opCtx); - uassert(ErrorCodes::UnknownFeatureCompatibilityVersion, - "FCV is not yet initialized, retry the command after FCV initialization " - "has completed", - serverGlobalParams.featureCompatibility.isVersionInitialized()); - uassert(ErrorCodes::IllegalOperation, "featureFlagClusterWideConfig not enabled", gFeatureFlagClusterWideConfig.isEnabled( diff --git a/src/mongo/db/s/config/initial_split_policy.cpp b/src/mongo/db/s/config/initial_split_policy.cpp index 15c5a345c59..5bdfb55d6c3 100644 --- a/src/mongo/db/s/config/initial_split_policy.cpp +++ b/src/mongo/db/s/config/initial_split_policy.cpp @@ -37,14 +37,12 @@ #include "mongo/db/bson/dotted_path_support.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/curop.h" -#include "mongo/db/pipeline/document_source.h" #include "mongo/db/pipeline/lite_parsed_pipeline.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" #include "mongo/db/pipeline/sharded_agg_helpers.h" #include "mongo/db/s/balancer/balancer_policy.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/vector_clock.h" -#include "mongo/logv2/log.h" #include "mongo/s/balancer_configuration.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/grid.h" @@ -56,30 +54,12 @@ namespace { using ChunkDistributionMap = stdx::unordered_map<ShardId, size_t>; using ZoneShardMap = StringMap<std::vector<ShardId>>; -using boost::intrusive_ptr; - -std::vector<ShardId> getAllNonDrainingShardIdsSorted(OperationContext* opCtx) { - const auto shardsAndOpTime = uassertStatusOKWithContext( - Grid::get(opCtx)->catalogClient()->getAllShards( - opCtx, repl::ReadConcernLevel::kMajorityReadConcern, true /* excludeDraining */), - "Cannot retrieve updated shard list from config server"); - const auto shards = std::move(shardsAndOpTime.value); - const auto lastVisibleOpTime = std::move(shardsAndOpTime.opTime); - - LOGV2_DEBUG(6566600, - 1, - "Successfully retrieved updated shard list from config server", - "nonDrainingShardsNumber"_attr = shards.size(), - "lastVisibleOpTime"_attr = lastVisibleOpTime); - - std::vector<ShardId> shardIds; - std::transform(shards.begin(), - shards.end(), - std::back_inserter(shardIds), - [](const ShardType& shard) { return ShardId(shard.getName()); }); +std::vector<ShardId> getAllShardIdsSorted(OperationContext* opCtx) { + // Many tests assume that chunks will be placed on shards + // according to their IDs in ascending lexical order. + auto shardIds = Grid::get(opCtx)->shardRegistry()->getAllShardIdsNoReload(); std::sort(shardIds.begin(), shardIds.end()); - return shardIds; } @@ -286,8 +266,7 @@ std::unique_ptr<InitialSplitPolicy> InitialSplitPolicy::calculateOptimizationStr const boost::optional<std::vector<BSONObj>>& initialSplitPoints, const std::vector<TagsType>& tags, size_t numShards, - bool collectionIsEmpty, - bool useAutoSplitter) { + bool collectionIsEmpty) { uassert(ErrorCodes::InvalidOptions, str::stream() << "numInitialChunks is only supported when the collection is empty " "and has a hashed field in the shard key pattern", @@ -331,11 +310,7 @@ std::unique_ptr<InitialSplitPolicy> InitialSplitPolicy::calculateOptimizationStr return std::make_unique<SingleChunkOnPrimarySplitPolicy>(); } - if (useAutoSplitter) { - return std::make_unique<AutoSplitInChunksOnPrimaryPolicy>(); - } - - return std::make_unique<SingleChunkOnPrimarySplitPolicy>(); + return std::make_unique<UnoptimizedSplitPolicy>(); } InitialSplitPolicy::ShardCollectionConfig SingleChunkOnPrimarySplitPolicy::createFirstChunks( @@ -359,7 +334,7 @@ InitialSplitPolicy::ShardCollectionConfig SingleChunkOnPrimarySplitPolicy::creat return {std::move(chunks)}; } -InitialSplitPolicy::ShardCollectionConfig AutoSplitInChunksOnPrimaryPolicy::createFirstChunks( +InitialSplitPolicy::ShardCollectionConfig UnoptimizedSplitPolicy::createFirstChunks( OperationContext* opCtx, const ShardKeyPattern& shardKeyPattern, const SplitPolicyParams& params) { @@ -397,7 +372,7 @@ InitialSplitPolicy::ShardCollectionConfig SplitPointsBasedSplitPolicy::createFir const SplitPolicyParams& params) { // On which shards are the generated chunks allowed to be placed. - const auto shardIds = getAllNonDrainingShardIdsSorted(opCtx); + const auto shardIds = getAllShardIdsSorted(opCtx); const auto currentTime = VectorClock::get(opCtx)->getTime(); const auto validAfter = currentTime.clusterTime().asTimestamp(); @@ -428,7 +403,7 @@ InitialSplitPolicy::ShardCollectionConfig AbstractTagsBasedSplitPolicy::createFi const SplitPolicyParams& params) { invariant(!_tags.empty()); - const auto shardIds = getAllNonDrainingShardIdsSorted(opCtx); + const auto shardIds = getAllShardIdsSorted(opCtx); const auto currentTime = VectorClock::get(opCtx)->getTime(); const auto validAfter = currentTime.clusterTime().asTimestamp(); const auto& keyPattern = shardKeyPattern.getKeyPattern(); @@ -675,29 +650,34 @@ std::vector<BSONObj> ReshardingSplitPolicy::createRawPipeline(const ShardKeyPatt std::vector<BSONObj> res; const auto& shardKeyFields = shardKey.getKeyPatternFields(); + + BSONObjBuilder projectValBuilder; BSONObjBuilder sortValBuilder; - using Doc = Document; - using Arr = std::vector<Value>; - using V = Value; - Arr arrayToObjectBuilder; + for (auto&& fieldRef : shardKeyFields) { // If the shard key includes a hashed field and current fieldRef is the hashed field. if (shardKey.isHashedPattern() && fieldRef->dottedField().compare(shardKey.getHashedField().fieldNameStringData()) == 0) { - arrayToObjectBuilder.emplace_back( - Doc{{"k", V{fieldRef->dottedField()}}, - {"v", Doc{{"$toHashedIndexKey", V{"$" + fieldRef->dottedField()}}}}}); + projectValBuilder.append(fieldRef->dottedField(), + BSON("$toHashedIndexKey" + << "$" + fieldRef->dottedField())); } else { - arrayToObjectBuilder.emplace_back(Doc{ - {"k", V{fieldRef->dottedField()}}, - {"v", Doc{{"$ifNull", V{Arr{V{"$" + fieldRef->dottedField()}, V{BSONNULL}}}}}}}); + projectValBuilder.append( + str::stream() << fieldRef->dottedField(), + BSON("$ifNull" << BSON_ARRAY("$" + fieldRef->dottedField() << BSONNULL))); } + sortValBuilder.append(fieldRef->dottedField().toString(), 1); } + + // Do not project _id if it's not part of the shard key. + if (!shardKey.hasId()) { + projectValBuilder.append("_id", 0); + } + res.push_back(BSON("$sample" << BSON("size" << numSplitPoints * samplesPerChunk))); + res.push_back(BSON("$project" << projectValBuilder.obj())); res.push_back(BSON("$sort" << sortValBuilder.obj())); - res.push_back( - Doc{{"$replaceWith", Doc{{"$arrayToObject", Arr{V{arrayToObjectBuilder}}}}}}.toBson()); return res; } @@ -763,12 +743,12 @@ InitialSplitPolicy::ShardCollectionConfig ReshardingSplitPolicy::createFirstChun } { - auto shardIds = getAllNonDrainingShardIdsSorted(opCtx); - for (const auto& shard : shardIds) { + auto allShardIds = getAllShardIdsSorted(opCtx); + for (const auto& shard : allShardIds) { chunkDistribution.emplace(shard, 0); } - zoneToShardMap.emplace("", std::move(shardIds)); + zoneToShardMap.emplace("", std::move(allShardIds)); } std::vector<ChunkType> chunks; @@ -820,34 +800,26 @@ void ReshardingSplitPolicy::_appendSplitPointsFromSample(BSONObjSet* splitPoints while (nextKey && nRemaining > 0) { // if key is hashed, nextKey values are already hashed - auto result = splitPoints->insert(nextKey->getOwned()); + auto result = splitPoints->insert( + dotted_path_support::extractElementsBasedOnTemplate(*nextKey, shardKey.toBSON()) + .getOwned()); + if (result.second) { nRemaining--; } + nextKey = _samples->getNext(); } } std::unique_ptr<ReshardingSplitPolicy::SampleDocumentSource> -ReshardingSplitPolicy::makePipelineDocumentSource_forTest(OperationContext* opCtx, - const NamespaceString& ns, - const ShardKeyPattern& shardKey, - int numInitialChunks, - int samplesPerChunk) { - MakePipelineOptions opts; - opts.attachCursorSource = false; - return _makePipelineDocumentSource( - opCtx, ns, shardKey, numInitialChunks, samplesPerChunk, std::move(opts)); -} - -std::unique_ptr<ReshardingSplitPolicy::SampleDocumentSource> ReshardingSplitPolicy::_makePipelineDocumentSource(OperationContext* opCtx, const NamespaceString& ns, const ShardKeyPattern& shardKey, int numInitialChunks, - int samplesPerChunk, - MakePipelineOptions opts) { + int samplesPerChunk) { auto rawPipeline = createRawPipeline(shardKey, numInitialChunks - 1, samplesPerChunk); + StringMap<ExpressionContext::ResolvedNamespace> resolvedNamespaces; resolvedNamespaces[ns.coll()] = {ns, std::vector<BSONObj>{}}; @@ -861,7 +833,7 @@ ReshardingSplitPolicy::_makePipelineDocumentSource(OperationContext* opCtx, boost::none, /* explain */ false, /* fromMongos */ false, /* needsMerge */ - true, /* allowDiskUse */ + false, /* allowDiskUse */ true, /* bypassDocumentValidation */ false, /* isMapReduceCommand */ ns, @@ -871,10 +843,8 @@ ReshardingSplitPolicy::_makePipelineDocumentSource(OperationContext* opCtx, std::move(resolvedNamespaces), boost::none); /* collUUID */ - expCtx->tempDir = storageGlobalParams.dbpath + "/tmp"; - - return std::make_unique<PipelineDocumentSource>( - Pipeline::makePipeline(rawPipeline, expCtx, opts), samplesPerChunk - 1); + return std::make_unique<PipelineDocumentSource>(Pipeline::makePipeline(rawPipeline, expCtx, {}), + samplesPerChunk - 1); } ReshardingSplitPolicy::PipelineDocumentSource::PipelineDocumentSource( diff --git a/src/mongo/db/s/config/initial_split_policy.h b/src/mongo/db/s/config/initial_split_policy.h index e492e9c4cb2..ced8d519a13 100644 --- a/src/mongo/db/s/config/initial_split_policy.h +++ b/src/mongo/db/s/config/initial_split_policy.h @@ -40,6 +40,7 @@ #include "mongo/s/shard_id.h" #include "mongo/s/shard_key_pattern.h" #include "mongo/util/string_map.h" + namespace mongo { struct SplitPolicyParams { @@ -52,11 +53,6 @@ public: /** * Returns the optimization strategy for building initial chunks based on the input parameters * and the collection state. - * - * The 'useAutoSplitter' flag indicates to the initial split strategy selected that in case the - * collection contains data, it should use the auto splitter to chop that data into chunks - * respective to the configured chunk size. If set to false, the policy will create as large of - * chunks as possible. */ static std::unique_ptr<InitialSplitPolicy> calculateOptimizationStrategy( OperationContext* opCtx, @@ -66,8 +62,7 @@ public: const boost::optional<std::vector<BSONObj>>& initialSplitPoints, const std::vector<TagsType>& tags, size_t numShards, - bool collectionIsEmpty, - bool useAutoSplitter = true /* Controlled by FCV, see the comment */); + bool collectionIsEmpty); virtual ~InitialSplitPolicy() {} @@ -145,7 +140,7 @@ public: * Split point building strategy to be used when no optimizations are available. We send a * splitVector command to the primary shard in order to calculate the appropriate split points. */ -class AutoSplitInChunksOnPrimaryPolicy : public InitialSplitPolicy { +class UnoptimizedSplitPolicy : public InitialSplitPolicy { public: ShardCollectionConfig createFirstChunks(OperationContext* opCtx, const ShardKeyPattern& shardKeyPattern, @@ -290,7 +285,6 @@ public: public: virtual ~SampleDocumentSource(){}; virtual boost::optional<BSONObj> getNext() = 0; - virtual Pipeline* getPipeline_forTest() = 0; }; // Provides documents from a real Pipeline @@ -299,9 +293,6 @@ public: PipelineDocumentSource() = delete; PipelineDocumentSource(SampleDocumentPipeline pipeline, int skip); boost::optional<BSONObj> getNext() override; - Pipeline* getPipeline_forTest() override { - return _pipeline.get(); - } private: SampleDocumentPipeline _pipeline; @@ -337,21 +328,13 @@ public: static constexpr int kDefaultSamplesPerChunk = 10; - static std::unique_ptr<SampleDocumentSource> makePipelineDocumentSource_forTest( - OperationContext* opCtx, - const NamespaceString& ns, - const ShardKeyPattern& shardKey, - int numInitialChunks, - int samplesPerChunk); - private: static std::unique_ptr<SampleDocumentSource> _makePipelineDocumentSource( OperationContext* opCtx, const NamespaceString& ns, const ShardKeyPattern& shardKey, int numInitialChunks, - int samplesPerChunk, - MakePipelineOptions opts = {}); + int samplesPerChunk); /** * Returns a set of split points to ensure that chunk boundaries will align with the zone diff --git a/src/mongo/db/s/config/initial_split_policy_test.cpp b/src/mongo/db/s/config/initial_split_policy_test.cpp index 0ef3bee06d1..75d1f2ae0a1 100644 --- a/src/mongo/db/s/config/initial_split_policy_test.cpp +++ b/src/mongo/db/s/config/initial_split_policy_test.cpp @@ -35,7 +35,6 @@ #include "mongo/db/s/config/config_server_test_fixture.h" #include "mongo/db/s/config/initial_split_policy.h" #include "mongo/db/vector_clock.h" -#include "mongo/logv2/log.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/unittest/unittest.h" @@ -1728,10 +1727,6 @@ public: return next; } - Pipeline* getPipeline_forTest() override { - return nullptr; - } - private: std::list<BSONObj> _toReturn; }; @@ -1770,9 +1765,9 @@ TEST_F(ReshardingInitSplitTest, NoZones) { shardRegistry()->reload(operationContext()); std::list<BSONObj> mockSamples; - mockSamples.push_back(BSON("y" << 10)); - mockSamples.push_back(BSON("y" << 20)); - mockSamples.push_back(BSON("y" << 30)); + mockSamples.push_back(BSON("x" << 10 << "y" << 10)); + mockSamples.push_back(BSON("x" << 10 << "y" << 20)); + mockSamples.push_back(BSON("x" << 10 << "y" << 30)); auto mockSampleSource = std::make_unique<MockPipelineSource>(std::move(mockSamples)); @@ -1807,9 +1802,9 @@ TEST_F(ReshardingInitSplitTest, HashedShardKey) { shardRegistry()->reload(operationContext()); std::list<BSONObj> mockSamples; - mockSamples.push_back(BSON("y" << 7766103514953448109LL)); - mockSamples.push_back(BSON("y" << -9117533237618642180LL)); - mockSamples.push_back(BSON("y" << -1196399207910989725LL)); + mockSamples.push_back(BSON("x" << 10 << "y" << 7766103514953448109LL)); + mockSamples.push_back(BSON("x" << 10 << "y" << -9117533237618642180LL)); + mockSamples.push_back(BSON("x" << 10 << "y" << -1196399207910989725LL)); auto mockSampleSource = std::make_unique<MockPipelineSource>(std::move(mockSamples)); @@ -1872,9 +1867,9 @@ TEST_F(ReshardingInitSplitTest, ZonesCoversEntireDomainButInsufficient) { shardRegistry()->reload(operationContext()); std::list<BSONObj> mockSamples; - mockSamples.push_back(BSON("y" << 10)); - mockSamples.push_back(BSON("y" << 20)); - mockSamples.push_back(BSON("y" << 30)); + mockSamples.push_back(BSON("x" << 10 << "y" << 10)); + mockSamples.push_back(BSON("x" << 10 << "y" << 20)); + mockSamples.push_back(BSON("x" << 10 << "y" << 30)); auto mockSampleSource = std::make_unique<MockPipelineSource>(std::move(mockSamples)); @@ -1912,9 +1907,9 @@ TEST_F(ReshardingInitSplitTest, SamplesCoincidingWithZones) { shardRegistry()->reload(operationContext()); std::list<BSONObj> mockSamples; - mockSamples.push_back(BSON("y" << 10)); - mockSamples.push_back(BSON("y" << 20)); - mockSamples.push_back(BSON("y" << 30)); + mockSamples.push_back(BSON("x" << 10 << "y" << 10)); + mockSamples.push_back(BSON("x" << 10 << "y" << 20)); + mockSamples.push_back(BSON("x" << 10 << "y" << 30)); auto mockSampleSource = std::make_unique<MockPipelineSource>(std::move(mockSamples)); diff --git a/src/mongo/db/s/config/sharding_catalog_manager.cpp b/src/mongo/db/s/config/sharding_catalog_manager.cpp index d380fe58468..6fbb06a0668 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager.cpp @@ -44,7 +44,6 @@ #include "mongo/db/internal_transactions_feature_flag_gen.h" #include "mongo/db/operation_context.h" #include "mongo/db/ops/write_ops.h" -#include "mongo/db/query/cursor_response.h" #include "mongo/db/query/query_request_helper.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/balancer/type_migration.h" @@ -447,12 +446,6 @@ Status ShardingCatalogManager::_initConfigIndexes(OperationContext* opCtx) { return result.withContext("couldn't create lock id index on config db"); } - result = configShard->createIndexOnConfig( - opCtx, LocksType::ConfigNS, BSON(LocksType::process() << 1), !unique); - if (!result.isOK()) { - return result.withContext("couldn't create lock process index on config db"); - } - result = configShard->createIndexOnConfig(opCtx, LocksType::ConfigNS, diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h index 8ec8fb02108..070c1d3d78f 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager.h +++ b/src/mongo/db/s/config/sharding_catalog_manager.h @@ -31,7 +31,6 @@ #include "mongo/base/status_with.h" #include "mongo/bson/bsonobj.h" -#include "mongo/client/fetcher.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/logical_session_cache.h" @@ -272,7 +271,8 @@ public: const boost::optional<Timestamp>& timestamp, const UUID& requestCollectionUUID, const ChunkRange& chunkRange, - const ShardId& shardId); + const ShardId& shardId, + const boost::optional<Timestamp>& validAfter); /** * Updates metadata in config.chunks collection to show the given chunk in its new shard. @@ -348,8 +348,7 @@ public: */ void splitOrMarkJumbo(OperationContext* opCtx, const NamespaceString& nss, - const BSONObj& minKey, - boost::optional<int64_t> optMaxChunkSizeBytes); + const BSONObj& minKey); /** * In a transaction, sets the 'allowMigrations' to the requested state and bumps the collection @@ -397,15 +396,6 @@ public: // # TODO SERVER-63983: remove enableSharding paramter when 6.0 becomes lastLTS bool enableSharding = false); - /** - * Updates the metadata in config.databases collection with the new primary shard for the given - * database. This also advances the database's lastmod. - */ - void commitMovePrimary(OperationContext* opCtx, - const StringData& dbName, - const DatabaseVersion& expectedDbVersion, - const ShardId& toShardId); - // // Collection Operations // @@ -606,11 +596,10 @@ private: * Runs a command against a "shard" that is not yet in the cluster and thus not present in the * ShardRegistry. */ - StatusWith<Shard::CommandResponse> _runCommandForAddShard( - OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, - StringData dbName, - const BSONObj& cmdObj); + StatusWith<Shard::CommandResponse> _runCommandForAddShard(OperationContext* opCtx, + RemoteCommandTargeter* targeter, + StringData dbName, + const BSONObj& cmdObj); /** * Helper method for running a count command against the config server with appropriate error @@ -647,29 +636,7 @@ private: * Sets the current cluster's user-write blocking state on the shard that is being added. */ void _setUserWriteBlockingStateOnNewShard(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter); - - using FetcherDocsCallbackFn = std::function<bool(const std::vector<BSONObj>& batch)>; - using FetcherStatusCallbackFn = std::function<void(const Status& status)>; - - /** - * Creates a Fetcher task for fetching documents in the given collection on the given shard. - * After the task is scheduled, applies 'processDocsCallback' to each fetched batch and - * 'processStatusCallback' to the fetch status. - */ - std::unique_ptr<Fetcher> _createFetcher(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, - const NamespaceString& nss, - const repl::ReadConcernLevel& readConcernLevel, - FetcherDocsCallbackFn processDocsCallback, - FetcherStatusCallbackFn processStatusCallback); - - /** - * Gets the cluster time keys on the given shard and then saves them locally. - */ - Status _pullClusterTimeKeys(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter); - + RemoteCommandTargeter* targeter); /** * Given a vector of cluster parameters in disk format, sets them locally. */ @@ -680,14 +647,14 @@ private: * Gets the cluster parameters set on the shard and then saves them locally. */ void _pullClusterParametersFromNewShard(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter); + RemoteCommandTargeter* targeter); /** * Clean all possible leftover cluster parameters on the new added shard and sets the ones * stored on the config server. */ void _pushClusterParametersToNewShard(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, + RemoteCommandTargeter* targeter, const std::vector<BSONObj>& clusterParameters); /** @@ -695,20 +662,7 @@ private: * converting from a replica set to a sharded cluster) or set the cluster parameters stored on * the config server in the newly added shard. */ - void _standardizeClusterParameters(OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter); - - /** - * Execute the merge chunk updates using the internal transaction API. - */ - void _mergeChunksInTransaction(OperationContext* opCtx, - const NamespaceString& nss, - const UUID& collectionUUID, - const ChunkVersion& mergeVersion, - const Timestamp& validAfter, - const ChunkRange& chunkRange, - const ShardId& shardId, - std::shared_ptr<std::vector<ChunkType>> chunksToMerge); + void _standardizeClusterParameters(OperationContext* opCtx, RemoteCommandTargeter* targeter); // The owning service context ServiceContext* const _serviceContext; diff --git a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp index aa483a030a5..0ed9c76d0f0 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_add_shard_test.cpp @@ -41,13 +41,11 @@ #include "mongo/db/ops/write_ops.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/replication_coordinator_mock.h" -#include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/add_shard_cmd_gen.h" #include "mongo/db/s/add_shard_util.h" #include "mongo/db/s/config/config_server_test_fixture.h" #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/type_shard_identity.h" -#include "mongo/db/time_proof_service.h" #include "mongo/idl/cluster_server_parameter_gen.h" #include "mongo/s/catalog/config_server_version.h" #include "mongo/s/catalog/type_changelog.h" @@ -85,32 +83,25 @@ protected: ASSERT_OK(clusterIdLoader->loadClusterId(operationContext(), repl::ReadConcernLevel::kLocalReadConcern)); _clusterId = clusterIdLoader->getClusterId(); - - WaitForMajorityService::get(getServiceContext()).startup(getServiceContext()); - } - - void tearDown() override { - WaitForMajorityService::get(getServiceContext()).shutDown(); - ConfigServerTestFixture::tearDown(); } /** - * addShard validates the host as a shard. It calls "hello" on the host to determine what + * addShard validates the host as a shard. It calls "isMaster" on the host to determine what * kind of host it is -- mongos, regular mongod, config mongod -- and whether the replica set - * details are correct. "helloResponse" defines the response of the "hello" request and + * details are correct. "isMasterResponse" defines the response of the "isMaster" request and * should be a command response BSONObj, or a failed Status. * * ShardingTestFixture::expectGetShards() should be called before this function, otherwise - * addShard will never reach the "hello" command -- a find query is called first. + * addShard will never reach the isMaster command -- a find query is called first. */ - void expectHello(const HostAndPort& target, StatusWith<BSONObj> helloResponse) { - onCommandForAddShard([&, target, helloResponse](const RemoteCommandRequest& request) { + void expectIsMaster(const HostAndPort& target, StatusWith<BSONObj> isMasterResponse) { + onCommandForAddShard([&, target, isMasterResponse](const RemoteCommandRequest& request) { ASSERT_EQ(request.target, target); ASSERT_EQ(request.dbname, "admin"); - ASSERT_BSONOBJ_EQ(request.cmdObj, BSON("hello" << 1)); + ASSERT_BSONOBJ_EQ(request.cmdObj, BSON("isMaster" << 1)); ASSERT_BSONOBJ_EQ(rpc::makeEmptyMetadata(), request.metadata); - return helloResponse; + return isMasterResponse; }); } @@ -234,7 +225,7 @@ protected: ASSERT_EQ(request.dbname, NamespaceString::kClusterParametersNamespace.db()); ASSERT_BSONOBJ_EQ(request.cmdObj, BSON("find" << NamespaceString::kClusterParametersNamespace.coll() - << "maxTimeMS" << 60000 << "readConcern" + << "maxTimeMS" << 30000 << "readConcern" << BSON("level" << "majority"))); auto cursorRes = CursorResponse(NamespaceString::kClusterParametersNamespace, 0, {}); @@ -242,24 +233,6 @@ protected: }); } - void expectClusterTimeKeysPullRequest(const HostAndPort& target) { - onCommandForAddShard([&](const RemoteCommandRequest& request) { - ASSERT_EQ(request.target, target); - ASSERT_BSONOBJ_EQ(request.cmdObj, - BSON("find" << NamespaceString::kKeysCollectionNamespace.coll() - << "maxTimeMS" << 60000 << "readConcern" - << BSON("level" - << "local"))); - - KeysCollectionDocument key(1); - key.setKeysCollectionDocumentBase( - {"dummy", TimeProofService::generateRandomKey(), LogicalTime(Timestamp(105, 0))}); - auto cursorRes = - CursorResponse(NamespaceString::kKeysCollectionNamespace, 0, {key.toBSON()}); - return cursorRes.toBSON(CursorResponse::ResponseType::InitialResponse); - }); - } - /** * Waits for a request for the shardIdentity document to be upserted into a shard from the * config server on addShard. @@ -460,7 +433,11 @@ TEST_F(AddShardTest, CreateShardIdentityUpsertForAddShard) { << shardName << "clusterId" << _clusterId << "configsvrConnectionString" << replicationCoordinator()->getConfigConnectionString().toString()) - << "multi" << false << "upsert" << true))); + << "multi" << false << "upsert" << true)) + << "writeConcern" + << BSON("w" + << "majority" + << "wtimeout" << 60000)); auto addShardCmd = add_shard_util::createAddShardCmd(operationContext(), shardName); auto actualBSON = add_shard_util::createShardIdentityUpsertForAddShard(addShardCmd); ASSERT_BSONOBJ_EQ(expectedBSON, actualBSON); @@ -505,9 +482,9 @@ TEST_F(AddShardTest, StandaloneBasicSuccess) { ASSERT_EQUALS(expectedShardName, shardName); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "maxWireVersion" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); // Get databases list from new shard expectListDatabases( @@ -520,9 +497,6 @@ TEST_F(AddShardTest, StandaloneBasicSuccess) { expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions")); - // The shard receives a find to pull all clusterTime keys from the new shard. - expectClusterTimeKeysPullRequest(shardTarget); - // The shard receives the _addShard command expectAddShardCmdReturnSuccess(shardTarget, expectedShardName); @@ -595,9 +569,9 @@ TEST_F(AddShardTest, StandaloneGenerateName) { ASSERT_EQUALS(expectedShardName, shardName); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "maxWireVersion" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); // Get databases list from new shard expectListDatabases( @@ -610,9 +584,6 @@ TEST_F(AddShardTest, StandaloneGenerateName) { expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions")); - // The shard receives a find to pull all clusterTime keys from the new shard. - expectClusterTimeKeysPullRequest(shardTarget); - // The shard receives the _addShard command expectAddShardCmdReturnSuccess(shardTarget, expectedShardName); @@ -702,7 +673,7 @@ TEST_F(AddShardTest, UnreachableHost) { }); Status hostUnreachableStatus = Status(ErrorCodes::HostUnreachable, "host unreachable"); - expectHello(shardTarget, hostUnreachableStatus); + expectIsMaster(shardTarget, hostUnreachableStatus); future.timed_get(kLongFutureTimeout); } @@ -727,9 +698,9 @@ TEST_F(AddShardTest, AddMongosAsShard) { ASSERT_EQUALS(ErrorCodes::IllegalOperation, status); }); - expectHello(shardTarget, - BSON("msg" - << "isdbgrid")); + expectIsMaster(shardTarget, + BSON("msg" + << "isdbgrid")); future.timed_get(kLongFutureTimeout); } @@ -755,10 +726,10 @@ TEST_F(AddShardTest, AddReplicaSetShardAsStandalone) { ASSERT_STRING_CONTAINS(status.getStatus().reason(), "use replica set url format"); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "myOtherSet" << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -785,9 +756,9 @@ TEST_F(AddShardTest, AddStandaloneHostShardAsReplicaSet) { ASSERT_STRING_CONTAINS(status.getStatus().reason(), "host did not return a set name"); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "maxWireVersion" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -814,10 +785,10 @@ TEST_F(AddShardTest, ReplicaSetMistmatchedReplicaSetName) { ASSERT_STRING_CONTAINS(status.getStatus().reason(), "does not match the actual set name"); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "myOtherSet" << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -846,10 +817,10 @@ TEST_F(AddShardTest, ShardIsCSRSConfigServer) { }); BSONObj commandResponse = - BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSON("ok" << 1 << "ismaster" << true << "setName" << "config" << "configsvr" << true << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -879,11 +850,11 @@ TEST_F(AddShardTest, ReplicaSetMissingHostsProvidedInSeedList) { BSONArrayBuilder hosts; hosts.append("host1:12345"); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "mySet" << "hosts" << hosts.arr() << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -914,11 +885,11 @@ TEST_F(AddShardTest, AddShardWithNameConfigFails) { BSONArrayBuilder hosts; hosts.append("host1:12345"); hosts.append("host2:12345"); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "mySet" << "hosts" << hosts.arr() << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); future.timed_get(kLongFutureTimeout); } @@ -960,11 +931,11 @@ TEST_F(AddShardTest, ShardContainsExistingDatabase) { BSONArrayBuilder hosts; hosts.append("host1:12345"); hosts.append("host2:12345"); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "mySet" << "hosts" << hosts.arr() << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); expectListDatabases(shardTarget, {BSON("name" << existingDB.getName())}); @@ -1004,20 +975,17 @@ TEST_F(AddShardTest, SuccessfullyAddReplicaSet) { BSONArrayBuilder hosts; hosts.append("host1:12345"); hosts.append("host2:12345"); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "mySet" << "hosts" << hosts.arr() << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); // Get databases list from new shard expectListDatabases(shardTarget, std::vector<BSONObj>{BSON("name" << discoveredDB.getName())}); expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions")); - // The shard receives a find to pull all clusterTime keys from the new shard. - expectClusterTimeKeysPullRequest(shardTarget); - // The shard receives the _addShard command expectAddShardCmdReturnSuccess(shardTarget, expectedShardName); @@ -1079,20 +1047,17 @@ TEST_F(AddShardTest, ReplicaSetExtraHostsDiscovered) { BSONArrayBuilder hosts; hosts.append("host1:12345"); hosts.append("host2:12345"); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "setName" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "setName" << "mySet" << "hosts" << hosts.arr() << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); // Get databases list from new shard expectListDatabases(shardTarget, std::vector<BSONObj>{BSON("name" << discoveredDB.getName())}); expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions")); - // The shard receives a find to pull all clusterTime keys from the new shard. - expectClusterTimeKeysPullRequest(shardTarget); - // The shard receives the _addShard command expectAddShardCmdReturnSuccess(shardTarget, expectedShardName); @@ -1163,9 +1128,9 @@ TEST_F(AddShardTest, AddShardSucceedsEvenIfAddingDBsFromNewShardFails) { ASSERT_EQUALS(expectedShardName, shardName); }); - BSONObj commandResponse = BSON("ok" << 1 << "isWritablePrimary" << true << "maxWireVersion" + BSONObj commandResponse = BSON("ok" << 1 << "ismaster" << true << "maxWireVersion" << WireVersion::LATEST_WIRE_VERSION); - expectHello(shardTarget, commandResponse); + expectIsMaster(shardTarget, commandResponse); // Get databases list from new shard expectListDatabases( @@ -1178,9 +1143,6 @@ TEST_F(AddShardTest, AddShardSucceedsEvenIfAddingDBsFromNewShardFails) { expectCollectionDrop(shardTarget, NamespaceString("config", "system.sessions")); - // The shard receives a find to pull all clusterTime keys from the new shard. - expectClusterTimeKeysPullRequest(shardTarget); - // The shard receives the _addShard command expectAddShardCmdReturnSuccess(shardTarget, expectedShardName); diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp index d1bc8fcd343..172729b575b 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp @@ -43,7 +43,6 @@ #include "mongo/db/logical_session_cache.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" -#include "mongo/db/query/cursor_response.h" #include "mongo/db/query/distinct_command_gen.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/sharding_logging.h" @@ -90,6 +89,76 @@ void appendShortVersion(BufBuilder* out, const ChunkType& chunk) { bb.done(); } +BSONArray buildMergeChunksTransactionUpdates(const std::vector<ChunkType>& chunksToMerge, + const ChunkVersion& mergeVersion, + const boost::optional<Timestamp>& validAfter) { + BSONArrayBuilder updates; + + // Build an update operation to expand the first chunk into the newly merged chunk + { + BSONObjBuilder op; + op.append("op", "u"); + op.appendBool("b", false); // no upsert + op.append("ns", ChunkType::ConfigNS.ns()); + + // expand first chunk into newly merged chunk + ChunkType mergedChunk(chunksToMerge.front()); + mergedChunk.setMax(chunksToMerge.back().getMax()); + + // fill in additional details for sending through transaction + mergedChunk.setVersion(mergeVersion); + mergedChunk.setEstimatedSizeBytes(boost::none); + + invariant(validAfter); + mergedChunk.setHistory({ChunkHistory(validAfter.get(), mergedChunk.getShard())}); + + // add the new chunk information as the update object + op.append("o", mergedChunk.toConfigBSON()); + + // query object + op.append("o2", BSON(ChunkType::name(mergedChunk.getName()))); + + updates.append(op.obj()); + } + + // Build update operations to delete the rest of the chunks to be merged. Remember not + // to delete the first chunk we're expanding + for (size_t i = 1; i < chunksToMerge.size(); ++i) { + BSONObjBuilder op; + op.append("op", "d"); + op.append("ns", ChunkType::ConfigNS.ns()); + + op.append("o", BSON(ChunkType::name(chunksToMerge[i].getName()))); + + updates.append(op.obj()); + } + + return updates.arr(); +} + +BSONArray buildMergeChunksTransactionPrecond(const std::vector<ChunkType>& chunksToMerge, + const ChunkVersion& collVersion) { + BSONArrayBuilder preCond; + + for (const auto& chunk : chunksToMerge) { + BSONObj query = BSON(ChunkType::min(chunk.getMin()) + << ChunkType::max(chunk.getMax()) << ChunkType::collectionUUID() + << chunk.getCollectionUUID()); + + const auto collectionIdentityMatchCondition = + BSON(ChunkType::collectionUUID() + << chunk.getCollectionUUID() << ChunkType::shard(chunk.getShard().toString())); + + BSONObjBuilder b; + b.append("ns", ChunkType::ConfigNS.ns()); + b.append("q", BSON("query" << query << "orderby" << BSON(ChunkType::lastmod() << -1))); + b.append("res", collectionIdentityMatchCondition); + + preCond.append(b.obj()); + } + return preCond.arr(); +} + /** * Check that the chunk still exists and return its metadata. */ @@ -97,11 +166,13 @@ StatusWith<ChunkType> findChunkContainingRange(OperationContext* opCtx, const UUID& uuid, const OID& epoch, const Timestamp& timestamp, - const ChunkRange& range) { + const BSONObj& min, + const BSONObj& max) { const auto chunkQuery = [&]() { BSONObjBuilder queryBuilder; queryBuilder << ChunkType::collectionUUID << uuid; - queryBuilder << ChunkType::min(BSON("$lte" << range.getMin())); + queryBuilder << ChunkType::min(BSON("$lte" << min)); + queryBuilder << ChunkType::max(BSON("$gte" << max)); return queryBuilder.obj(); }(); @@ -113,26 +184,21 @@ StatusWith<ChunkType> findChunkContainingRange(OperationContext* opCtx, repl::ReadConcernLevel::kLocalReadConcern, ChunkType::ConfigNS, chunkQuery, - BSON(ChunkType::min << -1), - 1 /* limit */); + BSONObj(), + 2 /* limit */); if (!findResponseWith.isOK()) { return findResponseWith.getStatus(); } - if (!findResponseWith.getValue().docs.empty()) { - const auto containingChunk = uassertStatusOK(ChunkType::parseFromConfigBSON( - findResponseWith.getValue().docs.front(), epoch, timestamp)); - - if (containingChunk.getRange().covers(range)) { - return containingChunk; - } + if (findResponseWith.getValue().docs.size() != 1) { + return {ErrorCodes::Error(40165), + str::stream() << "Could not find a chunk including bounds [" << min << ", " << max + << "). Cannot execute the migration commit with invalid chunks."}; } - return {ErrorCodes::Error(40165), - str::stream() << "Could not find a chunk including bounds [" << range.getMin() << ", " - << range.getMax() - << "). Cannot execute the migration commit with invalid chunks."}; + return uassertStatusOK( + ChunkType::parseFromConfigBSON(findResponseWith.getValue().docs.front(), epoch, timestamp)); } BSONObj makeCommitChunkTransactionCommand(const NamespaceString& nss, @@ -207,33 +273,6 @@ BSONObj makeCommitChunkTransactionCommand(const NamespaceString& nss, return BSON("applyOps" << updates.arr() << "alwaysUpsert" << false); } -BSONObj buildCountContiguousChunksByBounds(const UUID& collectionUUID, - const std::string& shard, - const std::vector<BSONObj>& boundsForChunks) { - AggregateCommandRequest countRequest(ChunkType::ConfigNS); - - invariant(boundsForChunks.size() > 1); - auto minBoundIt = boundsForChunks.begin(); - auto maxBoundIt = minBoundIt + 1; - - BSONArrayBuilder chunkDocArray; - while (maxBoundIt != boundsForChunks.end()) { - const auto query = BSON(ChunkType::min(*minBoundIt) - << ChunkType::max(*maxBoundIt) << ChunkType::collectionUUID() - << collectionUUID << ChunkType::shard() << shard); - - chunkDocArray.append(query); - ++minBoundIt; - ++maxBoundIt; - } - - std::vector<BSONObj> pipeline; - pipeline.push_back(BSON("$match" << BSON("$or" << chunkDocArray.arr()))); - pipeline.push_back(BSON("$count" << ChunkType::collectionUUID.name())); - countRequest.setPipeline(pipeline); - return countRequest.toBSON({}); -} - /** * Returns a chunk different from the one being migrated or 'none' if one doesn't exist. */ @@ -461,48 +500,6 @@ std::vector<ShardId> getShardsOwningChunksForCollection(OperationContext* opCtx, return shardIds; } -// Checks if the requested split already exists. It is possible that the split operation completed, -// but the router did not receive the response. This would result in the router retrying the split -// operation, in which case it is fine for the request to become a no-op. -auto isSplitAlreadyDone(OperationContext* opCtx, - const ChunkRange& range, - const std::string& shardName, - const ChunkType& origChunk, - const std::vector<BSONObj>& newChunkBounds) { - std::vector<BSONObj> expectedChunksBounds; - expectedChunksBounds.reserve(newChunkBounds.size() + 1); - expectedChunksBounds.push_back(range.getMin()); - expectedChunksBounds.insert( - std::end(expectedChunksBounds), std::begin(newChunkBounds), std::end(newChunkBounds)); - - auto countRequest = buildCountContiguousChunksByBounds( - origChunk.getCollectionUUID(), shardName, expectedChunksBounds); - - const auto expectedChunkCount = expectedChunksBounds.size() - 1; - - auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - - auto countResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts( - opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - NamespaceString::kConfigDb.toString(), - countRequest, - Shard::RetryPolicy::kIdempotent)); - - const auto docCount = [&]() { - auto cursorResponse = - uassertStatusOK(CursorResponse::parseFromBSON(countResponse.response)); - auto firstBatch = cursorResponse.getBatch(); - if (firstBatch.empty()) { - return 0; - } - - auto countObj = firstBatch.front(); - return countObj.getIntField(ChunkType::collectionUUID.name()); - }(); - return size_t(docCount) == expectedChunkCount; -} - } // namespace void ShardingCatalogManager::bumpMajorVersionOneChunkPerShard( @@ -573,11 +570,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( const std::vector<BSONObj>& splitPoints, const std::string& shardName, const bool fromChunkSplitter) { - - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk modifications and generate // strictly monotonously increasing collection versions Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); @@ -634,26 +626,11 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( std::vector<ChunkType> newChunks; ChunkVersion currentMaxVersion = collVersion; - const auto buildChunkVersionBSON = [](const ChunkVersion& version) { - BSONObjBuilder response; - version.serializeToBSON(kCollectionVersionField, &response); - version.serializeToBSON(ChunkVersion::kShardVersionField, &response); - return response.obj(); - }; auto startKey = range.getMin(); auto newChunkBounds(splitPoints); newChunkBounds.push_back(range.getMax()); - if (isSplitAlreadyDone(opCtx, range, shardName, origChunk.getValue(), newChunkBounds)) { - // In case the request was already fullfilled, we still need to wait until the original - // request is majority written. The timestamp is not known, so we use the system's last - // optime. Otherwise the next RoutingInfo cache refresh from the shard may not see the - // newest information. - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return buildChunkVersionBSON(collVersion); - } - auto shouldTakeOriginalChunkID = true; OID chunkID; @@ -713,7 +690,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( newChunk.setMin(startKey); newChunk.setMax(endKey); newChunk.setEstimatedSizeBytes(boost::none); - newChunk.setJumbo(false); op.append("o", newChunk.toConfigBSON()); @@ -802,83 +778,10 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( } } - return buildChunkVersionBSON(currentMaxVersion); -} - -void ShardingCatalogManager::_mergeChunksInTransaction( - OperationContext* opCtx, - const NamespaceString& nss, - const UUID& collectionUUID, - const ChunkVersion& mergeVersion, - const Timestamp& validAfter, - const ChunkRange& chunkRange, - const ShardId& shardId, - std::shared_ptr<std::vector<ChunkType>> chunksToMerge) { - withTransaction( - opCtx, ChunkType::ConfigNS, [&, this](OperationContext* opCtx, TxnNumber txnNumber) { - // Construct the new chunk by taking `min` from the first merged chunk and `max` - // from the last. - write_ops::UpdateCommandRequest updateOp(ChunkType::ConfigNS); - updateOp.setUpdates({[&] { - write_ops::UpdateOpEntry entry; - - ChunkType mergedChunk(chunksToMerge->front()); - entry.setQ(BSON(ChunkType::name(mergedChunk.getName()))); - mergedChunk.setMax(chunksToMerge->back().getMax()); - - // Fill in additional details for sending through transaction. - mergedChunk.setVersion(mergeVersion); - mergedChunk.setEstimatedSizeBytes(boost::none); - - mergedChunk.setHistory({ChunkHistory(validAfter, mergedChunk.getShard())}); - - entry.setU(write_ops::UpdateModification::parseFromClassicUpdate( - mergedChunk.toConfigBSON())); - entry.setMulti(false); - - return entry; - }()}); - - const auto updateRes = - writeToConfigDocumentInTxn(opCtx, ChunkType::ConfigNS, updateOp, txnNumber); - - const auto numDocsModified = UpdateOp::parseResponse(updateRes).getN(); - uassert(ErrorCodes::ConflictingOperationInProgress, - str::stream() << "Unexpected number of modified documents during chunks merge " - "commit. Modified " - << numDocsModified << " documents instead of 1", - numDocsModified == 1); - - // Delete the rest of the chunks to be merged. - // Remember not to delete the first chunk we're expanding. - BSONObjBuilder queryBuilder; - queryBuilder << ChunkType::collectionUUID << collectionUUID; - queryBuilder << ChunkType::shard(shardId.toString()); - queryBuilder << ChunkType::min(BSON("$gte" << chunksToMerge->front().getMax())); - queryBuilder << ChunkType::min(BSON("$lt" << chunksToMerge->back().getMax())); - - write_ops::DeleteCommandRequest deleteOp(ChunkType::ConfigNS); - deleteOp.setDeletes({[&] { - write_ops::DeleteOpEntry entry; - entry.setQ(queryBuilder.obj()); - entry.setMulti(true); - return entry; - }()}); - - const auto deleteRes = - writeToConfigDocumentInTxn(opCtx, ChunkType::ConfigNS, deleteOp, txnNumber); - - const auto numDocsDeleted = DeleteOp::parseResponse(deleteRes).getN(); - const int expectedNumDocsDeleted = chunksToMerge->size() - 1; - uassert(ErrorCodes::ConflictingOperationInProgress, - str::stream() << "Unexpected number of deleted documents during chunks merge " - "commit. Deleted " - << numDocsDeleted << " documents instead of " - << expectedNumDocsDeleted, - numDocsDeleted == expectedNumDocsDeleted); - - LOGV2_DEBUG(6583805, 1, "Finished all transaction operations in merge chunk command"); - }); + BSONObjBuilder response; + currentMaxVersion.serializeToBSON(kCollectionVersionField, &response); + currentMaxVersion.serializeToBSON(ChunkVersion::kShardVersionField, &response); + return response.obj(); } StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( @@ -888,11 +791,11 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( const boost::optional<Timestamp>& timestamp, const UUID& requestCollectionUUID, const ChunkRange& chunkRange, - const ShardId& shardId) { - - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); + const ShardId& shardId, + const boost::optional<Timestamp>& validAfter) { + if (!validAfter) { + return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; + } // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk modifications and generate // strictly monotonously increasing collection versions @@ -968,18 +871,11 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( // 3. Prepare the data for the merge // and ensure that the retrieved list of chunks covers the whole range. - - // The `validAfter` field must always be set. If not existing, it means the chunk - // always belonged to the same shard, hence it's valid to set `0` as the time at - // which the chunk started being valid. - Timestamp validAfter{0}; - - auto chunksToMerge = std::make_shared<std::vector<ChunkType>>(); - chunksToMerge->reserve(shardChunksInRangeResponse.docs.size()); + std::vector<ChunkType> chunksToMerge; for (const auto& chunkDoc : shardChunksInRangeResponse.docs) { auto chunk = uassertStatusOK( ChunkType::parseFromConfigBSON(chunkDoc, coll.getEpoch(), coll.getTimestamp())); - if (chunksToMerge->empty()) { + if (chunksToMerge.empty()) { uassert(ErrorCodes::IllegalOperation, str::stream() << "could not merge chunks, shard " << shardId @@ -992,41 +888,46 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( << "could not merge chunks, shard " << shardId << " does not contain a sequence of chunks that exactly fills the range " << chunkRange.toString(), - chunk.getMin().woCompare(chunksToMerge->back().getMax()) == 0); - } - - // Get the `validAfter` field from the most recent chunk placed on the shard - if (!chunk.getHistory().empty()) { - const auto& chunkValidAfter = chunk.getHistory().front().getValidAfter(); - if (validAfter < chunkValidAfter) { - validAfter = chunkValidAfter; - } + chunk.getMin().woCompare(chunksToMerge.back().getMax()) == 0); } - - chunksToMerge->push_back(std::move(chunk)); + chunksToMerge.push_back(std::move(chunk)); } uassert(ErrorCodes::IllegalOperation, str::stream() << "could not merge chunks, shard " << shardId << " does not contain a sequence of chunks that exactly fills the range " << chunkRange.toString(), - !chunksToMerge->empty() && - chunksToMerge->back().getMax().woCompare(chunkRange.getMax()) == 0); + !chunksToMerge.empty() && + chunksToMerge.back().getMax().woCompare(chunkRange.getMax()) == 0); ChunkVersion initialVersion = collVersion; ChunkVersion mergeVersion = initialVersion; mergeVersion.incMinor(); + auto updates = buildMergeChunksTransactionUpdates(chunksToMerge, mergeVersion, validAfter); + auto preCond = buildMergeChunksTransactionPrecond(chunksToMerge, initialVersion); + // 4. apply the batch of updates to local metadata - _mergeChunksInTransaction( - opCtx, nss, coll.getUuid(), mergeVersion, validAfter, chunkRange, shardId, chunksToMerge); + uassertStatusOK(Grid::get(opCtx)->catalogClient()->applyChunkOpsDeprecated( + opCtx, + updates, + preCond, + coll.getUuid(), + nss, + mergeVersion, + WriteConcernOptions(), + repl::ReadConcernLevel::kLocalReadConcern)); // 5. log changes BSONObjBuilder logDetail; + { + BSONArrayBuilder b(logDetail.subarrayStart("merged")); + for (const auto& chunkToMerge : chunksToMerge) { + b.append(chunkToMerge.toConfigBSON()); + } + } initialVersion.appendLegacyWithField(&logDetail, "prevShardVersion"); mergeVersion.appendLegacyWithField(&logDetail, "mergedVersion"); logDetail.append("owningShard", shardId); - chunkRange.append(&logDetail); - logDetail.append("numChunks", static_cast<int>(chunksToMerge->size())); ShardingLogging::get(opCtx)->logChange( opCtx, "merge", nss.ns(), logDetail.obj(), WriteConcernOptions()); @@ -1051,15 +952,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; } - uassertStatusOK( - ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(migratedChunk.getMin())); - uassertStatusOK( - ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(migratedChunk.getMax())); - - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Must hold the shard lock until the entire commit finishes to serialize with removeShard. Lock::SharedLock shardLock(opCtx->lockState(), _kShardMembershipLock); @@ -1149,8 +1041,12 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( migratedChunk.isVersionSet() && migratedChunk.getVersion().isSet()); // Check if range still exists and which shard owns it - auto swCurrentChunk = findChunkContainingRange( - opCtx, coll.getUuid(), coll.getEpoch(), coll.getTimestamp(), migratedChunk.getRange()); + auto swCurrentChunk = findChunkContainingRange(opCtx, + coll.getUuid(), + coll.getEpoch(), + coll.getTimestamp(), + migratedChunk.getMin(), + migratedChunk.getMax()); if (!swCurrentChunk.isOK()) { return swCurrentChunk.getStatus(); @@ -1367,10 +1263,6 @@ void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx, auto const catalogClient = Grid::get(opCtx)->catalogClient(); const auto shardRegistry = Grid::get(opCtx)->shardRegistry(); - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and // migrations. Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock); @@ -1410,12 +1302,10 @@ void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx, }()}); return updateOp; }()); + request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON()); - auto response = configShard->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - request, - ShardingCatalogClient::kLocalWriteConcern, - Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand( + opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); uassert(ErrorCodes::Error(5760502), @@ -1505,10 +1395,6 @@ void ShardingCatalogManager::clearJumboFlag(OperationContext* opCtx, const NamespaceString& nss, const OID& collectionEpoch, const ChunkRange& chunk) { - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk modifications and generate // strictly monotonously increasing collection versions Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); @@ -1627,10 +1513,6 @@ void ShardingCatalogManager::ensureChunkVersionIsGreaterThan(OperationContext* o const BSONObj& minKey, const BSONObj& maxKey, const ChunkVersion& version) { - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk modifications and generate // strictly monotonously increasing collection versions Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); @@ -1828,58 +1710,36 @@ void ShardingCatalogManager::bumpMultipleCollectionVersionsAndChangeMetadataInTx unique_function<void(OperationContext*, TxnNumber)> changeMetadataFunc, const WriteConcernOptions& writeConcern) { - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and // migrations Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); - withTransaction( - opCtx, - NamespaceString::kConfigReshardingOperationsNamespace, - [&collNames, &changeMetadataFunc](OperationContext* opCtx, TxnNumber txnNumber) { - for (const auto& nss : collNames) { - bumpCollectionMinorVersion(opCtx, nss, txnNumber); - } - changeMetadataFunc(opCtx, txnNumber); - }, - writeConcern); + withTransaction(opCtx, + NamespaceString::kConfigReshardingOperationsNamespace, + [&](OperationContext* opCtx, TxnNumber txnNumber) { + for (const auto& nss : collNames) { + bumpCollectionMinorVersion(opCtx, nss, txnNumber); + } + changeMetadataFunc(opCtx, txnNumber); + }, + writeConcern); } void ShardingCatalogManager::splitOrMarkJumbo(OperationContext* opCtx, const NamespaceString& nss, - const BSONObj& minKey, - boost::optional<int64_t> optMaxChunkSizeBytes) { + const BSONObj& minKey) { const auto cm = uassertStatusOK( Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(opCtx, nss)); auto chunk = cm.findIntersectingChunkWithSimpleCollation(minKey); try { - const auto maxChunkSizeBytes = [&]() -> int64_t { - if (optMaxChunkSizeBytes.has_value()) { - return *optMaxChunkSizeBytes; - } - - auto coll = Grid::get(opCtx)->catalogClient()->getCollection( - opCtx, nss, repl::ReadConcernLevel::kMajorityReadConcern); - return coll.getMaxChunkSizeBytes().value_or( - Grid::get(opCtx)->getBalancerConfiguration()->getMaxChunkSizeBytes()); - }(); - - // Limit the search to one split point: this code path is reached when a migration fails due - // to ErrorCodes::ChunkTooBig. In case there is a too frequent shard key, only select the - // next key in order to split the range in jumbo chunk + remaining range. - const int limit = 1; - auto splitPoints = uassertStatusOK( - shardutil::selectChunkSplitPoints(opCtx, - chunk.getShardId(), - nss, - cm.getShardKeyPattern(), - ChunkRange(chunk.getMin(), chunk.getMax()), - maxChunkSizeBytes, - limit)); + const auto splitPoints = uassertStatusOK(shardutil::selectChunkSplitPoints( + opCtx, + chunk.getShardId(), + nss, + cm.getShardKeyPattern(), + ChunkRange(chunk.getMin(), chunk.getMax()), + Grid::get(opCtx)->getBalancerConfiguration()->getMaxChunkSizeBytes())); if (splitPoints.empty()) { LOGV2(21873, @@ -1933,9 +1793,6 @@ void ShardingCatalogManager::splitOrMarkJumbo(OperationContext* opCtx, return; } - // Resize the vector because in multiversion scenarios the `autoSplitVector` command may end - // up ignoring the `limit` parameter and returning the whole list of split points. - splitPoints.resize(limit); uassertStatusOK( shardutil::splitChunkAtMultiplePoints(opCtx, chunk.getShardId(), @@ -1957,10 +1814,6 @@ void ShardingCatalogManager::setAllowMigrationsAndBumpOneChunk( bool allowMigrations) { std::set<ShardId> shardsIds; { - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata - // collections under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and // migrations Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); @@ -1977,10 +1830,7 @@ void ShardingCatalogManager::setAllowMigrationsAndBumpOneChunk( cm.getAllShardIds(&shardsIds); withTransaction( - opCtx, - CollectionType::ConfigNS, - [this, allowMigrations, &nss, &collectionUUID](OperationContext* opCtx, - TxnNumber txnNumber) { + opCtx, CollectionType::ConfigNS, [&](OperationContext* opCtx, TxnNumber txnNumber) { // Update the 'allowMigrations' field. An unset 'allowMigrations' field implies // 'true'. To ease backwards compatibility we omit 'allowMigrations' instead of // setting it explicitly to 'true'. @@ -2081,13 +1931,11 @@ bool ShardingCatalogManager::clearChunkEstimatedSize(OperationContext* opCtx, co }()}); return updateOp; }()); + request.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = configShard->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - request, - ShardingCatalogClient::kMajorityWriteConcern, - Shard::RetryPolicy::kIdempotent); + auto response = configShard->runBatchWriteCommand( + opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); return response.getN() > 0; diff --git a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp index 5ad10c12d09..6b2538bfeca 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp @@ -288,10 +288,6 @@ std::pair<std::vector<BSONObj>, std::vector<BSONObj>> makeChunkAndTagUpdatesForR void ShardingCatalogManager::refineCollectionShardKey(OperationContext* opCtx, const NamespaceString& nss, const ShardKeyPattern& newShardKeyPattern) { - // Mark opCtx as interruptible to ensure that all reads and writes to the metadata collections - // under the exclusive _kChunkOpLock happen on the same term. - opCtx->setAlwaysInterruptAtStepDownOrUp(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk modifications and generate // strictly monotonously increasing collection versions Lock::ExclusiveLock chunkLk(opCtx, opCtx->lockState(), _kChunkOpLock); @@ -327,80 +323,79 @@ void ShardingCatalogManager::refineCollectionShardKey(OperationContext* opCtx, Timestamp newTimestamp = now.clusterTime().asTimestamp(); collType.setTimestamp(newTimestamp); - auto updateCollectionAndChunksFn = - [this, &nss, &collType, &timers, &newFields](OperationContext* opCtx, TxnNumber txnNumber) { - // Update the config.collections entry for the given namespace. - updateShardingCatalogEntryForCollectionInTxn( - opCtx, nss, collType, false /* upsert */, txnNumber); - - LOGV2(21933, - "refineCollectionShardKey updated collection entry for {namespace}: took " - "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", - "refineCollectionShardKey updated collection entry", - "namespace"_attr = nss.ns(), - "durationMillis"_attr = timers->executionTimer.millis(), - "totalTimeMillis"_attr = timers->totalTimer.millis()); - timers->executionTimer.reset(); - - if (MONGO_unlikely(hangRefineCollectionShardKeyBeforeUpdatingChunks.shouldFail())) { - LOGV2(21934, "Hit hangRefineCollectionShardKeyBeforeUpdatingChunks failpoint"); - hangRefineCollectionShardKeyBeforeUpdatingChunks.pauseWhileSet(opCtx); - } + auto updateCollectionAndChunksFn = [&](OperationContext* opCtx, TxnNumber txnNumber) { + // Update the config.collections entry for the given namespace. + updateShardingCatalogEntryForCollectionInTxn( + opCtx, nss, collType, false /* upsert */, txnNumber); + + LOGV2(21933, + "refineCollectionShardKey updated collection entry for {namespace}: took " + "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", + "refineCollectionShardKey updated collection entry", + "namespace"_attr = nss.ns(), + "durationMillis"_attr = timers->executionTimer.millis(), + "totalTimeMillis"_attr = timers->totalTimer.millis()); + timers->executionTimer.reset(); + + if (MONGO_unlikely(hangRefineCollectionShardKeyBeforeUpdatingChunks.shouldFail())) { + LOGV2(21934, "Hit hangRefineCollectionShardKeyBeforeUpdatingChunks failpoint"); + hangRefineCollectionShardKeyBeforeUpdatingChunks.pauseWhileSet(opCtx); + } - auto [chunkUpdates, tagUpdates] = makeChunkAndTagUpdatesForRefine(newFields); + auto [chunkUpdates, tagUpdates] = makeChunkAndTagUpdatesForRefine(newFields); - // Update all config.chunks entries for the given namespace by setting (i) their bounds - // for each new field in the refined key to MinKey (except for the global max chunk - // where the max bounds are set to MaxKey), and unsetting (ii) their jumbo field. - const auto chunksQuery = BSON(ChunkType::collectionUUID << collType.getUuid()); - writeToConfigDocumentInTxn( - opCtx, - ChunkType::ConfigNS, - BatchedCommandRequest::buildPipelineUpdateOp(ChunkType::ConfigNS, - chunksQuery, - chunkUpdates, - false, // upsert - true // useMultiUpdate - ), - txnNumber); + // Update all config.chunks entries for the given namespace by setting (i) their bounds for + // each new field in the refined key to MinKey (except for the global max chunk where the + // max bounds are set to MaxKey), and unsetting (ii) their jumbo field. + const auto chunksQuery = BSON(ChunkType::collectionUUID << collType.getUuid()); + writeToConfigDocumentInTxn( + opCtx, + ChunkType::ConfigNS, + BatchedCommandRequest::buildPipelineUpdateOp(ChunkType::ConfigNS, + chunksQuery, + chunkUpdates, + false, // upsert + true // useMultiUpdate + ), + txnNumber); - LOGV2(21935, - "refineCollectionShardKey: updated chunk entries for {namespace}: took " - "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", - "refineCollectionShardKey: updated chunk entries", - "namespace"_attr = nss.ns(), - "durationMillis"_attr = timers->executionTimer.millis(), - "totalTimeMillis"_attr = timers->totalTimer.millis()); - timers->executionTimer.reset(); - - // Update all config.tags entries for the given namespace by setting their bounds for - // each new field in the refined key to MinKey (except for the global max tag where the - // max bounds are set to MaxKey). - writeToConfigDocumentInTxn( - opCtx, - TagsType::ConfigNS, - BatchedCommandRequest::buildPipelineUpdateOp(TagsType::ConfigNS, - BSON("ns" << nss.ns()), - tagUpdates, - false, // upsert - true // useMultiUpdate - ), - txnNumber); + LOGV2(21935, + "refineCollectionShardKey: updated chunk entries for {namespace}: took " + "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", + "refineCollectionShardKey: updated chunk entries", + "namespace"_attr = nss.ns(), + "durationMillis"_attr = timers->executionTimer.millis(), + "totalTimeMillis"_attr = timers->totalTimer.millis()); + timers->executionTimer.reset(); + + // Update all config.tags entries for the given namespace by setting their bounds for + // each new field in the refined key to MinKey (except for the global max tag where the + // max bounds are set to MaxKey). + writeToConfigDocumentInTxn( + opCtx, + TagsType::ConfigNS, + BatchedCommandRequest::buildPipelineUpdateOp(TagsType::ConfigNS, + BSON("ns" << nss.ns()), + tagUpdates, + false, // upsert + true // useMultiUpdate + ), + txnNumber); - LOGV2(21936, - "refineCollectionShardKey: updated zone entries for {namespace}: took " - "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", - "refineCollectionShardKey: updated zone entries", - "namespace"_attr = nss.ns(), - "durationMillis"_attr = timers->executionTimer.millis(), - "totalTimeMillis"_attr = timers->totalTimer.millis()); + LOGV2(21936, + "refineCollectionShardKey: updated zone entries for {namespace}: took " + "{durationMillis} ms. Total time taken: {totalTimeMillis} ms.", + "refineCollectionShardKey: updated zone entries", + "namespace"_attr = nss.ns(), + "durationMillis"_attr = timers->executionTimer.millis(), + "totalTimeMillis"_attr = timers->totalTimer.millis()); - if (MONGO_unlikely(hangRefineCollectionShardKeyBeforeCommit.shouldFail())) { - LOGV2(21937, "Hit hangRefineCollectionShardKeyBeforeCommit failpoint"); - hangRefineCollectionShardKeyBeforeCommit.pauseWhileSet(opCtx); - } - }; + if (MONGO_unlikely(hangRefineCollectionShardKeyBeforeCommit.shouldFail())) { + LOGV2(21937, "Hit hangRefineCollectionShardKeyBeforeCommit failpoint"); + hangRefineCollectionShardKeyBeforeCommit.pauseWhileSet(opCtx); + } + }; auto updateCollectionAndChunksWithAPIFn = [collType, newFields, nss, timers](const txn_api::TransactionClient& txnClient, @@ -563,42 +558,31 @@ void ShardingCatalogManager::configureCollectionBalancing( boost::optional<bool> defragmentCollection, boost::optional<bool> enableAutoSplitter) { + // Hold the FCV region to serialize with the setFeatureCompatibilityVersion command + FixedFCVRegion fcvRegion(opCtx); + uassert(ErrorCodes::IllegalOperation, + "_configsvrConfigureCollectionBalancing can only be run when the cluster is in feature " + "compatibility versions greater or equal than 5.3.", + serverGlobalParams.featureCompatibility.isGreaterThanOrEqualTo( + multiversion::FeatureCompatibilityVersion::kVersion_5_3)); + uassert(ErrorCodes::InvalidOptions, "invalid configure collection balancing update", chunkSizeMB || defragmentCollection || enableAutoSplitter); - // utility lambda to log the change - auto logConfigureCollectionBalancing = [&]() { - BSONObjBuilder logChangeDetail; - if (chunkSizeMB) { - logChangeDetail.append("chunkSizeMB", chunkSizeMB.get()); - } - - if (defragmentCollection) { - logChangeDetail.append("defragmentCollection", defragmentCollection.get()); - } - - if (enableAutoSplitter) { - logChangeDetail.append("enableAutoSplitter", enableAutoSplitter.get()); - } - - ShardingLogging::get(opCtx)->logChange( - opCtx, "configureCollectionBalancing", nss.ns(), logChangeDetail.obj()); - }; - short updatedFields = 0; BSONObjBuilder updateCmd; { BSONObjBuilder setBuilder(updateCmd.subobjStart("$set")); if (chunkSizeMB && *chunkSizeMB != 0) { - auto chunkSizeBytes = static_cast<int64_t>(*chunkSizeMB) * 1024 * 1024; - bool withinRange = nss == NamespaceString::kLogicalSessionsNamespace - ? (chunkSizeBytes > 0 && chunkSizeBytes <= 1024 * 1024 * 1024) - : ChunkSizeSettingsType::checkMaxChunkSizeValid(chunkSizeBytes); + // verify we got a positive integer in range [1MB, 1GB] uassert(ErrorCodes::InvalidOptions, str::stream() << "Chunk size '" << *chunkSizeMB << "' out of range [1MB, 1GB]", - withinRange); - setBuilder.append(CollectionType::kMaxChunkSizeBytesFieldName, chunkSizeBytes); + *chunkSizeMB > 0 && + *chunkSizeMB < std::numeric_limits<int32_t>::max() / (1024 * 1024) && + ChunkSizeSettingsType::checkMaxChunkSizeValid(*chunkSizeMB * 1024 * 1024)); + setBuilder.append(CollectionType::kMaxChunkSizeBytesFieldName, + *chunkSizeMB * 1024 * 1024); updatedFields++; } if (defragmentCollection) { @@ -624,7 +608,6 @@ void ShardingCatalogManager::configureCollectionBalancing( } if (updatedFields == 0) { - logConfigureCollectionBalancing(); return; } @@ -634,27 +617,26 @@ void ShardingCatalogManager::configureCollectionBalancing( // migrations Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); - withTransaction(opCtx, - CollectionType::ConfigNS, - [this, &nss, &update](OperationContext* opCtx, TxnNumber txnNumber) { - const auto query = BSON(CollectionType::kNssFieldName << nss.ns()); - const auto res = writeToConfigDocumentInTxn( - opCtx, - CollectionType::ConfigNS, - BatchedCommandRequest::buildUpdateOp(CollectionType::ConfigNS, - query, - update /* update */, - false /* upsert */, - false /* multi */), - txnNumber); - const auto numDocsModified = UpdateOp::parseResponse(res).getN(); - uassert(ErrorCodes::NamespaceNotSharded, - str::stream() << "Expected to match one doc for query " << query - << " but matched " << numDocsModified, - numDocsModified == 1); - - bumpCollectionMinorVersionInTxn(opCtx, nss, txnNumber); - }); + withTransaction( + opCtx, CollectionType::ConfigNS, [&](OperationContext* opCtx, TxnNumber txnNumber) { + const auto query = BSON(CollectionType::kNssFieldName << nss.ns()); + const auto res = writeToConfigDocumentInTxn( + opCtx, + CollectionType::ConfigNS, + BatchedCommandRequest::buildUpdateOp(CollectionType::ConfigNS, + query, + update /* update */, + false /* upsert */, + false /* multi */), + txnNumber); + const auto numDocsModified = UpdateOp::parseResponse(res).getN(); + uassert(ErrorCodes::ConflictingOperationInProgress, + str::stream() << "Expected to match one doc for query " << query + << " but matched " << numDocsModified, + numDocsModified == 1); + + bumpCollectionMinorVersionInTxn(opCtx, nss, txnNumber); + }); // Now any migrations that change the list of shards will see the results of the transaction // during refresh, so it is safe to release the chunk lock. } @@ -672,8 +654,6 @@ void ShardingCatalogManager::configureCollectionBalancing( executor); Balancer::get(opCtx)->notifyPersistedBalancerSettingsChanged(opCtx); - - logConfigureCollectionBalancing(); } void ShardingCatalogManager::renameShardedMetadata( @@ -723,9 +703,7 @@ void ShardingCatalogManager::updateTimeSeriesGranularity(OperationContext* opCtx cm.getAllShardIds(&shardIds); withTransaction( - opCtx, - CollectionType::ConfigNS, - [this, &nss, granularity, &shardIds](OperationContext* opCtx, TxnNumber txnNumber) { + opCtx, CollectionType::ConfigNS, [&](OperationContext* opCtx, TxnNumber txnNumber) { // Update granularity value in config.collections. auto granularityFieldName = CollectionType::kTimeseriesFieldsFieldName + "." + TypeCollectionTimeseriesFields::kGranularityFieldName; diff --git a/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp index b320c7f2407..6c95002292b 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_config_initialization_test.cpp @@ -34,7 +34,7 @@ #include "mongo/bson/json.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" @@ -312,8 +312,6 @@ TEST_F(ConfigInitializationTest, BuildsNecessaryIndexes) { << "_id_"), BSON("v" << 2 << "key" << BSON("ts" << 1) << "name" << "ts_1"), - BSON("v" << 2 << "key" << BSON("process" << 1) << "name" - << "process_1"), BSON("v" << 2 << "key" << BSON("state" << 1 << "process" << 1) << "name" << "state_1_process_1")}; auto expectedShardsIndexes = std::vector<BSONObj>{ diff --git a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp index 561a8911029..934eca19e8e 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp @@ -37,7 +37,6 @@ #include "mongo/db/dbdirectclient.h" #include "mongo/db/namespace_string.h" #include "mongo/db/ops/write_ops.h" -#include "mongo/db/persistent_task_store.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/dist_lock_manager.h" #include "mongo/db/server_options.h" @@ -94,14 +93,9 @@ DatabaseType ShardingCatalogManager::createDatabase(OperationContext* opCtx, dbName.toString(), ShardId::kConfigServerId, DatabaseVersion::makeFixed()); } - // It is not allowed to create the 'admin' or 'local' databases, including any alternative - // casing. It is allowed to create the 'config' database (handled by the early return above), - // but only with that exact casing. uassert(ErrorCodes::InvalidOptions, - str::stream() << "Cannot manually create database '" << dbName << "'", - !dbName.equalCaseInsensitive(NamespaceString::kAdminDb) && - !dbName.equalCaseInsensitive(NamespaceString::kLocalDb) && - !dbName.equalCaseInsensitive(NamespaceString::kConfigDb)); + str::stream() << "Cannot manually create database'" << dbName << "'", + dbName != NamespaceString::kAdminDb && dbName != NamespaceString::kLocalDb); uassert(ErrorCodes::InvalidNamespace, str::stream() << "Invalid db name specified: " << dbName, @@ -168,7 +162,7 @@ DatabaseType ShardingCatalogManager::createDatabase(OperationContext* opCtx, // Do another loop, with the db lock held in order to avoid taking the expensive path on // concurrent create database operations dbLock.emplace(DistLockManager::get(opCtx)->lockDirectLocally( - opCtx, str::toLower(dbName), DistLockManager::kDefaultLockTimeout)); + opCtx, dbName, DistLockManager::kDefaultLockTimeout)); } // Expensive createDatabase code path @@ -274,70 +268,4 @@ DatabaseType ShardingCatalogManager::createDatabase(OperationContext* opCtx, return database; } -void ShardingCatalogManager::commitMovePrimary(OperationContext* opCtx, - const StringData& dbName, - const DatabaseVersion& expectedDbVersion, - const ShardId& toShardId) { - // Hold the shard lock until the entire commit finishes to serialize with removeShard. - Lock::SharedLock shardLock(opCtx->lockState(), _kShardMembershipLock); - - const auto toShardDoc = [&] { - DBDirectClient dbClient(opCtx); - return dbClient.findOne(NamespaceString::kConfigsvrShardsNamespace, - BSON(ShardType::name << toShardId)); - }(); - uassert(ErrorCodes::ShardNotFound, - "Requested primary shard {} does not exist"_format(toShardId.toString()), - !toShardDoc.isEmpty()); - - const auto toShardEntry = uassertStatusOK(ShardType::fromBSON(toShardDoc)); - uassert(ErrorCodes::ShardNotFound, - "Requested primary shard {} is draining"_format(toShardId.toString()), - !toShardEntry.getDraining()); - - const auto updateOp = [&] { - const auto query = [&] { - BSONObjBuilder bsonBuilder; - bsonBuilder.append(DatabaseType::kNameFieldName, dbName); - // Include the version in the update filter to be resilient to potential network retries - // and delayed messages. - for (const auto [fieldName, fieldValue] : expectedDbVersion.toBSON()) { - const auto dottedFieldName = DatabaseType::kVersionFieldName + "." + fieldName; - bsonBuilder.appendAs(fieldValue, dottedFieldName); - } - return bsonBuilder.obj(); - }(); - - const auto update = [&] { - auto newDbVersion = expectedDbVersion.makeUpdated(); - const auto now = VectorClock::get(opCtx)->getTime(); - const auto clusterTime = now.clusterTime().asTimestamp(); - - newDbVersion.setTimestamp(clusterTime); - tassert(8235300, - "New database timestamp must be newer than previous one", - newDbVersion.getTimestamp() > expectedDbVersion.getTimestamp()); - - BSONObjBuilder bsonBuilder; - bsonBuilder.append(DatabaseType::kPrimaryFieldName, toShardId); - bsonBuilder.append(DatabaseType::kVersionFieldName, newDbVersion.toBSON()); - return BSON("$set" << bsonBuilder.obj()); - }(); - - write_ops::UpdateCommandRequest updateOp(NamespaceString::kConfigDatabasesNamespace); - updateOp.setUpdates({[&] { - write_ops::UpdateOpEntry entry; - entry.setQ(query); - entry.setU(write_ops::UpdateModification::parseFromClassicUpdate(update)); - return entry; - }()}); - - return updateOp; - }(); - - DBDirectClient dbClient(opCtx); - const auto commandResponse = dbClient.runCommand(updateOp.serialize({})); - uassertStatusOK(getStatusFromWriteCommandReply(commandResponse->getCommandReply())); -} - } // namespace mongo diff --git a/src/mongo/db/s/config/sharding_catalog_manager_database_operations_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_database_operations_test.cpp deleted file mode 100644 index 029a229d73f..00000000000 --- a/src/mongo/db/s/config/sharding_catalog_manager_database_operations_test.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright (C) 2023-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/s/config/config_server_test_fixture.h" -#include "mongo/db/s/config/sharding_catalog_manager.h" - -namespace mongo { -namespace { - -class ShardingCatalogManagerDatabaseOperationsTest : public ConfigServerTestFixture { -public: - void setUp() override { - ConfigServerTestFixture::setUp(); - _opCtx = operationContext(); - } - -protected: - OperationContext* _opCtx; -}; - -TEST_F(ShardingCatalogManagerDatabaseOperationsTest, CreateDatabaseAdminFails) { - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "admin"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); - - // Alternative capitalizations are also invalid - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "Admin"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); - - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "aDmIn"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); -} - -TEST_F(ShardingCatalogManagerDatabaseOperationsTest, CreateDatabaseLocalFails) { - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "local"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); - - // Alternative capitalizations are also invalid - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "Local"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); - - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "lOcAl"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); -} - -TEST_F(ShardingCatalogManagerDatabaseOperationsTest, CreateDatabaseConfig) { - // It is allowed to create the "config" database. - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "config"_sd, boost::none); - - // But alternative capitalizations are invalid. - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "Config"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); - - ASSERT_THROWS_CODE( - ShardingCatalogManager::get(_opCtx)->createDatabase(_opCtx, "cOnFiG"_sd, boost::none), - DBException, - ErrorCodes::InvalidOptions); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/s/config/sharding_catalog_manager_merge_chunks_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_merge_chunks_test.cpp index be08bede01c..d8a006c2286 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_merge_chunks_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_merge_chunks_test.cpp @@ -30,15 +30,9 @@ #include "mongo/platform/basic.h" #include "mongo/client/read_preference.h" -#include "mongo/db/dbdirectclient.h" -#include "mongo/db/logical_session_cache_noop.h" #include "mongo/db/namespace_string.h" -#include "mongo/db/read_write_concern_defaults.h" -#include "mongo/db/read_write_concern_defaults_cache_lookup_mock.h" #include "mongo/db/s/config/config_server_test_fixture.h" #include "mongo/db/s/config/sharding_catalog_manager.h" -#include "mongo/db/s/transaction_coordinator_service.h" -#include "mongo/db/session_catalog_mongod.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog/type_chunk.h" @@ -50,37 +44,18 @@ using unittest::assertGet; class MergeChunkTest : public ConfigServerTestFixture { protected: std::string _shardName = "shard0000"; - void setUp() override { ConfigServerTestFixture::setUp(); - ShardType shard; shard.setName(_shardName); shard.setHost(_shardName + ":12"); setupShards({shard}); - - DBDirectClient client(operationContext()); - client.createCollection(NamespaceString::kSessionTransactionsTableNamespace.ns()); - client.createIndexes(NamespaceString::kSessionTransactionsTableNamespace.ns(), - {MongoDSessionCatalog::getConfigTxnPartialIndexSpec()}); - client.createCollection(CollectionType::ConfigNS.ns()); - - ReadWriteConcernDefaults::create(getServiceContext(), _lookupMock.getFetchDefaultsFn()); - LogicalSessionCache::set(getServiceContext(), std::make_unique<LogicalSessionCacheNoop>()); - TransactionCoordinatorService::get(operationContext()) - ->onShardingInitialization(operationContext(), true); - } - - void tearDown() override { - TransactionCoordinatorService::get(operationContext())->onStepDown(); - ConfigServerTestFixture::tearDown(); } const ShardId _shardId{_shardName}; const NamespaceString _nss1{"TestDB.TestColl1"}; const NamespaceString _nss2{"TestDB.TestColl2"}; const KeyPattern _keyPattern{BSON("x" << 1)}; - ReadWriteConcernDefaultsLookupMock _lookupMock; }; TEST_F(MergeChunkTest, MergeExistingChunksCorrectlyShouldSucceed) { @@ -100,11 +75,6 @@ TEST_F(MergeChunkTest, MergeExistingChunksCorrectlyShouldSucceed) { auto chunk2(chunk); chunk2.setName(OID::gen()); - // set histories - chunk.setHistory({ChunkHistory{Timestamp{100, 0}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 0}, _shardId}}); - - // set boundaries auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkMax = BSON("a" << 10); @@ -117,6 +87,8 @@ TEST_F(MergeChunkTest, MergeExistingChunksCorrectlyShouldSucceed) { setupCollection(_nss1, _keyPattern, {chunk, chunk2}); + Timestamp validAfter{100, 0}; + ChunkRange rangeToBeMerged(chunk.getMin(), chunk2.getMax()); auto versions = assertGet(ShardingCatalogManager::get(operationContext()) @@ -126,7 +98,8 @@ TEST_F(MergeChunkTest, MergeExistingChunksCorrectlyShouldSucceed) { collTimestamp, collUuid, rangeToBeMerged, - _shardId)); + _shardId, + validAfter)); auto collVersion = ChunkVersion::fromBSONPositionalOrNewerFormat(versions["collectionVersion"]); auto shardVersion = ChunkVersion::fromBSONPositionalOrNewerFormat(versions["shardVersion"]); @@ -168,8 +141,7 @@ TEST_F(MergeChunkTest, MergeExistingChunksCorrectlyShouldSucceed) { // Make sure history is there ASSERT_EQ(1UL, mergedChunk.getHistory().size()); - ASSERT_EQ(chunk2.getHistory().front().getValidAfter(), - mergedChunk.getHistory().front().getValidAfter()); + ASSERT_EQ(validAfter, mergedChunk.getHistory().front().getValidAfter()); } TEST_F(MergeChunkTest, MergeSeveralChunksCorrectlyShouldSucceed) { @@ -190,11 +162,6 @@ TEST_F(MergeChunkTest, MergeSeveralChunksCorrectlyShouldSucceed) { chunk2.setName(OID::gen()); chunk3.setName(OID::gen()); - // set histories - chunk.setHistory({ChunkHistory{Timestamp{100, 10}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 1}, _shardId}}); - chunk3.setHistory({ChunkHistory{Timestamp{50, 0}, _shardId}}); - auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkBound2 = BSON("a" << 7); @@ -212,6 +179,8 @@ TEST_F(MergeChunkTest, MergeSeveralChunksCorrectlyShouldSucceed) { setupCollection(_nss1, _keyPattern, {chunk, chunk2, chunk3}); ChunkRange rangeToBeMerged(chunk.getMin(), chunk3.getMax()); + Timestamp validAfter{100, 0}; + ASSERT_OK(ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), _nss1, @@ -219,7 +188,8 @@ TEST_F(MergeChunkTest, MergeSeveralChunksCorrectlyShouldSucceed) { collTimestamp, collUuid, rangeToBeMerged, - _shardId)); + _shardId, + validAfter)); const auto query BSON(ChunkType::collectionUUID() << collUuid); auto findResponse = uassertStatusOK( @@ -250,8 +220,7 @@ TEST_F(MergeChunkTest, MergeSeveralChunksCorrectlyShouldSucceed) { // Make sure history is there ASSERT_EQ(1UL, mergedChunk.getHistory().size()); - ASSERT_EQ(chunk2.getHistory().front().getValidAfter(), - mergedChunk.getHistory().front().getValidAfter()); + ASSERT_EQ(validAfter, mergedChunk.getHistory().front().getValidAfter()); } TEST_F(MergeChunkTest, NewMergeShouldClaimHighestVersion) { @@ -272,10 +241,6 @@ TEST_F(MergeChunkTest, NewMergeShouldClaimHighestVersion) { auto chunk2(chunk); chunk2.setName(OID::gen()); - // set histories - chunk.setHistory({ChunkHistory{Timestamp{100, 0}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 0}, _shardId}}); - auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkMax = BSON("a" << 10); @@ -298,6 +263,8 @@ TEST_F(MergeChunkTest, NewMergeShouldClaimHighestVersion) { setupCollection(_nss1, _keyPattern, {chunk, chunk2, otherChunk}); + Timestamp validAfter{100, 0}; + ASSERT_OK(ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), _nss1, @@ -305,7 +272,8 @@ TEST_F(MergeChunkTest, NewMergeShouldClaimHighestVersion) { collTimestamp, collUuid, rangeToBeMerged, - _shardId)); + _shardId, + validAfter)); const auto query = BSON(ChunkType::collectionUUID() << collUuid); auto findResponse = uassertStatusOK( @@ -336,8 +304,7 @@ TEST_F(MergeChunkTest, NewMergeShouldClaimHighestVersion) { // Make sure history is there ASSERT_EQ(1UL, mergedChunk.getHistory().size()); - ASSERT_EQ(chunk2.getHistory().front().getValidAfter(), - mergedChunk.getHistory().front().getValidAfter()); + ASSERT_EQ(validAfter, mergedChunk.getHistory().front().getValidAfter()); } TEST_F(MergeChunkTest, MergeLeavesOtherChunksAlone) { @@ -357,10 +324,6 @@ TEST_F(MergeChunkTest, MergeLeavesOtherChunksAlone) { auto chunk2(chunk); chunk2.setName(OID::gen()); - // set histories - chunk.setHistory({ChunkHistory{Timestamp{100, 5}, shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 1}, shardId}}); - auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkMax = BSON("a" << 10); @@ -391,7 +354,8 @@ TEST_F(MergeChunkTest, MergeLeavesOtherChunksAlone) { collTimestamp, collUuid, rangeToBeMerged, - shardId)); + shardId, + validAfter)); const auto query = BSON(ChunkType::collectionUUID() << collUuid); auto findResponse = uassertStatusOK( getConfigShard()->exhaustiveFindOnConfig(operationContext(), @@ -434,16 +398,11 @@ TEST_F(MergeChunkTest, NonExistingNamespace) { chunk.setCollectionUUID(UUID::gen()); auto origVersion = ChunkVersion(1, 0, collEpoch, collTimestamp); - chunk.setShard(_shardId); chunk.setVersion(origVersion); // Construct chunk to be merged auto chunk2(chunk); - // set history - chunk.setHistory({ChunkHistory{Timestamp{100, 0}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 0}, _shardId}}); - auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkMax = BSON("a" << 10); @@ -458,6 +417,8 @@ TEST_F(MergeChunkTest, NonExistingNamespace) { setupCollection(_nss1, _keyPattern, {chunk, chunk2}); + Timestamp validAfter{1}; + ASSERT_THROWS(ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), NamespaceString("TestDB.NonExistingColl"), @@ -465,7 +426,8 @@ TEST_F(MergeChunkTest, NonExistingNamespace) { collTimestamp, collUuidAtRequest, rangeToBeMerged, - _shardId), + _shardId, + validAfter), DBException); } @@ -484,10 +446,6 @@ TEST_F(MergeChunkTest, NonMatchingUUIDsOfChunkAndRequestErrors) { // Construct chunk to be merged auto chunk2(chunk); - // set histories - chunk.setHistory({ChunkHistory{Timestamp{100, 0}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 0}, _shardId}}); - auto chunkMin = BSON("a" << 1); auto chunkBound = BSON("a" << 5); auto chunkMax = BSON("a" << 10); @@ -502,6 +460,8 @@ TEST_F(MergeChunkTest, NonMatchingUUIDsOfChunkAndRequestErrors) { setupCollection(_nss1, _keyPattern, {chunk, chunk2}); + Timestamp validAfter{1}; + auto mergeStatus = ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), _nss1, @@ -509,7 +469,8 @@ TEST_F(MergeChunkTest, NonMatchingUUIDsOfChunkAndRequestErrors) { collTimestamp, requestUuid, rangeToBeMerged, - _shardId); + _shardId, + validAfter); ASSERT_EQ(ErrorCodes::InvalidUUID, mergeStatus); } @@ -533,11 +494,12 @@ TEST_F(MergeChunkTest, MergeAlreadyHappenedSucceeds) { mergedChunk.setName(OID::gen()); mergedChunk.setCollectionUUID(collUuid); mergedChunk.setShard(_shardId); - mergedChunk.setHistory({ChunkHistory{Timestamp{100, 0}, _shardId}}); setupCollection(_nss1, _keyPattern, {mergedChunk}); + Timestamp validAfter{1}; + ASSERT_OK(ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), _nss1, @@ -545,7 +507,8 @@ TEST_F(MergeChunkTest, MergeAlreadyHappenedSucceeds) { collTimestamp, collUuid, rangeToBeMerged, - _shardId)); + _shardId, + validAfter)); // Verify that no change to config.chunks happened. const auto query = BSON(ChunkType::collectionUUID() << collUuid); @@ -592,11 +555,6 @@ TEST_F(MergeChunkTest, MergingChunksWithDollarPrefixShouldSucceed) { auto chunkBound2 = BSON("a" << BSON("$mixKey" << 1)); auto chunkMax = BSON("a" << kMaxBSONKey); - // set histories - chunk1.setHistory({ChunkHistory{Timestamp{100, 9}, _shardId}}); - chunk2.setHistory({ChunkHistory{Timestamp{200, 5}, _shardId}}); - chunk3.setHistory({ChunkHistory{Timestamp{156, 1}, _shardId}}); - // first chunk boundaries chunk1.setMin(chunkMin); chunk1.setMax(chunkBound1); @@ -611,6 +569,7 @@ TEST_F(MergeChunkTest, MergingChunksWithDollarPrefixShouldSucceed) { // Record chunk boundaries for passing into commitChunksMerge ChunkRange rangeToBeMerged(chunk1.getMin(), chunk3.getMax()); + Timestamp validAfter{100, 0}; ASSERT_OK(ShardingCatalogManager::get(operationContext()) ->commitChunksMerge(operationContext(), @@ -619,7 +578,8 @@ TEST_F(MergeChunkTest, MergingChunksWithDollarPrefixShouldSucceed) { collTimestamp, collUuid, rangeToBeMerged, - _shardId)); + _shardId, + validAfter)); const auto query = BSON(ChunkType::collectionUUID() << collUuid); auto findResponse = uassertStatusOK( @@ -650,8 +610,7 @@ TEST_F(MergeChunkTest, MergingChunksWithDollarPrefixShouldSucceed) { // Make sure history is there ASSERT_EQ(1UL, mergedChunk.getHistory().size()); - ASSERT_EQ(chunk2.getHistory().front().getValidAfter(), - mergedChunk.getHistory().front().getValidAfter()); + ASSERT_EQ(validAfter, mergedChunk.getHistory().front().getValidAfter()); } } // namespace diff --git a/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp index 1fc9f4e8c85..6b15afaf5b0 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_shard_collection_test.cpp @@ -40,6 +40,7 @@ #include "mongo/rpc/metadata/tracking_metadata.h" #include "mongo/s/balancer_configuration.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/s/client/shard_registry.h" #include "mongo/s/request_types/sharded_ddl_commands_gen.h" #include "mongo/s/shard_key_pattern.h" @@ -143,11 +144,12 @@ TEST_F(CreateFirstChunksTest, NonEmptyCollection_SplitPoints_FromSplitVector_Man targeterFactory()->addTargeterToReturn(connStr, std::move(targeter)); setupShards(kShards); + shardRegistry()->reload(operationContext()); auto uuid = UUID::gen(); CollectionCatalog::write(getServiceContext(), [&](CollectionCatalog& catalog) { - catalog.registerCollection(operationContext(), - std::make_shared<CollectionMock>(uuid, kNamespace)); + catalog.registerCollection( + operationContext(), uuid, std::make_shared<CollectionMock>(kNamespace)); }); auto future = launchAsync([&] { @@ -158,7 +160,7 @@ TEST_F(CreateFirstChunksTest, NonEmptyCollection_SplitPoints_FromSplitVector_Man request.setNumInitialChunks(0); request.setPresplitHashedZones(false); auto optimization = InitialSplitPolicy::calculateOptimizationStrategy( - opCtx.get(), + operationContext(), kShardKeyPattern, request.getNumInitialChunks().get(), request.getPresplitHashedZones().get(), @@ -193,6 +195,7 @@ TEST_F(CreateFirstChunksTest, NonEmptyCollection_SplitPoints_FromClient_ManyChun targeterFactory()->addTargeterToReturn(connStr, std::move(targeter)); setupShards(kShards); + shardRegistry()->reload(operationContext()); auto future = launchAsync([&] { ThreadClient tc("Test", getServiceContext()); @@ -207,7 +210,7 @@ TEST_F(CreateFirstChunksTest, NonEmptyCollection_SplitPoints_FromClient_ManyChun request.setPresplitHashedZones(false); request.setInitialSplitPoints(splitPoints); auto optimization = InitialSplitPolicy::calculateOptimizationStrategy( - opCtx.get(), + operationContext(), kShardKeyPattern, request.getNumInitialChunks().get(), request.getPresplitHashedZones().get(), @@ -231,6 +234,7 @@ TEST_F(CreateFirstChunksTest, NonEmptyCollection_WithZones_OneChunkToPrimary) { ShardType("shard1", "rs1/shard1:123", {"TestZone"}), ShardType("shard2", "rs2/shard2:123")}; setupShards(kShards); + shardRegistry()->reload(operationContext()); std::vector<TagsType> zones{ TagsType(kNamespace, @@ -273,6 +277,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_SplitPoints_FromClient_ManyChunksD targeterFactory()->addTargeterToReturn(connStr, std::move(targeter)); setupShards(kShards); + shardRegistry()->reload(operationContext()); auto future = launchAsync([&] { ThreadClient tc("Test", getServiceContext()); @@ -287,7 +292,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_SplitPoints_FromClient_ManyChunksD request.setPresplitHashedZones(false); request.setInitialSplitPoints(splitPoints); auto optimization = InitialSplitPolicy::calculateOptimizationStrategy( - opCtx.get(), + operationContext(), kShardKeyPattern, request.getNumInitialChunks().get(), request.getPresplitHashedZones().get(), @@ -298,7 +303,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_SplitPoints_FromClient_ManyChunksD ASSERT(optimization->isOptimized()); return optimization->createFirstChunks( - opCtx.get(), kShardKeyPattern, {UUID::gen(), ShardId("shard1")}); + operationContext(), kShardKeyPattern, {UUID::gen(), ShardId("shard1")}); }); const auto& firstChunks = future.default_timed_get(); @@ -322,6 +327,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_NoSplitPoints_OneChunkToPrimary) { targeterFactory()->addTargeterToReturn(connStr, std::move(targeter)); setupShards(kShards); + shardRegistry()->reload(operationContext()); auto future = launchAsync([&] { ThreadClient tc("Test", getServiceContext()); @@ -336,7 +342,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_NoSplitPoints_OneChunkToPrimary) { request.setPresplitHashedZones(false); request.setInitialSplitPoints(splitPoints); auto optimization = InitialSplitPolicy::calculateOptimizationStrategy( - opCtx.get(), + operationContext(), kShardKeyPattern, request.getNumInitialChunks().get(), request.getPresplitHashedZones().get(), @@ -347,7 +353,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_NoSplitPoints_OneChunkToPrimary) { ASSERT(optimization->isOptimized()); return optimization->createFirstChunks( - opCtx.get(), kShardKeyPattern, {UUID::gen(), ShardId("shard1")}); + operationContext(), kShardKeyPattern, {UUID::gen(), ShardId("shard1")}); }); const auto& firstChunks = future.default_timed_get(); @@ -360,6 +366,7 @@ TEST_F(CreateFirstChunksTest, EmptyCollection_WithZones_ManyChunksOnFirstZoneSha ShardType("shard1", "rs1/shard1:123", {"TestZone"}), ShardType("shard2", "rs2/shard2:123")}; setupShards(kShards); + shardRegistry()->reload(operationContext()); std::vector<BSONObj> splitPoints{}; std::vector<TagsType> zones{ diff --git a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp index 16c2bde5d02..da32d388438 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp @@ -41,6 +41,7 @@ #include "mongo/bson/bsonobj_comparator.h" #include "mongo/bson/util/bson_extract.h" #include "mongo/client/connection_string.h" +#include "mongo/client/fetcher.h" #include "mongo/client/read_preference.h" #include "mongo/client/remote_command_targeter.h" #include "mongo/client/replica_set_monitor.h" @@ -53,7 +54,6 @@ #include "mongo/db/commands/feature_compatibility_version_parser.h" #include "mongo/db/commands/set_cluster_parameter_invocation.h" #include "mongo/db/commands/set_feature_compatibility_version_gen.h" -#include "mongo/db/keys_collection_util.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/persistent_task_store.h" @@ -61,10 +61,8 @@ #include "mongo/db/repl/hello_gen.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/repl_set_config.h" -#include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/add_shard_cmd_gen.h" #include "mongo/db/s/add_shard_util.h" -#include "mongo/db/s/sharding_config_server_parameters_gen.h" #include "mongo/db/s/sharding_logging.h" #include "mongo/db/s/type_shard_identity.h" #include "mongo/db/s/user_writes_critical_section_document_gen.h" @@ -108,8 +106,6 @@ const WriteConcernOptions kMajorityWriteConcern{WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, WriteConcernOptions::kNoTimeout}; -const Seconds kRemoteCommandTimeout{60}; - /** * Generates a unique name to be given to a newly added shard. */ @@ -157,7 +153,7 @@ StatusWith<std::string> generateNewShardName(OperationContext* opCtx) { StatusWith<Shard::CommandResponse> ShardingCatalogManager::_runCommandForAddShard( OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, + RemoteCommandTargeter* targeter, StringData dbName, const BSONObj& cmdObj) { auto swHost = targeter->findHost(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}); @@ -167,7 +163,7 @@ StatusWith<Shard::CommandResponse> ShardingCatalogManager::_runCommandForAddShar auto host = std::move(swHost.getValue()); executor::RemoteCommandRequest request( - host, dbName.toString(), cmdObj, rpc::makeEmptyMetadata(), opCtx, kRemoteCommandTimeout); + host, dbName.toString(), cmdObj, rpc::makeEmptyMetadata(), opCtx, Seconds(60)); executor::RemoteCommandResponse response = Status(ErrorCodes::InternalError, "Internal error running command"); @@ -323,8 +319,8 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( std::shared_ptr<RemoteCommandTargeter> targeter, const std::string* shardProposedName, const ConnectionString& connectionString) { - auto swCommandResponse = - _runCommandForAddShard(opCtx, targeter, NamespaceString::kAdminDb, BSON("hello" << 1)); + auto swCommandResponse = _runCommandForAddShard( + opCtx, targeter.get(), NamespaceString::kAdminDb, BSON("isMaster" << 1)); if (swCommandResponse.getStatus() == ErrorCodes::IncompatibleServerVersion) { return swCommandResponse.getStatus().withReason( str::stream() << "Cannot add " << connectionString.toString() @@ -335,16 +331,17 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( } // Check for a command response error - auto resHelloStatus = std::move(swCommandResponse.getValue().commandStatus); - if (!resHelloStatus.isOK()) { - return resHelloStatus.withContext(str::stream() << "Error running 'hello' against " - << targeter->connectionString().toString()); + auto resIsMasterStatus = std::move(swCommandResponse.getValue().commandStatus); + if (!resIsMasterStatus.isOK()) { + return resIsMasterStatus.withContext(str::stream() + << "Error running isMaster against " + << targeter->connectionString().toString()); } - auto resHello = std::move(swCommandResponse.getValue().response); + auto resIsMaster = std::move(swCommandResponse.getValue().response); // Fail if the node being added is a mongos. - const std::string msg = resHello.getStringField("msg").toString(); + const std::string msg = resIsMaster.getStringField("msg").toString(); if (msg == "isdbgrid") { return {ErrorCodes::IllegalOperation, "cannot add a mongos as a shard"}; } @@ -355,23 +352,23 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( // because of our internal wire version protocol. So we can safely invariant here that the node // is compatible. long long maxWireVersion; - Status status = bsonExtractIntegerField(resHello, "maxWireVersion", &maxWireVersion); + Status status = bsonExtractIntegerField(resIsMaster, "maxWireVersion", &maxWireVersion); if (!status.isOK()) { - return status.withContext(str::stream() << "hello returned invalid 'maxWireVersion' " + return status.withContext(str::stream() << "isMaster returned invalid 'maxWireVersion' " << "field when attempting to add " << connectionString.toString() << " as a shard"); } - // Check whether the host is a writable primary. If not, the replica set may not have been - // initiated. If the connection is a standalone, it will return true for "isWritablePrimary". - bool isWritablePrimary; - status = bsonExtractBooleanField(resHello, "isWritablePrimary", &isWritablePrimary); + // Check whether there is a master. If there isn't, the replica set may not have been + // initiated. If the connection is a standalone, it will return true for isMaster. + bool isMaster; + status = bsonExtractBooleanField(resIsMaster, "ismaster", &isMaster); if (!status.isOK()) { - return status.withContext(str::stream() << "hello returned invalid 'isWritablePrimary' " + return status.withContext(str::stream() << "isMaster returned invalid 'ismaster' " << "field when attempting to add " << connectionString.toString() << " as a shard"); } - if (!isWritablePrimary) { + if (!isMaster) { return {ErrorCodes::NotWritablePrimary, str::stream() << connectionString.toString() @@ -380,7 +377,7 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( } const std::string providedSetName = connectionString.getSetName(); - const std::string foundSetName = resHello["setName"].str(); + const std::string foundSetName = resIsMaster["setName"].str(); // Make sure the specified replica set name (if any) matches the actual shard's replica set if (providedSetName.empty() && !foundSetName.empty()) { @@ -393,7 +390,7 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( if (!providedSetName.empty() && foundSetName.empty()) { return {ErrorCodes::OperationFailed, str::stream() << "host did not return a set name; " - << "is the replica set still initializing? " << resHello}; + << "is the replica set still initializing? " << resIsMaster}; } // Make sure the set name specified in the connection string matches the one where its hosts @@ -405,14 +402,14 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( } // Is it a config server? - if (resHello.hasField("configsvr")) { + if (resIsMaster.hasField("configsvr")) { return {ErrorCodes::OperationFailed, str::stream() << "Cannot add " << connectionString.toString() << " as a shard since it is a config server"}; } - if (resHello.hasField(HelloCommandReply::kIsImplicitDefaultMajorityWCFieldName) && - !resHello.getBoolField(HelloCommandReply::kIsImplicitDefaultMajorityWCFieldName) && + if (resIsMaster.hasField(HelloCommandReply::kIsImplicitDefaultMajorityWCFieldName) && + !resIsMaster.getBoolField(HelloCommandReply::kIsImplicitDefaultMajorityWCFieldName) && !ReadWriteConcernDefaults::get(opCtx).isCWWCSet(opCtx)) { return { ErrorCodes::OperationFailed, @@ -425,11 +422,11 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( "using the setDefaultRWConcern command and try again."}; } - if (resHello.hasField(HelloCommandReply::kCwwcFieldName)) { - auto cwwcOnShard = - WriteConcernOptions::parse(resHello.getObjectField(HelloCommandReply::kCwwcFieldName)) - .getValue() - .toBSON(); + if (resIsMaster.hasField(HelloCommandReply::kCwwcFieldName)) { + auto cwwcOnShard = WriteConcernOptions::parse( + resIsMaster.getObjectField(HelloCommandReply::kCwwcFieldName)) + .getValue() + .toBSON(); auto cachedCWWC = ReadWriteConcernDefaults::get(opCtx).getCWWC(opCtx); if (!cachedCWWC) { @@ -462,20 +459,20 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( if (!providedSetName.empty()) { std::set<std::string> hostSet; - BSONObjIterator iter(resHello["hosts"].Obj()); + BSONObjIterator iter(resIsMaster["hosts"].Obj()); while (iter.more()) { hostSet.insert(iter.next().String()); // host:port } - if (resHello["passives"].isABSONObj()) { - BSONObjIterator piter(resHello["passives"].Obj()); + if (resIsMaster["passives"].isABSONObj()) { + BSONObjIterator piter(resIsMaster["passives"].Obj()); while (piter.more()) { hostSet.insert(piter.next().String()); // host:port } } - if (resHello["arbiters"].isABSONObj()) { - BSONObjIterator piter(resHello["arbiters"].Obj()); + if (resIsMaster["arbiters"].isABSONObj()) { + BSONObjIterator piter(resIsMaster["arbiters"].Obj()); while (piter.more()) { hostSet.insert(piter.next().String()); // host:port } @@ -487,7 +484,7 @@ StatusWith<ShardType> ShardingCatalogManager::_validateHostAsShard( return {ErrorCodes::OperationFailed, str::stream() << "in seed list " << connectionString.toString() << ", host " << host << " does not belong to replica set " << foundSetName - << "; found " << resHello.toString()}; + << "; found " << resIsMaster.toString()}; } } } @@ -529,7 +526,7 @@ Status ShardingCatalogManager::_dropSessionsCollection( } auto swCommandResponse = _runCommandForAddShard( - opCtx, targeter, NamespaceString::kLogicalSessionsNamespace.db(), builder.done()); + opCtx, targeter.get(), NamespaceString::kLogicalSessionsNamespace.db(), builder.done()); if (!swCommandResponse.isOK()) { return swCommandResponse.getStatus(); } @@ -547,7 +544,7 @@ StatusWith<std::vector<std::string>> ShardingCatalogManager::_getDBNamesListFrom auto swCommandResponse = _runCommandForAddShard(opCtx, - targeter, + targeter.get(), NamespaceString::kAdminDb, BSON("listDatabases" << 1 << "nameOnly" << true)); if (!swCommandResponse.isOK()) { @@ -657,11 +654,6 @@ StatusWith<std::string> ShardingCatalogManager::addShard( "collection from the shard manually and try again."); } - auto pullKeysStatus = _pullClusterTimeKeys(opCtx, targeter); - if (!pullKeysStatus.isOK()) { - return pullKeysStatus; - } - // If a name for a shard wasn't provided, generate one if (shardType.getName().empty()) { auto result = generateNewShardName(opCtx); @@ -678,7 +670,7 @@ StatusWith<std::string> ShardingCatalogManager::addShard( // Helper function that runs a command on the to-be shard and returns the status auto runCmdOnNewShard = [this, &opCtx, &targeter](const BSONObj& cmd) -> Status { auto swCommandResponse = - _runCommandForAddShard(opCtx, targeter, NamespaceString::kAdminDb, cmd); + _runCommandForAddShard(opCtx, targeter.get(), NamespaceString::kAdminDb, cmd); if (!swCommandResponse.isOK()) { return swCommandResponse.getStatus(); } @@ -700,10 +692,10 @@ StatusWith<std::string> ShardingCatalogManager::addShard( } // Set the user-writes blocking state on the new shard. - _setUserWriteBlockingStateOnNewShard(opCtx, targeter); + _setUserWriteBlockingStateOnNewShard(opCtx, targeter.get()); // Determine the set of cluster parameters to be used. - _standardizeClusterParameters(opCtx, targeter); + _standardizeClusterParameters(opCtx, targeter.get()); { // Keep the FCV stable across checking the FCV, sending setFCV to the new shard and writing @@ -729,7 +721,7 @@ StatusWith<std::string> ShardingCatalogManager::addShard( auto versionResponse = _runCommandForAddShard(opCtx, - targeter, + targeter.get(), NamespaceString::kAdminDb, setFcvCmd.toBSON(BSON(WriteConcernOptions::kWriteConcernField << opCtx->getWriteConcern().toBSON()))); @@ -1067,8 +1059,8 @@ StatusWith<long long> ShardingCatalogManager::_runCountCommandOnConfig(Operation return result; } -void ShardingCatalogManager::_setUserWriteBlockingStateOnNewShard( - OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) { +void ShardingCatalogManager::_setUserWriteBlockingStateOnNewShard(OperationContext* opCtx, + RemoteCommandTargeter* targeter) { // Delete all the config.user_writes_critical_sections documents from the new shard. { write_ops::DeleteCommandRequest deleteOp( @@ -1134,109 +1126,6 @@ void ShardingCatalogManager::_setUserWriteBlockingStateOnNewShard( }); } -std::unique_ptr<Fetcher> ShardingCatalogManager::_createFetcher( - OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, - const NamespaceString& nss, - const repl::ReadConcernLevel& readConcernLevel, - FetcherDocsCallbackFn processDocsCallback, - FetcherStatusCallbackFn processStatusCallback) { - auto host = uassertStatusOK( - targeter->findHost(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly})); - - FindCommandRequest findCommand(nss); - const auto readConcern = - repl::ReadConcernArgs(boost::optional<repl::ReadConcernLevel>(readConcernLevel)); - findCommand.setReadConcern(readConcern.toBSONInner()); - const Milliseconds maxTimeMS = - std::min(opCtx->getRemainingMaxTimeMillis(), Milliseconds(kRemoteCommandTimeout)); - findCommand.setMaxTimeMS(durationCount<Milliseconds>(maxTimeMS)); - - auto fetcherCallback = [processDocsCallback, - processStatusCallback](const Fetcher::QueryResponseStatus& dataStatus, - Fetcher::NextAction* nextAction, - BSONObjBuilder* getMoreBob) { - // Throw out any accumulated results on error. - if (!dataStatus.isOK()) { - processStatusCallback(dataStatus.getStatus()); - return; - } - const auto& data = dataStatus.getValue(); - - try { - if (!processDocsCallback(data.documents)) { - *nextAction = Fetcher::NextAction::kNoAction; - } - } catch (DBException& ex) { - processStatusCallback(ex.toStatus()); - return; - } - processStatusCallback(Status::OK()); - - if (!getMoreBob) { - return; - } - getMoreBob->append("getMore", data.cursorId); - getMoreBob->append("collection", data.nss.coll()); - }; - - return std::make_unique<Fetcher>(_executorForAddShard.get(), - host, - nss.db().toString(), - findCommand.toBSON({}), - fetcherCallback, - BSONObj(), /* metadata tracking, only used for shards */ - maxTimeMS, /* command network timeout */ - maxTimeMS /* getMore network timeout */); -} - -Status ShardingCatalogManager::_pullClusterTimeKeys( - OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) { - Status fetchStatus = - Status(ErrorCodes::InternalError, "Internal error running cursor callback in command"); - std::vector<ExternalKeysCollectionDocument> keyDocs; - - auto expireAt = opCtx->getServiceContext()->getFastClockSource()->now() + - Seconds(gNewShardExistingClusterTimeKeysExpirationSecs.load()); - auto fetcher = _createFetcher( - opCtx, - targeter, - NamespaceString::kKeysCollectionNamespace, - repl::ReadConcernLevel::kLocalReadConcern, - [&](const std::vector<BSONObj>& docs) -> bool { - for (const BSONObj& doc : docs) { - keyDocs.push_back(keys_collection_util::makeExternalClusterTimeKeyDoc( - doc.getOwned(), boost::none /* migrationId */, expireAt)); - } - return true; - }, - [&](const Status& status) { fetchStatus = status; }); - - auto scheduleStatus = fetcher->schedule(); - if (!scheduleStatus.isOK()) { - return scheduleStatus; - } - - auto joinStatus = fetcher->join(opCtx); - if (!joinStatus.isOK()) { - return joinStatus; - } - - if (keyDocs.empty()) { - return fetchStatus; - } - - auto opTime = keys_collection_util::storeExternalClusterTimeKeyDocs(opCtx, std::move(keyDocs)); - auto waitStatus = WaitForMajorityService::get(opCtx->getServiceContext()) - .waitUntilMajority(opTime, opCtx->getCancellationToken()) - .getNoThrow(); - if (!waitStatus.isOK()) { - return waitStatus; - } - - return fetchStatus; -} - void ShardingCatalogManager::_setClusterParametersLocally(OperationContext* opCtx, const std::vector<BSONObj>& parameters) { DBDirectClient client(opCtx); @@ -1256,40 +1145,78 @@ void ShardingCatalogManager::_setClusterParametersLocally(OperationContext* opCt } } -void ShardingCatalogManager::_pullClusterParametersFromNewShard( - OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) { +void ShardingCatalogManager::_pullClusterParametersFromNewShard(OperationContext* opCtx, + RemoteCommandTargeter* targeter) { LOGV2(6538600, "Pulling cluster parameters from new shard"); // We can safely query the cluster parameters because the replica set must have been started // with --shardsvr in order to add it into the cluster, and in this mode no setClusterParameter // can be called on the replica set directly. + auto host = uassertStatusOK( + targeter->findHost(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly})); + + const Milliseconds maxTimeMS = + std::min(opCtx->getRemainingMaxTimeMillis(), Milliseconds(Seconds{30})); + BSONObjBuilder findCmdBuilder; + { + FindCommandRequest findCommand(NamespaceString::kClusterParametersNamespace); + auto readConcern = repl::ReadConcernArgs( + boost::optional<repl::ReadConcernLevel>(repl::ReadConcernLevel::kMajorityReadConcern)); + findCommand.setReadConcern(readConcern.toBSONInner()); + findCommand.setMaxTimeMS(durationCount<Milliseconds>(maxTimeMS)); + findCommand.serialize(BSONObj(), &findCmdBuilder); + } // If for some reason the callback never gets invoked, we will return this status in response. - Status fetchStatus = + Status status = Status(ErrorCodes::InternalError, "Internal error running cursor callback in command"); + std::vector<BSONObj> parameters; + auto fetcherCallback = + [this, &status, ¶meters](const Fetcher::QueryResponseStatus& dataStatus, + Fetcher::NextAction* nextAction, + BSONObjBuilder* getMoreBob) { + // Throw out any accumulated results on error + if (!dataStatus.isOK()) { + status = dataStatus.getStatus(); + return; + } + const auto& data = dataStatus.getValue(); + + for (const BSONObj& doc : data.documents) { + parameters.push_back(doc.getOwned()); + } + + status = Status::OK(); + + if (!getMoreBob) { + return; + } + getMoreBob->append("getMore", data.cursorId); + getMoreBob->append("collection", data.nss.coll()); + }; + + Fetcher fetcher(_executorForAddShard.get(), + std::move(host), + NamespaceString::kClusterParametersNamespace.db().toString(), + findCmdBuilder.obj(), + fetcherCallback, + BSONObj(), /* metadata tracking, only used for shards */ + maxTimeMS, /* command network timeout */ + maxTimeMS /* getMore network timeout */); + + uassertStatusOK(fetcher.schedule()); + + uassertStatusOK(fetcher.join(opCtx)); + + uassertStatusOK(status); - auto fetcher = _createFetcher(opCtx, - targeter, - NamespaceString::kClusterParametersNamespace, - repl::ReadConcernLevel::kMajorityReadConcern, - [¶meters](const std::vector<BSONObj>& docs) -> bool { - for (const BSONObj& doc : docs) { - parameters.push_back(doc.getOwned()); - } - return true; - }, - [&fetchStatus](const Status& status) { fetchStatus = status; }); - - uassertStatusOK(fetcher->schedule()); - uassertStatusOK(fetcher->join(opCtx)); - uassertStatusOK(fetchStatus); _setClusterParametersLocally(opCtx, parameters); } void ShardingCatalogManager::_pushClusterParametersToNewShard( OperationContext* opCtx, - std::shared_ptr<RemoteCommandTargeter> targeter, + RemoteCommandTargeter* targeter, const std::vector<BSONObj>& clusterParameters) { LOGV2(6360600, "Pushing cluster parameters into new shard"); @@ -1325,8 +1252,8 @@ void ShardingCatalogManager::_pushClusterParametersToNewShard( } } -void ShardingCatalogManager::_standardizeClusterParameters( - OperationContext* opCtx, std::shared_ptr<RemoteCommandTargeter> targeter) { +void ShardingCatalogManager::_standardizeClusterParameters(OperationContext* opCtx, + RemoteCommandTargeter* targeter) { if (!gFeatureFlagClusterWideConfig.isEnabled(serverGlobalParams.featureCompatibility)) return; diff --git a/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp index fb288258edd..cd7f46ebfda 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp @@ -300,72 +300,6 @@ TEST_F(SplitChunkTest, NewSplitShouldClaimHighestVersion) { test(_nss2, Timestamp(42)); } -TEST_F(SplitChunkTest, Idempotency) { - auto test = [&](const NamespaceString& nss, - const Timestamp& collTimestamp, - BSONObj chunkMin, - BSONObj chunkMax, - const std::vector<BSONObj>& splitPoints) { - const auto collEpoch = OID::gen(); - const auto collUuid = UUID::gen(); - - ChunkType chunk; - chunk.setName(OID::gen()); - chunk.setCollectionUUID(collUuid); - - auto origVersion = ChunkVersion(1, 0, collEpoch, collTimestamp); - chunk.setVersion(origVersion); - chunk.setShard(ShardId(_shardName)); - - chunk.setMin(chunkMin); - chunk.setMax(chunkMax); - - setupCollection(nss, _keyPattern, {chunk}); - - const auto doSplit = [&]() { - return ShardingCatalogManager::get(operationContext()) - ->commitChunkSplit(operationContext(), - nss, - collEpoch, - collTimestamp, - ChunkRange(chunkMin, chunkMax), - splitPoints, - "shard0000", - false); - }; - - // Split. - ASSERT_OK(doSplit().getStatus()); - // Retry. - ASSERT_OK(doSplit().getStatus()); - - const auto verifyChunk = [&](BSONObj min, BSONObj max) { - auto chunkDocStatus = - getChunkDoc(operationContext(), collUuid, min, collEpoch, collTimestamp); - ASSERT_OK(chunkDocStatus.getStatus()); - - auto chunkDoc = chunkDocStatus.getValue(); - ASSERT_BSONOBJ_EQ(max, chunkDoc.getMax()); - }; - - // Sanity check. - std::vector<BSONObj> expectedChunkBounds; - expectedChunkBounds.push_back(chunkMin); - expectedChunkBounds.insert( - expectedChunkBounds.end(), splitPoints.begin(), splitPoints.end()); - expectedChunkBounds.push_back(chunkMax); - - for (auto minIt = expectedChunkBounds.begin(); minIt != expectedChunkBounds.end() - 1; - ++minIt) { - auto maxIt = minIt + 1; - verifyChunk(*minIt, *maxIt); - } - }; - - test(_nss1, Timestamp(42), BSON("a" << 1), BSON("a" << 10), {BSON("a" << 5)}); - test(_nss2, Timestamp(42), BSON("a" << 1), BSON("a" << 10), {BSON("a" << 3), BSON("a" << 7)}); -} - TEST_F(SplitChunkTest, PreConditionFailErrors) { auto test = [&](const NamespaceString& nss, const Timestamp& collTimestamp) { const auto collEpoch = OID::gen(); @@ -683,54 +617,5 @@ TEST_F(SplitChunkTest, CantCommitSplitFromChunkSplitterDuringDefragmentation) { false /* fromChunkSplitter*/)); } -TEST_F(SplitChunkTest, SplitJumboChunkShouldUnsetJumboFlag) { - const auto& nss = _nss2; - const auto collTimestamp = Timestamp(42); - const auto collEpoch = OID::gen(); - const auto collUuid = UUID::gen(); - - ChunkType chunk; - chunk.setName(OID::gen()); - chunk.setCollectionUUID(collUuid); - - auto origVersion = ChunkVersion(1, 0, collEpoch, collTimestamp); - chunk.setVersion(origVersion); - chunk.setShard(ShardId(_shardName)); - chunk.setJumbo(true); - - auto chunkMin = BSON("a" << 1); - auto chunkMax = BSON("a" << 10); - chunk.setMin(chunkMin); - chunk.setMax(chunkMax); - - auto chunkSplitPoint = BSON("a" << 5); - std::vector<BSONObj> splitPoints{chunkSplitPoint}; - - setupCollection(nss, _keyPattern, {chunk}); - - ASSERT_EQ(true, chunk.getJumbo()); - - uassertStatusOK(ShardingCatalogManager::get(operationContext()) - ->commitChunkSplit(operationContext(), - nss, - collEpoch, - collTimestamp, - ChunkRange(chunkMin, chunkMax), - splitPoints, - "shard0000", - false /* fromChunkSplitter*/)); - - // Both resulting chunks must not be jumbo - auto chunkDocLeft = - getChunkDoc(operationContext(), collUuid, chunkMin, collEpoch, collTimestamp); - ASSERT_OK(chunkDocLeft.getStatus()); - - auto chunkDocRight = - getChunkDoc(operationContext(), collUuid, chunkSplitPoint, collEpoch, collTimestamp); - ASSERT_OK(chunkDocRight.getStatus()); - - ASSERT_EQ(false, chunkDocLeft.getValue().getJumbo()); - ASSERT_EQ(false, chunkDocRight.getValue().getJumbo()); -} } // namespace } // namespace mongo diff --git a/src/mongo/db/s/config/sharding_catalog_manager_zone_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_zone_operations.cpp index f293963ff50..ec31cf0ca9f 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_zone_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_zone_operations.cpp @@ -66,7 +66,7 @@ Status checkForOverlappingZonedKeyRange(OperationContext* opCtx, const ChunkRange& range, const std::string& zoneName, const KeyPattern& shardKeyPattern) { - ZoneInfo zoneInfo; + DistributionStatus chunkDist(nss, ShardToChunksMap{}); auto tagStatus = configServer->exhaustiveFindOnConfig(opCtx, kConfigPrimarySelector, @@ -89,7 +89,7 @@ Status checkForOverlappingZonedKeyRange(OperationContext* opCtx, // Always extend ranges to full shard key to be compatible with tags created before // the zone commands were implemented. const auto& parsedTagDoc = tagParseStatus.getValue(); - auto overlapStatus = zoneInfo.addRangeToZone( + auto overlapStatus = chunkDist.addRangeToZone( ZoneRange(shardKeyPattern.extendRangeBound(parsedTagDoc.getMinKey(), false), shardKeyPattern.extendRangeBound(parsedTagDoc.getMaxKey(), false), parsedTagDoc.getTag())); @@ -99,7 +99,7 @@ Status checkForOverlappingZonedKeyRange(OperationContext* opCtx, } auto overlapStatus = - zoneInfo.addRangeToZone(ZoneRange(range.getMin(), range.getMax(), zoneName)); + chunkDist.addRangeToZone(ZoneRange(range.getMin(), range.getMax(), zoneName)); if (!overlapStatus.isOK()) { return overlapStatus; } diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index 2acd0017ab7..18e83a4b994 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -208,10 +208,6 @@ public: size_t numberOfPrePostImagesToWrite, Date_t wallClockTime) override {} - void onTransactionPrepareNonPrimary(OperationContext* opCtx, - const std::vector<repl::OplogEntry>& statements, - const repl::OpTime& prepareOpTime) override {} - void onTransactionAbort(OperationContext* opCtx, boost::optional<OplogSlot> abortOplogEntryOpTime) override {} diff --git a/src/mongo/db/s/create_collection_coordinator.cpp b/src/mongo/db/s/create_collection_coordinator.cpp index 85b2c655c37..7501bd05b9b 100644 --- a/src/mongo/db/s/create_collection_coordinator.cpp +++ b/src/mongo/db/s/create_collection_coordinator.cpp @@ -39,7 +39,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/commands/create_gen.h" #include "mongo/db/commands/feature_compatibility_version.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/s/collection_sharding_runtime.h" @@ -52,7 +52,6 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/timeseries/catalog_helper.h" #include "mongo/db/timeseries/timeseries_constants.h" -#include "mongo/db/vector_clock_mutable.h" #include "mongo/logv2/log.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/s/cluster_commands_helpers.h" @@ -60,11 +59,6 @@ #include "mongo/s/grid.h" #include "mongo/s/sharding_feature_flags_gen.h" - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding -MONGO_FAIL_POINT_DEFINE(failAtCommitCreateCollectionCoordinator); - - namespace mongo { namespace { @@ -293,15 +287,11 @@ void insertChunks(OperationContext* opCtx, for (const auto& chunk : chunks) { entries.push_back(chunk.toConfigBSON()); } - insertOp.setDocuments(std::move(entries)); - insertOp.setWriteCommandRequestBase([] { - write_ops::WriteCommandRequestBase wcb; - wcb.setOrdered(false); - return wcb; - }()); + insertOp.setDocuments(entries); return insertOp; }()); + insertRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); { auto newClient = opCtx->getServiceContext()->makeClient("CreateCollectionCoordinator::insertChunks"); @@ -317,7 +307,6 @@ void insertChunks(OperationContext* opCtx, cc().makeOperationContext(), opCtx->getCancellationToken(), executor); newOpCtx->setLogicalSessionId(*osi.getSessionId()); newOpCtx->setTxnNumber(*osi.getTxnNumber()); - newOpCtx->setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern); BatchedCommandResponse response; BatchWriteExecStats stats; @@ -334,14 +323,16 @@ void insertCollectionEntry(OperationContext* opCtx, BatchedCommandRequest insertRequest( write_ops::InsertCommandRequest(CollectionType::ConfigNS, {coll.toBSON()})); - const auto cmdObj = CommandHelpers::appendMajorityWriteConcern(insertRequest.toBSON()); + insertRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); + + const BSONObj cmdObj = insertRequest.toBSON().addFields(osi.toBSON()); BatchedCommandResponse unusedResponse; uassertStatusOK(Shard::CommandResponse::processBatchWriteResponse( configShard->runCommand(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, CollectionType::ConfigNS.db().toString(), - cmdObj.addFields(osi.toBSON()), + cmdObj, Shard::kDefaultConfigCommandTimeout, Shard::RetryPolicy::kIdempotent), &unusedResponse)); @@ -351,8 +342,8 @@ void broadcastDropCollection(OperationContext* opCtx, const NamespaceString& nss, const std::shared_ptr<executor::TaskExecutor>& executor, const OperationSessionInfo& osi) { - const ShardsvrDropCollectionParticipant dropCollectionParticipant(nss); const auto primaryShardId = ShardingState::get(opCtx)->shardId(); + const ShardsvrDropCollectionParticipant dropCollectionParticipant(nss); auto participants = Grid::get(opCtx)->shardRegistry()->getAllShardIds(opCtx); // Remove primary shard from participants @@ -360,7 +351,7 @@ void broadcastDropCollection(OperationContext* opCtx, participants.end()); sharding_ddl_util::sendDropCollectionParticipantCommandToShards( - opCtx, nss, participants, executor, osi, true /* fromMigrate */); + opCtx, nss, participants, executor, osi); } /** @@ -441,14 +432,11 @@ void CreateCollectionCoordinator::checkIfOptionsConflict(const BSONObj& doc) con const auto otherDoc = CreateCollectionCoordinatorDocument::parse( IDLParserErrorContext("CreateCollectionCoordinatorDocument"), doc); - const auto& selfReq = _request.toBSON(); - const auto& otherReq = otherDoc.getCreateCollectionRequest().toBSON(); - uassert(ErrorCodes::ConflictingOperationInProgress, - str::stream() << "Another create collection with different arguments is already " - "running for the same namespace: " - << selfReq, - SimpleBSONObjComparator::kInstance.evaluate(selfReq == otherReq)); + "Another create collection with different arguments is already running for the same " + "namespace", + SimpleBSONObjComparator::kInstance.evaluate( + _request.toBSON() == otherDoc.getCreateCollectionRequest().toBSON())); } ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( @@ -488,8 +476,6 @@ ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( // Log the start of the event only if we're not recovering. _logStartCreateCollection(opCtx); - _checkCollectionUUIDMismatch(opCtx); - // Quick check (without critical section) to see if another create collection // already succeeded. if (auto createCollectionResponseOpt = @@ -499,13 +485,7 @@ ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( _shardKeyPattern->getKeyPattern().toBSON(), getCollation(opCtx, nss(), _request.getCollation()).second, _request.getUnique().value_or(false))) { - - // Ensure that the completion of the request gets recorded at least once on the - // oplog. - _writeOplogMessage(opCtx, - nss(), - *createCollectionResponseOpt->getCollectionUUID(), - _request.toBSON()); + _checkCollectionUUIDMismatch(opCtx); // The critical section can still be held here if the node committed the // sharding of the collection but then it stepped down before it managed to @@ -515,15 +495,9 @@ ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( opCtx, nss(), _getCriticalSectionReason(), - ShardingCatalogClient::kMajorityWriteConcern, - false /* throwIfReasonDiffers */); + ShardingCatalogClient::kMajorityWriteConcern); _result = createCollectionResponseOpt; - - // Checkpoint configTime in order to preserve causality of operations in case of - // a stepdown. - VectorClockMutable::get(opCtx)->waitForDurable().get(opCtx); - return; } @@ -557,6 +531,7 @@ ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( } } + _checkCollectionUUIDMismatch(opCtx); _createPolicy(opCtx); _createCollectionAndIndexes(opCtx); @@ -627,8 +602,7 @@ ExecutorFuture<void> CreateCollectionCoordinator::_runImpl( opCtx, nss(), _getCriticalSectionReason(), - ShardingCatalogClient::kMajorityWriteConcern, - false /* throwIfReasonDiffers */); + ShardingCatalogClient::kMajorityWriteConcern); } return status; }); @@ -818,8 +792,7 @@ void CreateCollectionCoordinator::_createPolicy(OperationContext* opCtx) { _request.getInitialSplitPoints(), getTagsAndValidate(opCtx, nss(), _shardKeyPattern->toBSON()), getNumShards(opCtx), - *_collectionEmpty, - !feature_flags::gNoMoreAutoSplitter.isEnabled(serverGlobalParams.featureCompatibility)); + *_collectionEmpty); } void CreateCollectionCoordinator::_createChunks(OperationContext* opCtx) { @@ -899,12 +872,6 @@ void CreateCollectionCoordinator::_createCollectionOnNonPrimaryShards( void CreateCollectionCoordinator::_commit(OperationContext* opCtx) { LOGV2_DEBUG(5277906, 2, "Create collection _commit", "namespace"_attr = nss()); - if (MONGO_unlikely(failAtCommitCreateCollectionCoordinator.shouldFail())) { - LOGV2_DEBUG(6960301, 2, "About to hit failAtCommitCreateCollectionCoordinator fail point"); - uasserted(ErrorCodes::InterruptedAtShutdown, - "failAtCommitCreateCollectionCoordinator fail point"); - } - // Upsert Chunks. _doc = _updateSession(opCtx, _doc); insertChunks(opCtx, _initialChunks->chunks, getCurrentSession(_doc)); @@ -954,9 +921,6 @@ void CreateCollectionCoordinator::_commit(OperationContext* opCtx) { throw; } - // Checkpoint configTime in order to preserve causality of operations in case of a stepdown. - VectorClockMutable::get(opCtx)->waitForDurable().get(opCtx); - // Best effort refresh to warm up cache of all involved shards so we can have a cluster ready to // receive operations. auto shardRegistry = Grid::get(opCtx)->shardRegistry(); diff --git a/src/mongo/db/s/database_sharding_state.cpp b/src/mongo/db/s/database_sharding_state.cpp index aed788ee1fb..a4c0dff2667 100644 --- a/src/mongo/db/s/database_sharding_state.cpp +++ b/src/mongo/db/s/database_sharding_state.cpp @@ -121,12 +121,10 @@ std::shared_ptr<DatabaseShardingState> DatabaseShardingState::getSharedForLockFr } void DatabaseShardingState::enterCriticalSectionCatchUpPhase(OperationContext* opCtx, - DSSLock& dssLock, + DSSLock&, const BSONObj& reason) { invariant(opCtx->lockState()->isDbLockedForMode(_dbName, MODE_X)); _critSec.enterCriticalSectionCatchUpPhase(reason); - - cancelDbMetadataRefresh(dssLock); } void DatabaseShardingState::enterCriticalSectionCommitPhase(OperationContext* opCtx, @@ -222,27 +220,4 @@ void DatabaseShardingState::clearMovePrimarySourceManager(OperationContext* opCt _sourceMgr = nullptr; } -void DatabaseShardingState::setDbMetadataRefreshFuture(SharedSemiFuture<void> future, - CancellationSource cancellationSource, - const DSSLock&) { - invariant(!_dbMetadataRefresh); - _dbMetadataRefresh.emplace(std::move(future), std::move(cancellationSource)); -} - -boost::optional<SharedSemiFuture<void>> DatabaseShardingState::getDbMetadataRefreshFuture( - const DSSLock&) const { - return _dbMetadataRefresh ? boost::optional<SharedSemiFuture<void>>(_dbMetadataRefresh->future) - : boost::none; -} - -void DatabaseShardingState::resetDbMetadataRefreshFuture(const DSSLock&) { - _dbMetadataRefresh = boost::none; -} - -void DatabaseShardingState::cancelDbMetadataRefresh(const DSSLock&) { - if (_dbMetadataRefresh) { - _dbMetadataRefresh->cancellationSource.cancel(); - } -} - } // namespace mongo diff --git a/src/mongo/db/s/database_sharding_state.h b/src/mongo/db/s/database_sharding_state.h index 0431e801df8..4a263405c2b 100644 --- a/src/mongo/db/s/database_sharding_state.h +++ b/src/mongo/db/s/database_sharding_state.h @@ -74,13 +74,6 @@ public: StringData dbName); /** - * Returns the name of the database related to the current sharding state. - */ - std::string getDbName() const { - return _dbName; - } - - /** * Checks if this shard is the primary shard for the given DB. * * Throws an IllegalOperation exception otherwise. @@ -102,10 +95,6 @@ public: return _critSec.getSignal(op); } - auto getCriticalSectionReason(DSSLock&) const { - return _critSec.getReason() ? _critSec.getReason()->toString() : "Unknown"; - } - /** * Returns this shard server's cached dbVersion, if one is cached. * @@ -160,43 +149,9 @@ public: */ void clearMovePrimarySourceManager(OperationContext* opCtx); - /** - * Sets the database metadata refresh future for other threads to wait on it. - */ - void setDbMetadataRefreshFuture(SharedSemiFuture<void> future, - CancellationSource cancellationSource, - const DSSLock&); - - /** - * If there is an ongoing database metadata refresh, returns the future to wait on it, otherwise - * `boost::none`. - */ - boost::optional<SharedSemiFuture<void>> getDbMetadataRefreshFuture(const DSSLock&) const; - - /** - * Resets the database metadata refresh future to `boost::none`. - */ - void resetDbMetadataRefreshFuture(const DSSLock&); - - /** - * Cancel any ongoing database metadata refresh. - */ - void cancelDbMetadataRefresh(const DSSLock&); - private: friend DSSLock; - struct DbMetadataRefresh { - DbMetadataRefresh(SharedSemiFuture<void> future, CancellationSource cancellationSource) - : future(std::move(future)), cancellationSource(std::move(cancellationSource)){}; - - // Tracks the ongoing database metadata refresh. - SharedSemiFuture<void> future; - - // Cancellation source to cancel the ongoing database metadata refresh. - CancellationSource cancellationSource; - }; - // Object-wide ResourceMutex to protect changes to the DatabaseShardingState or objects held // within. Lock::ResourceMutex _stateChangeMutex{"DatabaseShardingState"}; @@ -218,10 +173,6 @@ private: // // NOTE: The source manager is not owned by this class. MovePrimarySourceManager* _sourceMgr{nullptr}; - - // Tracks the ongoing database metadata refresh. Possibly keeps a future for other threads to - // wait on it, and a cancellation source to cancel the ongoing database metadata refresh. - boost::optional<DbMetadataRefresh> _dbMetadataRefresh; }; } // namespace mongo diff --git a/src/mongo/db/s/database_sharding_state_test.cpp b/src/mongo/db/s/database_sharding_state_test.cpp index fff4ed3cccd..81d5218dbd4 100644 --- a/src/mongo/db/s/database_sharding_state_test.cpp +++ b/src/mongo/db/s/database_sharding_state_test.cpp @@ -94,16 +94,14 @@ public: StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { return repl::OpTimeWith<std::vector<ShardType>>(_shards); } - std::vector<CollectionType> getCollections(OperationContext* opCtx, - StringData dbName, - repl::ReadConcernLevel readConcernLevel, - const BSONObj& sort) override { + std::vector<CollectionType> getCollections( + OperationContext* opCtx, + StringData dbName, + repl::ReadConcernLevel readConcernLevel) override { return _colls; } @@ -170,7 +168,7 @@ TEST_F(DatabaseShardingStateTestWithMockedLoader, ForceDatabaseRefresh) { auto opCtx = operationContext(); _mockCatalogCacheLoader->setDatabaseRefreshReturnValue(newDb); - ASSERT_OK(onDbVersionMismatchNoExcept(opCtx, kDbName, boost::none)); + forceDatabaseRefresh(opCtx, kDbName); boost::optional<DatabaseVersion> activeDbVersion = [&] { AutoGetDb autoDb(opCtx, kDbName, MODE_IS); diff --git a/src/mongo/db/s/dist_lock_catalog_replset.cpp b/src/mongo/db/s/dist_lock_catalog_replset.cpp index 15e5c76e492..5a0dd02d83d 100644 --- a/src/mongo/db/s/dist_lock_catalog_replset.cpp +++ b/src/mongo/db/s/dist_lock_catalog_replset.cpp @@ -377,15 +377,38 @@ Status DistLockCatalogImpl::unlockAll(OperationContext* opCtx, }()}); return updateOp; }()); + request.setWriteConcern(kLocalWriteConcern.toBSON()); + + BSONObj cmdObj = request.toBSON(); auto const shardRegistry = Grid::get(opCtx)->shardRegistry(); - auto batchResponse = - shardRegistry->getConfigShard()->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - request, - kLocalWriteConcern, - Shard::RetryPolicy::kIdempotent); + auto response = shardRegistry->getConfigShard()->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + _locksNS.db().toString(), + cmdObj, + Shard::kDefaultConfigCommandTimeout, + Shard::RetryPolicy::kIdempotent); + if (!response.isOK()) { + return response.getStatus(); + } + if (!response.getValue().commandStatus.isOK()) { + return response.getValue().commandStatus; + } + if (!response.getValue().writeConcernStatus.isOK()) { + return response.getValue().writeConcernStatus; + } + + BatchedCommandResponse batchResponse; + std::string errmsg; + if (!batchResponse.parseBSON(response.getValue().response, &errmsg)) { + return Status(ErrorCodes::FailedToParse, + str::stream() + << "Failed to parse config server response to batch request for " + "unlocking existing distributed locks" + << causedBy(errmsg)); + } return batchResponse.toStatus(); } diff --git a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp index 63b9d91700c..a05f6db250d 100644 --- a/src/mongo/db/s/dist_lock_catalog_replset_test.cpp +++ b/src/mongo/db/s/dist_lock_catalog_replset_test.cpp @@ -1234,6 +1234,8 @@ TEST_F(DistLockCatalogReplSetTest, BasicUnlockAll) { const auto opMsgRequest(OpMsgRequest::fromDBAndBody(request.dbname, request.cmdObj)); const auto commandRequest(BatchedCommandRequest::parseUpdate(opMsgRequest)); + ASSERT_BSONOBJ_EQ(BSON("w" << 1 << "wtimeout" << 0), commandRequest.getWriteConcern()); + const auto& updateOp = commandRequest.getUpdateRequest(); ASSERT_EQUALS(LocksType::ConfigNS, updateOp.getNamespace()); diff --git a/src/mongo/db/s/drop_collection_coordinator.cpp b/src/mongo/db/s/drop_collection_coordinator.cpp index 16269e0f9e8..a9591eae302 100644 --- a/src/mongo/db/s/drop_collection_coordinator.cpp +++ b/src/mongo/db/s/drop_collection_coordinator.cpp @@ -76,39 +76,27 @@ boost::optional<BSONObj> DropCollectionCoordinator::reportForCurrentOp( return bob.obj(); } -void DropCollectionCoordinator::dropCollectionLocally(OperationContext* opCtx, - const NamespaceString& nss, - bool fromMigrate) { +DropReply DropCollectionCoordinator::dropCollectionLocally(OperationContext* opCtx, + const NamespaceString& nss) { { // Clear CollectionShardingRuntime entry Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); Lock::CollectionLock collLock(opCtx, nss, MODE_IX); auto* csr = CollectionShardingRuntime::get(opCtx, nss); - csr->clearFilteringMetadataForDroppedCollection(opCtx); + csr->clearFilteringMetadata(opCtx); } - try { - DropReply unused; - uassertStatusOK( - dropCollection(opCtx, - nss, - &unused, - DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops, - fromMigrate)); - } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { - // Note that even if the namespace was not found we have to execute the code below! - LOGV2_DEBUG(5280920, - 1, - "Namespace not found while trying to delete local collection", - "namespace"_attr = nss); - } + DropReply result; + uassertStatusOK(dropCollection( + opCtx, nss, &result, DropCollectionSystemCollectionMode::kDisallowSystemCollectionDrops)); - // Force the refresh of the catalog cache to purge outdated information. Note also that this - // code is indirectly used to notify to secondary nodes to clear their filtering information. + // Force the refresh of the catalog cache to purge outdated information const auto catalog = Grid::get(opCtx)->catalogCache(); uassertStatusOK(catalog->getCollectionRoutingInfoWithRefresh(opCtx, nss)); CatalogCacheLoader::get(opCtx).waitForCollectionFlush(opCtx, nss); repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); + + return result; } void DropCollectionCoordinator::_enterPhase(Phase newPhase) { @@ -227,23 +215,13 @@ ExecutorFuture<void> DropCollectionCoordinator::_runImpl( participants.end()); sharding_ddl_util::sendDropCollectionParticipantCommandToShards( - opCtx, - nss(), - participants, - **executor, - getCurrentSession(_doc), - false /*fromMigrate*/); + opCtx, nss(), participants, **executor, getCurrentSession(_doc)); // The sharded collection must be dropped on the primary shard after it has been // dropped on all of the other shards to ensure it can only be re-created as // unsharded with a higher optime than all of the drops. sharding_ddl_util::sendDropCollectionParticipantCommandToShards( - opCtx, - nss(), - {primaryShardId}, - **executor, - getCurrentSession(_doc), - false /*fromMigrate*/); + opCtx, nss(), {primaryShardId}, **executor, getCurrentSession(_doc)); ShardingLogging::get(opCtx)->logChange(opCtx, "dropCollection", nss().ns()); LOGV2(5390503, "Collection dropped", "namespace"_attr = nss()); diff --git a/src/mongo/db/s/drop_collection_coordinator.h b/src/mongo/db/s/drop_collection_coordinator.h index b77b8b55cc1..140013e41e1 100644 --- a/src/mongo/db/s/drop_collection_coordinator.h +++ b/src/mongo/db/s/drop_collection_coordinator.h @@ -52,12 +52,8 @@ public: /** * Locally drops a collection, cleans its CollectionShardingRuntime metadata and refreshes the * catalog cache. - * The oplog entry associated with the drop collection will be generated with the fromMigrate - * flag. */ - static void dropCollectionLocally(OperationContext* opCtx, - const NamespaceString& nss, - bool fromMigrate); + static DropReply dropCollectionLocally(OperationContext* opCtx, const NamespaceString& nss); private: ShardingDDLCoordinatorMetadata const& metadata() const override { diff --git a/src/mongo/db/s/drop_database_coordinator.cpp b/src/mongo/db/s/drop_database_coordinator.cpp index 6260a26a12f..3645f186d72 100644 --- a/src/mongo/db/s/drop_database_coordinator.cpp +++ b/src/mongo/db/s/drop_database_coordinator.cpp @@ -39,7 +39,6 @@ #include "mongo/db/s/sharding_logging.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/s/type_shard_database.h" -#include "mongo/db/vector_clock_mutable.h" #include "mongo/logv2/log.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/client/shard_registry.h" @@ -103,25 +102,6 @@ private: const BSONObj _reason; }; -bool isDbAlreadyDropped(OperationContext* opCtx, - const boost::optional<mongo::DatabaseVersion>& dbVersion, - const StringData& dbName) { - if (dbVersion) { - try { - auto const catalogClient = Grid::get(opCtx)->catalogClient(); - const auto db = catalogClient->getDatabase( - opCtx, dbName, repl::ReadConcernLevel::kMajorityReadConcern); - if (dbVersion->getUuid() != db.getVersion().getUuid()) { - // The database was dropped and re-created with a different UUID - return true; - } - } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { - // The database was already dropped - return true; - } - } - return false; -} } // namespace @@ -153,13 +133,13 @@ void DropDatabaseCoordinator::_dropShardedCollection( participants.erase(std::remove(participants.begin(), participants.end(), primaryShardId), participants.end()); sharding_ddl_util::sendDropCollectionParticipantCommandToShards( - opCtx, nss, participants, **executor, getCurrentSession(_doc), false /* fromMigrate */); + opCtx, nss, participants, **executor, getCurrentSession(_doc)); // The sharded collection must be dropped on the primary shard after it has been dropped on all // of the other shards to ensure it can only be re-created as unsharded with a higher optime // than all of the drops. sharding_ddl_util::sendDropCollectionParticipantCommandToShards( - opCtx, nss, {primaryShardId}, **executor, getCurrentSession(_doc), false /* fromMigrate */); + opCtx, nss, {primaryShardId}, **executor, getCurrentSession(_doc)); } DropDatabaseCoordinator::DropDatabaseCoordinator(ShardingDDLCoordinatorService* service, @@ -273,18 +253,16 @@ ExecutorFuture<void> DropDatabaseCoordinator::_runImpl( sharding_ddl_util::performNoopMajorityWriteLocally(opCtx); // ensure we do not delete collections of a different DB - if (!_firstExecution && - isDbAlreadyDropped(opCtx, _doc.getDatabaseVersion(), _dbName)) { - // Clear the database sharding state so that all subsequent write operations - // with the old database version will fail due to StaleDbVersion. - // Note: because we are using an scoped critical section it could happen that - // the dbversion being deleted is recovered once we return. It is a rare - // occurence, but it might lead to a situation where the now former primary - // will believe to still be primary. - _clearDatabaseInfoOnPrimary(opCtx); - _clearDatabaseInfoOnSecondaries(opCtx); - VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); - return; // skip to FlushDatabaseCacheUpdates + if (!_firstExecution && _doc.getDatabaseVersion()) { + try { + const auto db = catalogClient->getDatabase( + opCtx, _dbName, repl::ReadConcernLevel::kMajorityReadConcern); + if (_doc.getDatabaseVersion()->getUuid() != db.getVersion().getUuid()) { + return; // skip to FlushDatabaseCacheUpdates + } + } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { + return; // skip to FlushDatabaseCacheUpdates + } } if (_doc.getCollInfo()) { @@ -369,8 +347,6 @@ ExecutorFuture<void> DropDatabaseCoordinator::_runImpl( removeDatabaseMetadataFromConfig( opCtx, _dbName, *metadata().getDatabaseVersion()); - - VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); } })) .then([this, executor = executor, anchor = shared_from_this()] { diff --git a/src/mongo/db/s/flush_database_cache_updates_command.cpp b/src/mongo/db/s/flush_database_cache_updates_command.cpp index 729572272f6..de95a293333 100644 --- a/src/mongo/db/s/flush_database_cache_updates_command.cpp +++ b/src/mongo/db/s/flush_database_cache_updates_command.cpp @@ -38,7 +38,6 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/commands.h" -#include "mongo/db/dbdirectclient.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/database_sharding_state.h" @@ -53,29 +52,6 @@ namespace mongo { namespace { -/** - * Inserts a database collection entry with fixed metadata for the `config` or `admin` database. If - * the entry key already exists, it's not updated. - */ -Status insertDatabaseEntryForBackwardCompatibility(OperationContext* opCtx, - const StringData& dbName) { - invariant(dbName == NamespaceString::kAdminDb || dbName == NamespaceString::kConfigDb); - - DBDirectClient client(opCtx); - auto commandResponse = client.runCommand([&] { - auto dbMetadata = - DatabaseType(dbName.toString(), ShardId::kConfigServerId, DatabaseVersion::makeFixed()); - dbMetadata.setSharded(true); - - write_ops::InsertCommandRequest insertOp(NamespaceString::kShardConfigDatabasesNamespace); - insertOp.setDocuments({dbMetadata.toBSON()}); - return insertOp.serialize({}); - }()); - - auto commandStatus = getStatusFromWriteCommandReply(commandResponse->getCommandReply()); - return commandStatus.code() == ErrorCodes::DuplicateKey ? Status::OK() : commandStatus; -} - template <typename Derived> class FlushDatabaseCacheUpdatesCmdBase : public TypedCommand<Derived> { public: @@ -141,26 +117,6 @@ public: "Can't call _flushDatabaseCacheUpdates if in read-only mode", !storageGlobalParams.readOnly); - if (_dbName() == NamespaceString::kAdminDb || _dbName() == NamespaceString::kConfigDb) { - // The admin and config databases have fixed metadata that does not need to be - // refreshed. - - if (Base::request().getSyncFromConfig()) { - // To ensure compatibility with old secondaries that still call the - // _flushDatabaseCacheUpdates command to get updated database metadata from - // primary, an entry with fixed metadata is inserted in the - // config.cache.databases collection. - - LOGV2_DEBUG(6910800, - 1, - "Inserting a database collection entry with fixed metadata", - "db"_attr = _dbName()); - uassertStatusOK(insertDatabaseEntryForBackwardCompatibility(opCtx, _dbName())); - } - - return; - } - boost::optional<SharedSemiFuture<void>> criticalSectionSignal; { @@ -173,8 +129,8 @@ public: // consistency guarantee. const auto dss = DatabaseShardingState::get(opCtx, _dbName()); auto dssLock = DatabaseShardingState::DSSLock::lockShared(opCtx, dss); - criticalSectionSignal = dss->getCriticalSectionSignal( - ShardingMigrationCriticalSection::kWrite, dssLock); + criticalSectionSignal = + dss->getCriticalSectionSignal(ShardingMigrationCriticalSection::kRead, dssLock); } if (criticalSectionSignal) diff --git a/src/mongo/db/s/flush_resharding_state_change_command.cpp b/src/mongo/db/s/flush_resharding_state_change_command.cpp index 53b3f9953ff..c1049660307 100644 --- a/src/mongo/db/s/flush_resharding_state_change_command.cpp +++ b/src/mongo/db/s/flush_resharding_state_change_command.cpp @@ -38,6 +38,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" #include "mongo/db/commands.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/op_observer.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/repl_client_info.h" @@ -109,11 +110,7 @@ public: "Can't call _flushReshardingStateChange if in read-only mode", !storageGlobalParams.readOnly); - // We use the fixed executor here since it may cause the thread to block. This would - // cause potential liveness issues since the arbitrary executor is a NetworkInterfaceTL - // executor in sharded clusters and that executor is one that executes networking - // operations. - ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getFixedExecutor()) + ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getArbitraryExecutor()) .then([svcCtx = opCtx->getServiceContext(), nss = ns()] { ThreadClient tc("FlushReshardingStateChange", svcCtx); { diff --git a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp index 6562c664519..ac2e22b78c8 100644 --- a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp +++ b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp @@ -112,11 +112,6 @@ public: << " if in read-only mode", !storageGlobalParams.readOnly); - uassert(ErrorCodes::IllegalOperation, - str::stream() << "Can only call " << Derived::Request::kCommandName - << " on collections", - !ns().coll().empty()); - boost::optional<SharedSemiFuture<void>> criticalSectionSignal; { diff --git a/src/mongo/db/s/get_stats_for_balancing.idl b/src/mongo/db/s/get_stats_for_balancing.idl new file mode 100644 index 00000000000..7ee96da4eaa --- /dev/null +++ b/src/mongo/db/s/get_stats_for_balancing.idl @@ -0,0 +1,84 @@ +# Copyright (C) 2022-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + +structs: + NamespaceWithOptionalUUID: + description: 'Namespace with an optional collection UUID' + strict: false + fields: + ns: + description: 'Namespace of the collection' + type: namespacestring + UUID: + description: 'Collection UUID' + type: uuid + optional: true # optional because the caller may not attach the collection UUID + + CollStatsForBalancing: + description: 'Collection stats for a specific collection' + strict: false + fields: + namespace: + description: 'Namespace of the collection' + type: namespacestring + cpp_name: ns + collSize: + description: 'size of data currently owned by this shard for this collection' + type: safeInt64 + + ShardsvrGetStatsForBalancingReply: + description: 'Response for ShardsvrGetStatsForBalancing command' + strict: false + fields: + stats: + description: 'List of stats for each of the requested collection' + type: array<CollStatsForBalancing> + +commands: + _shardsvrGetStatsForBalancing: + command_name: _shardsvrGetStatsForBalancing + cpp_name: ShardsvrGetStatsForBalancing + description: 'Internal command used by the balancer to retrieve stats for balancing.' + namespace: ignored + api_version: '' + strict: false + reply_type: ShardsvrGetStatsForBalancingReply + fields: + collections: + description: 'List of namespaces to retrieve statistic for' + type: array<NamespaceWithOptionalUUID> + scaleFactor: + description: 'Scale factor for data size units. If omitted 1048576 (MiB) will be used' + type: exactInt64 + optional: true diff --git a/src/mongo/db/s/implicit_collection_creation_test.cpp b/src/mongo/db/s/implicit_collection_creation_test.cpp index 478b2aaa815..02dbe9ed456 100644 --- a/src/mongo/db/s/implicit_collection_creation_test.cpp +++ b/src/mongo/db/s/implicit_collection_creation_test.cpp @@ -32,7 +32,6 @@ #include "mongo/platform/basic.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/operation_sharding_state.h" #include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/unittest/unittest.h" @@ -63,25 +62,6 @@ TEST_F(ImplicitCollectionCreationTest, AllowImplicitCollectionCreate) { WriteUnitOfWork wuow(operationContext()); ASSERT_OK(db->userCreateNS(operationContext(), nss, CollectionOptions{})); wuow.commit(); - - auto* const csr = CollectionShardingRuntime::get(operationContext(), nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(operationContext(), csr); - ASSERT_TRUE(csr->getCurrentMetadataIfKnown()); -} - -TEST_F(ImplicitCollectionCreationTest, AllowImplicitCollectionCreateWithSetCSRAsUnknown) { - NamespaceString nss("AllowImplicitCollectionCreateDB.TestColl"); - OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE unsafeCreateCollection( - operationContext(), /* forceCSRAsUnknownAfterCollectionCreation */ true); - AutoGetCollection autoColl(operationContext(), nss, MODE_IX); - auto db = autoColl.ensureDbExists(operationContext()); - WriteUnitOfWork wuow(operationContext()); - ASSERT_OK(db->userCreateNS(operationContext(), nss, CollectionOptions{})); - wuow.commit(); - - auto* const csr = CollectionShardingRuntime::get(operationContext(), nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(operationContext(), csr); - ASSERT_FALSE(csr->getCurrentMetadataIfKnown()); } } // namespace diff --git a/src/mongo/db/s/metadata_manager.cpp b/src/mongo/db/s/metadata_manager.cpp index bcf333cbf6b..6f57d95f93a 100644 --- a/src/mongo/db/s/metadata_manager.cpp +++ b/src/mongo/db/s/metadata_manager.cpp @@ -36,7 +36,6 @@ #include "mongo/base/string_data.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/bson/util/builder.h" -#include "mongo/db/cursor_manager.h" #include "mongo/db/query/internal_plans.h" #include "mongo/db/range_arithmetic.h" #include "mongo/db/s/migration_util.h" @@ -277,15 +276,14 @@ SharedSemiFuture<void> MetadataManager::cleanUpRange(ChunkRange const& range, shouldDelayBeforeDeletion ? Seconds(orphanCleanupDelaySecs.load()) : Seconds(0); if (overlapMetadata) { - const auto openCursorsIds = - CursorManager::get(_serviceContext)->getCursorIdsForNamespace(_nss); - LOGV2_INFO_OPTIONS( - 7179200, - {logv2::LogComponent::kShardingRangeDeleter}, - "Range deletion will be scheduled after all possibly dependent queries finish", - "namespace"_attr = _nss, - "range"_attr = range.toString(), - "cursorsDirectlyReferringTheNamespace"_attr = openCursorsIds); + LOGV2_OPTIONS(21989, + {logv2::LogComponent::kShardingMigration}, + "Deletion of {namespace} range {range} will be scheduled after all possibly " + "dependent queries finish", + "Deletion of the collection's specified range will be scheduled after all " + "possibly dependent queries finish", + "namespace"_attr = _nss.ns(), + "range"_attr = redact(range.toString())); ++overlapMetadata->numContingentRangeDeletionTasks; // Schedule the range for deletion once the overlapping metadata object is destroyed // (meaning no more queries can be using the range) and obtain a future which will be @@ -375,6 +373,12 @@ SharedSemiFuture<void> MetadataManager::_submitRangeForDeletion( const ChunkRange& range, const UUID& migrationId, Seconds delayForActiveQueriesOnSecondariesToComplete) { + + int maxToDelete = rangeDeleterBatchSize.load(); + if (maxToDelete <= 0) { + maxToDelete = kRangeDeleterBatchSizeDefault; + } + auto cleanupComplete = removeDocumentsInRange(_executor, std::move(waitForActiveQueriesToComplete), @@ -383,6 +387,7 @@ SharedSemiFuture<void> MetadataManager::_submitRangeForDeletion( _metadata.back()->metadata->getKeyPattern().getOwned(), range, migrationId, + maxToDelete, delayForActiveQueriesOnSecondariesToComplete); _rangesScheduledForDeletion.emplace_front(range, cleanupComplete); @@ -390,25 +395,11 @@ SharedSemiFuture<void> MetadataManager::_submitRangeForDeletion( // from the _rangesScheduledForDeletion. std::list iterators are never invalidated, which // allows us to save the iterator pointing to the newly added element for use later when // deleting it. - cleanupComplete.thenRunOn(_executor).getAsync([self = shared_from_this(), - it = _rangesScheduledForDeletion.begin(), - range = range, - migrationId = migrationId](Status s) { - stdx::lock_guard<Latch> lg(self->_managerLock); - self->_rangesScheduledForDeletion.erase(it); - if (s.code() == ErrorCodes::IndexNotFound) { - // We cannot complete this range deletion right now because we do not have an index - // built on the shard key. This situation is expected for a hashed shard key and - // recoverable for a range shard key. This index may be rebuilt in the future, so - // reschedule the task at the end of the queue. - auto _ = self->_submitRangeForDeletion(lg, - SemiFuture<void>::makeReady(), - range, - migrationId, - Seconds(orphanCleanupDelaySecs.load())); - } - }); - + cleanupComplete.thenRunOn(_executor).getAsync( + [self = shared_from_this(), it = _rangesScheduledForDeletion.begin()](Status s) { + stdx::lock_guard<Latch> lg(self->_managerLock); + self->_rangesScheduledForDeletion.erase(it); + }); return cleanupComplete; } diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp index b1936e3e47f..2f39ef09147 100644 --- a/src/mongo/db/s/metadata_manager_test.cpp +++ b/src/mongo/db/s/metadata_manager_test.cpp @@ -72,12 +72,6 @@ protected: orphanCleanupDelaySecs.store(1); } - void tearDown() override { - // Restore original `orphanCleanupDelaySecs` value for next unit tests - orphanCleanupDelaySecs.store(_defaultOrphanCleanupDelaySecs); - ShardServerTestFixture::tearDown(); - } - /** * Returns an instance of CollectionMetadata which has no chunks owned by 'thisShard'. */ @@ -190,9 +184,6 @@ protected: } std::shared_ptr<MetadataManager> _manager; - -private: - const int _defaultOrphanCleanupDelaySecs = orphanCleanupDelaySecs.load(); }; // The 'pending' field must not be set in order for a range deletion task to succeed, but the diff --git a/src/mongo/db/s/migration_batch_fetcher.cpp b/src/mongo/db/s/migration_batch_fetcher.cpp deleted file mode 100644 index 5dda53ead14..00000000000 --- a/src/mongo/db/s/migration_batch_fetcher.cpp +++ /dev/null @@ -1,276 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - -#include "mongo/db/s/migration_batch_fetcher.h" - -#include "mongo/logv2/log.h" -#include "mongo/util/timer.h" - -namespace mongo { - -template <typename Inserter> -void MigrationBatchFetcher<Inserter>::BufferSizeTracker::waitUntilSpaceAvailableAndAdd( - OperationContext* opCtx, int sizeBytes) { - if (_maxSizeBytes == MigrationBatchFetcher<Inserter>::BufferSizeTracker::kUnlimited) { - return; - } - - uassert(8120100, - str::stream() << "chunkMigrationFetcherMaxBufferedSizeBytesPerThread setting of " - << _maxSizeBytes << " is too small for received batch size of " - << sizeBytes, - sizeBytes <= _maxSizeBytes); - - stdx::unique_lock lk(_mutex); - opCtx->waitForConditionOrInterrupt(_hasAvailableSpace, lk, [this, sizeBytes] { - return (_currentSize + sizeBytes) <= _maxSizeBytes; - }); - _currentSize += sizeBytes; -} - -template <typename Inserter> -void MigrationBatchFetcher<Inserter>::BufferSizeTracker::remove(int sizeBytes) { - if (_maxSizeBytes == MigrationBatchFetcher<Inserter>::BufferSizeTracker::kUnlimited) { - return; - } - - stdx::unique_lock lk(_mutex); - invariant(_currentSize >= sizeBytes); - - _currentSize -= sizeBytes; - _hasAvailableSpace.notify_one(); -} - -template <typename Inserter> -MigrationBatchFetcher<Inserter>::MigrationBatchFetcher( - OperationContext* outerOpCtx, - OperationContext* innerOpCtx, - NamespaceString nss, - MigrationSessionId sessionId, - const WriteConcernOptions& writeConcern, - const ShardId& fromShardId, - const ChunkRange& range, - const UUID& migrationId, - const UUID& collectionId, - std::shared_ptr<MigrationCloningProgressSharedState> migrationProgress, - bool parallelFetchingSupported, - int maxBufferedSizeBytesPerThread) - : _nss{std::move(nss)}, - _chunkMigrationConcurrency{ - mongo::feature_flags::gConcurrencyInChunkMigration.isEnabledAndIgnoreFCV() - ? chunkMigrationConcurrency.load() - : 1}, - _sessionId{std::move(sessionId)}, - _inserterWorkers{[&]() { - ThreadPool::Options options; - options.poolName = "ChunkMigrationInserters"; - options.minThreads = _chunkMigrationConcurrency; - options.maxThreads = _chunkMigrationConcurrency; - options.onCreateThread = Inserter::onCreateThread; - return std::make_unique<ThreadPool>(options); - }()}, - _migrateCloneRequest{_createMigrateCloneRequest()}, - _outerOpCtx{outerOpCtx}, - _innerOpCtx{innerOpCtx}, - _fromShard{uassertStatusOK( - Grid::get(_outerOpCtx)->shardRegistry()->getShard(_outerOpCtx, fromShardId))}, - _migrationProgress{migrationProgress}, - _range{range}, - _collectionUuid(collectionId), - _migrationId{migrationId}, - _writeConcern{writeConcern}, - _isParallelFetchingSupported{parallelFetchingSupported}, - _secondaryThrottleTicket(1, outerOpCtx->getServiceContext()), - _bufferSizeTracker(maxBufferedSizeBytesPerThread) { - _inserterWorkers->startup(); -} - -template <typename Inserter> -BSONObj MigrationBatchFetcher<Inserter>::_fetchBatch(OperationContext* opCtx) { - auto commandResponse = uassertStatusOKWithContext( - _fromShard->runCommand(opCtx, - ReadPreferenceSetting(ReadPreference::PrimaryOnly), - "admin", - _migrateCloneRequest, - Shard::RetryPolicy::kNoRetry), - "_migrateClone failed: "); - - uassertStatusOKWithContext(Shard::CommandResponse::getEffectiveStatus(commandResponse), - "_migrateClone failed: "); - - return commandResponse.response; -} - -template <typename Inserter> -void MigrationBatchFetcher<Inserter>::fetchAndScheduleInsertion() { - auto numFetchers = _isParallelFetchingSupported ? _chunkMigrationConcurrency : 1; - auto fetchersThreadPool = [&]() { - ThreadPool::Options options; - options.poolName = "ChunkMigrationFetchers"; - options.minThreads = numFetchers; - options.maxThreads = numFetchers; - options.onCreateThread = onCreateThread; - return std::make_unique<ThreadPool>(options); - }(); - fetchersThreadPool->startup(); - for (int i = 0; i < numFetchers; ++i) { - fetchersThreadPool->schedule([this](Status status) { this->_runFetcher(); }); - } - - fetchersThreadPool->shutdown(); - fetchersThreadPool->join(); -} - - -template <typename Inserter> -void MigrationBatchFetcher<Inserter>::_runFetcher() try { - auto executor = - Grid::get(_innerOpCtx->getServiceContext())->getExecutorPool()->getFixedExecutor(); - - auto applicationOpCtx = CancelableOperationContext( - cc().makeOperationContext(), _innerOpCtx->getCancellationToken(), executor); - - auto opCtx = applicationOpCtx.get(); - auto assertNotAborted = [&]() { - { - stdx::lock_guard<Client> lk(*_outerOpCtx->getClient()); - _outerOpCtx->checkForInterrupt(); - } - opCtx->checkForInterrupt(); - }; - - LOGV2_DEBUG(6718405, 0, "Chunk migration data fetch start", "migrationId"_attr = _migrationId); - while (true) { - Timer totalTimer; - BSONObj nextBatch = _fetchBatch(opCtx); - assertNotAborted(); - if (_isEmptyBatch(nextBatch)) { - LOGV2_DEBUG(6718404, - 0, - "Chunk migration initial clone complete", - "migrationId"_attr = _migrationId, - "duration"_attr = totalTimer.elapsed()); - break; - } - - const auto batchSize = nextBatch.objsize(); - const auto fetchTime = totalTimer.elapsed(); - LOGV2_DEBUG(6718416, - 0, - "Chunk migration initial clone fetch end", - "migrationId"_attr = _migrationId, - "batchSize"_attr = batchSize, - "fetch"_attr = duration_cast<Milliseconds>(fetchTime)); - - _bufferSizeTracker.waitUntilSpaceAvailableAndAdd(opCtx, batchSize); - - Inserter inserter{_outerOpCtx, - _innerOpCtx, - nextBatch.getOwned(), - _nss, - _range, - _writeConcern, - _collectionUuid, - _migrationProgress, - _migrationId, - _chunkMigrationConcurrency, - &_secondaryThrottleTicket}; - - _inserterWorkers->schedule([this, - batchSize, - fetchTime, - totalTimer = std::move(totalTimer), - insertTimer = Timer(), - migrationId = _migrationId, - inserter = std::move(inserter)](Status status) { - ON_BLOCK_EXIT([&] { _bufferSizeTracker.remove(batchSize); }); - inserter.run(status); - - const auto checkDivByZero = [](auto divisor, auto expression) { - return divisor == 0 ? -1 : expression(); - }; - const auto calcThroughput = [&](auto bytes, auto duration) { - return checkDivByZero(durationCount<Microseconds>(duration), [&]() { - return static_cast<double>(bytes) / durationCount<Microseconds>(duration); - }); - }; - - const auto insertTime = insertTimer.elapsed(); - const auto totalTime = totalTimer.elapsed(); - const auto batchThroughputMBps = calcThroughput(batchSize, totalTime); - const auto insertThroughputMBps = calcThroughput(batchSize, insertTime); - const auto fetchThroughputMBps = calcThroughput(batchSize, fetchTime); - - LOGV2_DEBUG(6718417, - 1, - "Chunk migration initial clone apply batch", - "migrationId"_attr = migrationId, - "batchSize"_attr = batchSize, - "total"_attr = duration_cast<Milliseconds>(totalTime), - "totalThroughputMBps"_attr = batchThroughputMBps, - "fetch"_attr = duration_cast<Milliseconds>(fetchTime), - "fetchThroughputMBps"_attr = fetchThroughputMBps, - "insert"_attr = duration_cast<Milliseconds>(insertTime), - "insertThroughputMBps"_attr = insertThroughputMBps); - }); - } -} catch (const DBException& e) { - stdx::lock_guard<Client> lk(*_innerOpCtx->getClient()); - _innerOpCtx->getServiceContext()->killOperation(lk, _innerOpCtx, ErrorCodes::Error(6718400)); - LOGV2_ERROR(6718413, - "Chunk migration failure fetching data", - "migrationId"_attr = _migrationId, - "failure"_attr = e.toStatus()); -} - -template <typename Inserter> -MigrationBatchFetcher<Inserter>::~MigrationBatchFetcher() { - LOGV2(6718401, - "Shutting down and joining inserter threads for migration {migrationId}", - "migrationId"_attr = _migrationId); - - // Call waitForIdle first since join can spawn another thread while ignoring the maxPoolSize - // to finish the pending task. This is safe as long as ThreadPool::shutdown can't be - // interleaved with this call. - _inserterWorkers->waitForIdle(); - _inserterWorkers->shutdown(); - _inserterWorkers->join(); - - LOGV2(6718415, - "Inserter threads for migration {migrationId} joined", - "migrationId"_attr = _migrationId); -} - -template class MigrationBatchFetcher<MigrationBatchInserter>; - -template class MigrationBatchFetcher<MigrationBatchMockInserter>; - -} // namespace mongo diff --git a/src/mongo/db/s/migration_batch_fetcher.h b/src/mongo/db/s/migration_batch_fetcher.h deleted file mode 100644 index 690c6b3cf00..00000000000 --- a/src/mongo/db/s/migration_batch_fetcher.h +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/base/error_extra_info.h" -#include "mongo/db/client.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/repl/optime.h" -#include "mongo/db/s/migration_batch_inserter.h" -#include "mongo/db/s/migration_batch_mock_inserter.h" -#include "mongo/db/s/migration_session_id.h" -#include "mongo/s/client/shard.h" -#include "mongo/s/grid.h" -#include "mongo/s/shard_id.h" -#include "mongo/util/cancellation.h" -#include "mongo/util/concurrency/thread_pool.h" -#include "mongo/util/concurrency/ticketholder.h" -#include "mongo/util/producer_consumer_queue.h" - -#pragma once - -namespace mongo { - - -// This class is only instantiated on the destination of a chunk migration and -// has a single purpose: to manage two thread pools, one -// on which threads perform inserters, and one on which -// threads run _migrateClone requests (to fetch batches of documents to insert). -// -// The constructor creates and starts the inserter thread pool. The destructor shuts down -// and joins the inserter thread pool. -// -// The main work of the class is in method fetchAndScheduleInsertion. That method -// starts a thread pool for fetchers. Each thread in that thread pool sits in a loop -// sending out _migrateClone requests, blocking on the response, and scheduling an -// inserter on the inserter thread pool. This function joins and shuts down the -// fetcher thread pool once all batches have been fetched. -// -// Inserter is templated only to allow a mock inserter to exist. -// There is only one implementation of inserter currently, which is MigrationBatchInserter. -// -// A few things to note: -// - After fetchAndScheduleInsertion returns, insertions are still being executed (although fetches -// are not). -// - Sending out _migrateClone requests in parallel implies the need for synchronization on the -// source. See the comments in migration_chunk_cloner_source.h for details around -// that. -// - The requirement on source side synchronization implies that care must be taken on upgrade. -// In particular, if the source is running an earlier binary that doesn't have code for -// source side synchronization, it is unsafe to send _migrateClone requests in parallel. -// To handle that case, when the source is prepared to service _migrateClone requests in -// parallel, the field "parallelMigrateCloneSupported" is included in the "_recvChunkStart" -// command. The inclusion of that field indicates to the destination that it is safe -// to send _migrateClone requests in parallel. Its exclusion indicates that it is unsafe. -template <typename Inserter> -class MigrationBatchFetcher { -public: - MigrationBatchFetcher(OperationContext* outerOpCtx, - OperationContext* innerOpCtx, - NamespaceString nss, - MigrationSessionId sessionId, - const WriteConcernOptions& writeConcern, - const ShardId& fromShardId, - const ChunkRange& range, - const UUID& migrationId, - const UUID& collectionId, - std::shared_ptr<MigrationCloningProgressSharedState> migrationInfo, - bool parallelFetchingSupported, - int maxBufferedSizeBytesPerThread); - - ~MigrationBatchFetcher(); - - // Repeatedly fetch batches (using _migrateClone request) and schedule inserter jobs - // on thread pool. - void fetchAndScheduleInsertion(); - - // Get inserter thread pool stats. - ThreadPool::Stats getThreadPoolStats() const { - return _inserterWorkers->getStats(); - } - -private: - /** - * Keeps track of memory usage and makes sure it won't exceed the limit. - */ - class BufferSizeTracker { - public: - const static int kUnlimited{0}; - - BufferSizeTracker(int maxSizeBytes) : _maxSizeBytes(maxSizeBytes) {} - - /** - * If adding the given amount of bytes will go over the limit, wait until there's - * enough space then add. - */ - void waitUntilSpaceAvailableAndAdd(OperationContext* opCtx, int sizeBytes); - - /** - * Subtracts the tracked bytes by the given amount. - */ - void remove(int sizeBytes); - - private: - Mutex _mutex = MONGO_MAKE_LATCH("MigrationBatchFetcher::BufferSizeTracker::_mutex"); - stdx::condition_variable _hasAvailableSpace; - - const int _maxSizeBytes; - int _currentSize{0}; - }; - - NamespaceString _nss; - - // Size of thread pools. - int _chunkMigrationConcurrency; - - MigrationSessionId _sessionId; - - // Inserter thread pool. - std::unique_ptr<ThreadPool> _inserterWorkers; - - BSONObj _migrateCloneRequest; - - OperationContext* _outerOpCtx; - - OperationContext* _innerOpCtx; - - std::shared_ptr<Shard> _fromShard; - - // Shared state, by which the progress of migration is communicated - // to MigrationDestinationManager. - std::shared_ptr<MigrationCloningProgressSharedState> _migrationProgress; - - ChunkRange _range; - - UUID _collectionUuid; - - UUID _migrationId; - - WriteConcernOptions _writeConcern; - - // Indicates if source is prepared to service _migrateClone requests in parallel. - bool _isParallelFetchingSupported; - - SemaphoreTicketHolder _secondaryThrottleTicket; - - BufferSizeTracker _bufferSizeTracker; - - // Given session id and namespace, create migrateCloneRequest. - // Only should be created once for the lifetime of the object. - BSONObj _createMigrateCloneRequest() const { - BSONObjBuilder builder; - builder.append("_migrateClone", _nss.ns()); - _sessionId.append(&builder); - return builder.obj(); - } - - void _runFetcher(); - - // Fetches next batch using _migrateClone request and return it. May return an empty batch. - BSONObj _fetchBatch(OperationContext* opCtx); - - static bool _isEmptyBatch(const BSONObj& batch) { - return batch.getField("objects").Obj().isEmpty(); - } - - static void onCreateThread(const std::string& threadName) { - Client::initThread(threadName, getGlobalServiceContext(), nullptr); - { - stdx::lock_guard<Client> lk(cc()); - cc().setSystemOperationKillableByStepdown(lk); - } - } - -}; // namespace mongo - -} // namespace mongo diff --git a/src/mongo/db/s/migration_batch_fetcher_test.cpp b/src/mongo/db/s/migration_batch_fetcher_test.cpp deleted file mode 100644 index 21643a2b1d4..00000000000 --- a/src/mongo/db/s/migration_batch_fetcher_test.cpp +++ /dev/null @@ -1,274 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - -#include "mongo/base/status_with.h" -#include "mongo/bson/bsonobj.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/s/migration_batch_fetcher.h" -#include "mongo/db/s/migration_session_id.h" -#include "mongo/db/s/shard_server_test_fixture.h" -#include "mongo/db/write_concern_options.h" -#include "mongo/dbtests/mock/mock_replica_set.h" -#include "mongo/executor/cancelable_executor.h" -#include "mongo/executor/network_interface_mock.h" -#include "mongo/executor/thread_pool_mock.h" -#include "mongo/executor/thread_pool_task_executor.h" -#include "mongo/executor/thread_pool_task_executor_test_fixture.h" -#include "mongo/idl/server_parameter_test_util.h" -#include "mongo/logv2/log.h" -#include "mongo/platform/basic.h" -#include "mongo/s/catalog/sharding_catalog_client_mock.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/stdx/future.h" -#include "mongo/stdx/thread.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/duration.h" -#include "mongo/util/net/hostandport.h" -#include "mongo/util/uuid.h" - -namespace mongo { -namespace { - -using unittest::assertGet; -using namespace std::chrono_literals; - -const ConnectionString kDonorConnStr = - ConnectionString::forReplicaSet("Donor", - {HostAndPort("DonorHost1:1234"), - HostAndPort{"DonorHost2:1234"}, - HostAndPort{"DonorHost3:1234"}}); -const ConnectionString kRecipientConnStr = - ConnectionString::forReplicaSet("Recipient", - {HostAndPort("RecipientHost1:1234"), - HostAndPort("RecipientHost2:1234"), - HostAndPort("RecipientHost3:1234")}); - -class MigrationBatchFetcherTestFixture : public ShardServerTestFixture { - -protected: - /** - * Sets up the task executor as well as a TopologyListenerMock for each unit test. - */ - void setUp() override { - ShardServerTestFixture::setUp(); - - { - auto donorShard = assertGet( - shardRegistry()->getShard(operationContext(), kDonorConnStr.getSetName())); - RemoteCommandTargeterMock::get(donorShard->getTargeter()) - ->setConnectionStringReturnValue(kDonorConnStr); - RemoteCommandTargeterMock::get(donorShard->getTargeter()) - ->setFindHostReturnValue(kDonorConnStr.getServers()[0]); - } - } - - void tearDown() override { - ShardServerTestFixture::tearDown(); - } - - /** - * Instantiates a BSON object in which both "_id" and "X" are set to value. - */ - static BSONObj createDocument(int value) { - return BSON("_id" << value << "X" << value); - } - static BSONObj createEmpty() { - return BSONObj{}; - } - /** - * Creates a list of documents to clone. - */ - static std::vector<BSONObj> createDocumentsToClone() { - return {createDocument(1), createDocument(2), createDocument(3)}; - } - - /** - * Creates a list of documents to clone and converts it to a BSONArray. - */ - static BSONArray createDocumentsToCloneArray() { - BSONArrayBuilder arrayBuilder; - for (auto& doc : createDocumentsToClone()) { - arrayBuilder.append(doc); - } - return arrayBuilder.arr(); - } - static BSONArray createEmptyCloneArray() { - return BSONArrayBuilder().arr(); - } - - static BSONObj getTerminalBsonObj() { - return BSON("Status" - << "OK" - << "ok" << 1 << "objects" << createEmptyCloneArray()); - } - - static BSONObj getBatchBsonObj() { - return BSON("Status" - << "OK" - << "ok" << 1 << "objects" << createDocumentsToCloneArray()); - } - -private: - OperationContext* _opCtx; - ServiceContext* _svcCtx; - executor::NetworkInterfaceMock* _net; - - std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient() override { - class StaticCatalogClient final : public ShardingCatalogClientMock { - public: - StaticCatalogClient() = default; - - StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { - - ShardType donorShard; - donorShard.setName(kDonorConnStr.getSetName()); - donorShard.setHost(kDonorConnStr.toString()); - - ShardType recipientShard; - recipientShard.setName(kRecipientConnStr.getSetName()); - recipientShard.setHost(kRecipientConnStr.toString()); - - return repl::OpTimeWith<std::vector<ShardType>>({donorShard, recipientShard}); - } - }; - - return std::make_unique<StaticCatalogClient>(); - } -}; - -auto getOnMigrateCloneCommandCb(BSONObj ret) { - return [ret](const executor::RemoteCommandRequest& request) -> StatusWith<BSONObj> { - ASSERT_EQ(request.cmdObj.getField("_migrateClone").String(), "test.foo"); - return ret; - }; -} - -TEST_F(MigrationBatchFetcherTestFixture, BasicEmptyFetchingTest) { - NamespaceString nss{"test", "foo"}; - ShardId fromShard{"Donor"}; - auto msid = MigrationSessionId::generate(fromShard, "Recipient"); - auto outerOpCtx = operationContext(); - auto newClient = outerOpCtx->getServiceContext()->makeClient("MigrationCoordinator"); - - int concurrency = 30; - RAIIServerParameterControllerForTest featureFlagController( - "featureFlagConcurrencyInChunkMigration", true); - RAIIServerParameterControllerForTest setMigrationConcurrencyParam{"chunkMigrationConcurrency", - concurrency}; - - AlternativeClientRegion acr(newClient); - auto executor = - Grid::get(outerOpCtx->getServiceContext())->getExecutorPool()->getFixedExecutor(); - auto newOpCtxPtr = CancelableOperationContext( - cc().makeOperationContext(), outerOpCtx->getCancellationToken(), executor); - auto opCtx = newOpCtxPtr.get(); - - auto fetcher = std::make_unique<MigrationBatchFetcher<MigrationBatchMockInserter>>( - outerOpCtx, - opCtx, - nss, - msid, - WriteConcernOptions::parse(WriteConcernOptions::Majority).getValue(), - fromShard, - ChunkRange{BSON("x" << 1), BSON("x" << 2)}, - UUID::gen(), - UUID::gen(), - nullptr, - true, - 0 /* maxBytesPerThread */); - - // Start asynchronous task for responding to _migrateClone requests. - // Must name the return of value std::async. The destructor of std::future joins the - // asynchrounous task. (If it were left unnamed, the destructor would run inline, and the test - // would hang forever.) - auto fut = stdx::async(stdx::launch::async, [&]() { - // One terminal response for each thread - for (int i = 0; i < concurrency; ++i) { - onCommand(getOnMigrateCloneCommandCb(getTerminalBsonObj())); - } - }); - fetcher->fetchAndScheduleInsertion(); -} - -TEST_F(MigrationBatchFetcherTestFixture, BasicFetching) { - NamespaceString nss{"test", "foo"}; - ShardId fromShard{"Donor"}; - auto msid = MigrationSessionId::generate(fromShard, "Recipient"); - - auto outerOpCtx = operationContext(); - auto newClient = outerOpCtx->getServiceContext()->makeClient("MigrationCoordinator"); - AlternativeClientRegion acr(newClient); - - auto executor = - Grid::get(outerOpCtx->getServiceContext())->getExecutorPool()->getFixedExecutor(); - auto newOpCtxPtr = CancelableOperationContext( - cc().makeOperationContext(), outerOpCtx->getCancellationToken(), executor); - auto opCtx = newOpCtxPtr.get(); - - - int concurrency = 30; - RAIIServerParameterControllerForTest featureFlagController( - "featureFlagConcurrencyInChunkMigration", true); - RAIIServerParameterControllerForTest setMigrationConcurrencyParam{"chunkMigrationConcurrency", - concurrency}; - - auto fetcher = std::make_unique<MigrationBatchFetcher<MigrationBatchMockInserter>>( - outerOpCtx, - opCtx, - nss, - msid, - WriteConcernOptions::parse(WriteConcernOptions::Majority).getValue(), - fromShard, - ChunkRange{BSON("x" << 1), BSON("x" << 2)}, - UUID::gen(), - UUID::gen(), - nullptr, - true, - 0 /* maxBytesPerThread */); - - auto fut = stdx::async(stdx::launch::async, [&]() { - for (int i = 0; i < 8; ++i) { - onCommand(getOnMigrateCloneCommandCb(getBatchBsonObj())); - } - // One terminal response for each thread - for (int i = 0; i < concurrency; ++i) { - onCommand(getOnMigrateCloneCommandCb(getTerminalBsonObj())); - } - }); - fetcher->fetchAndScheduleInsertion(); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/s/migration_batch_inserter.cpp b/src/mongo/db/s/migration_batch_inserter.cpp deleted file mode 100644 index ac0a59126a0..00000000000 --- a/src/mongo/db/s/migration_batch_inserter.cpp +++ /dev/null @@ -1,211 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kShardingMigration - -#include "mongo/db/s/migration_batch_inserter.h" - -#include "mongo/db/s/migration_util.h" -#include "mongo/db/transaction_participant.h" -#include "mongo/logv2/log.h" -#include "mongo/util/concurrency/ticketholder.h" - -namespace mongo { - -namespace { - -void checkOutSessionAndVerifyTxnState(OperationContext* opCtx) { - MongoDOperationContextSession::checkOut(opCtx); - TransactionParticipant::get(opCtx).beginOrContinue(opCtx, - {*opCtx->getTxnNumber()}, - boost::none /* autocommit */, - boost::none /* startTransaction */); -} - -template <typename Callable> -constexpr bool returnsVoid() { - return std::is_void_v<std::invoke_result_t<Callable>>; -} - -// Yields the checked out session before running the given function. If the function runs without -// throwing, will reacquire the session and verify it is still valid to proceed with the migration. -template <typename Callable, std::enable_if_t<!returnsVoid<Callable>(), int> = 0> -auto runWithoutSession(OperationContext* opCtx, Callable callable) { - MongoDOperationContextSession::checkIn(opCtx, OperationContextSession::CheckInReason::kYield); - - auto retVal = callable(); - - // The below code can throw, so it cannot run in a scope guard. - opCtx->checkForInterrupt(); - checkOutSessionAndVerifyTxnState(opCtx); - - return retVal; -} - -// Same as runWithoutSession above but takes a void function. -template <typename Callable, std::enable_if_t<returnsVoid<Callable>(), int> = 0> -void runWithoutSession(OperationContext* opCtx, Callable callable) { - MongoDOperationContextSession::checkIn(opCtx, OperationContextSession::CheckInReason::kYield); - - callable(); - - // The below code can throw, so it cannot run in a scope guard. - opCtx->checkForInterrupt(); - checkOutSessionAndVerifyTxnState(opCtx); -} -} // namespace - - -void MigrationBatchInserter::onCreateThread(const std::string& threadName) { - Client::initThread(threadName, getGlobalServiceContext(), nullptr); - { - stdx::lock_guard<Client> lk(cc()); - cc().setSystemOperationKillableByStepdown(lk); - } -} - -void MigrationBatchInserter::run(Status status) const try { - // Run is passed in a non-ok status if this function runs inline. - // That happens if we schedule this task on a ThreadPool that is - // already shutdown. If we were to schedule a task on a shutdown ThreadPool, - // then there is a logic error in our code. Therefore, we assert that here. - - invariant(status.isOK()); - auto arr = _batch["objects"].Obj(); - if (arr.isEmpty()) - return; - - auto executor = - Grid::get(_innerOpCtx->getServiceContext())->getExecutorPool()->getFixedExecutor(); - - auto applicationOpCtx = CancelableOperationContext( - cc().makeOperationContext(), _innerOpCtx->getCancellationToken(), executor); - - auto opCtx = applicationOpCtx.get(); - - auto assertNotAborted = [&]() { - { - stdx::lock_guard<Client> lk(*_outerOpCtx->getClient()); - _outerOpCtx->checkForInterrupt(); - } - opCtx->checkForInterrupt(); - }; - - auto it = arr.begin(); - while (it != arr.end()) { - int batchNumCloned = 0; - int batchClonedBytes = 0; - const int batchMaxCloned = migrateCloneInsertionBatchSize.load(); - - assertNotAborted(); - - write_ops::InsertCommandRequest insertOp(_nss); - insertOp.getWriteCommandRequestBase().setOrdered(true); - insertOp.setDocuments([&] { - std::vector<BSONObj> toInsert; - while (it != arr.end() && (batchMaxCloned <= 0 || batchNumCloned < batchMaxCloned)) { - const auto& doc = *it; - BSONObj docToClone = doc.Obj(); - toInsert.push_back(docToClone); - batchNumCloned++; - batchClonedBytes += docToClone.objsize(); - ++it; - } - return toInsert; - }()); - - { - // Disable the schema validation (during document inserts and updates) - // and any internal validation for opCtx for performInserts() - DisableDocumentValidation documentValidationDisabler( - opCtx, - DocumentValidationSettings::kDisableSchemaValidation | - DocumentValidationSettings::kDisableInternalValidation); - const auto reply = - write_ops_exec::performInserts(opCtx, insertOp, OperationSource::kFromMigrate); - for (unsigned long i = 0; i < reply.results.size(); ++i) { - uassertStatusOKWithContext( - reply.results[i], - str::stream() << "Insert of " << insertOp.getDocuments()[i] << " failed."); - } - // Revert to the original DocumentValidationSettings for opCtx - } - - migrationutil::persistUpdatedNumOrphans( - opCtx, _migrationId, _collectionUuid, batchNumCloned); - _migrationProgress->updateMaxOptime( - repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp()); - - ShardingStatistics::get(opCtx).countDocsClonedOnRecipient.addAndFetch(batchNumCloned); - ShardingStatistics::get(opCtx).countBytesClonedOnRecipient.addAndFetch(batchClonedBytes); - LOGV2(6718408, - "Incrementing numCloned count by {batchNumCloned} and numClonedBytes by " - "{batchClonedBytes}", - "batchNumCloned"_attr = batchNumCloned, - "batchClonedBytes"_attr = batchClonedBytes); - _migrationProgress->incNumCloned(batchNumCloned); - _migrationProgress->incNumBytes(batchClonedBytes); - - if (_writeConcern.needToWaitForOtherNodes() && _threadCount == 1) { - AdmissionContext admissionContext; - if (auto ticket = _secondaryThrottleTicket->tryAcquire(&admissionContext)) { - TicketHolderReleaser releaser( - std::move(*ticket), &admissionContext, _secondaryThrottleTicket); - runWithoutSession(_outerOpCtx, [&] { - repl::ReplicationCoordinator::StatusAndDuration replStatus = - repl::ReplicationCoordinator::get(opCtx)->awaitReplication( - opCtx, - repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(), - _writeConcern); - if (replStatus.status.code() == ErrorCodes::WriteConcernFailed) { - LOGV2_WARNING(22011, - "secondaryThrottle on, but doc insert timed out; continuing", - "migrationId"_attr = _migrationId.toBSON()); - } else { - uassertStatusOK(replStatus.status); - } - }); - } else { - // Ticket should always be available unless thread pool max size 1 setting is not - // being respected. - dassert(false); - } - } - - sleepmillis(migrateCloneInsertionBatchDelayMS.load()); - } -} catch (const DBException& e) { - stdx::lock_guard<Client> lk(*_innerOpCtx->getClient()); - _innerOpCtx->getServiceContext()->killOperation(lk, _innerOpCtx, ErrorCodes::Error(6718402)); - LOGV2(6718407, - "Batch application failed: {error}", - "Batch application failed", - "error"_attr = e.toStatus()); -} -} // namespace mongo diff --git a/src/mongo/db/s/migration_batch_inserter.h b/src/mongo/db/s/migration_batch_inserter.h deleted file mode 100644 index a4dd8d1a318..00000000000 --- a/src/mongo/db/s/migration_batch_inserter.h +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "boost/optional/optional.hpp" -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/db/cancelable_operation_context.h" -#include "mongo/db/catalog/document_validation.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/ops/write_ops_exec.h" -#include "mongo/db/ops/write_ops_gen.h" -#include "mongo/db/repl/repl_client_info.h" -#include "mongo/db/repl/replication_coordinator.h" -#include "mongo/db/s/migration_session_id.h" -#include "mongo/db/s/range_deletion_util.h" -#include "mongo/db/s/sharding_runtime_d_params_gen.h" -#include "mongo/db/s/sharding_statistics.h" -#include "mongo/db/session_catalog_mongod.h" -#include "mongo/db/write_concern_options.h" -#include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/grid.h" -#include "mongo/util/concurrency/ticketholder.h" -#include "mongo/util/uuid.h" - -#pragma once - -namespace mongo { - -// The purpose of this type is to allow inserters to communicate -// their progress to the outside world. -class MigrationCloningProgressSharedState { - mutable Mutex _m; - repl::OpTime _maxOptime; - long long _numCloned = 0; - long long _numBytes = 0; - -public: - void updateMaxOptime(const repl::OpTime& _newOptime) { - stdx::lock_guard lk(_m); - _maxOptime = std::max(_maxOptime, _newOptime); - } - repl::OpTime getMaxOptime() const { - stdx::lock_guard lk(_m); - return _maxOptime; - } - void incNumCloned(int num) { - stdx::lock_guard lk(_m); - _numCloned += num; - } - void incNumBytes(int num) { - stdx::lock_guard lk(_m); - _numBytes += num; - } - long long getNumCloned() const { - stdx::lock_guard lk(_m); - return _numCloned; - } - long long getNumBytes() const { - stdx::lock_guard lk(_m); - return _numBytes; - } -}; - -// This type contains a BSONObj _batch corresponding to a _migrateClone response. -// The purpose of this type is to perform the insertions for this batch. -// Those insertions happen in its "run" method. The MigrationBatchFetcher -// schedules these jobs on a thread pool. This class has no knowledge that it runs -// on a thread pool. It sole purpose is to perform insertions and communicate its progress -// (inluding the new max opTime). -class MigrationBatchInserter { -public: - // Do inserts. - void run(Status status) const; - - MigrationBatchInserter(OperationContext* outerOpCtx, - OperationContext* innerOpCtx, - BSONObj batch, - const NamespaceString& nss, - const ChunkRange& range, - const WriteConcernOptions& writeConcern, - const UUID& collectionUuid, - std::shared_ptr<MigrationCloningProgressSharedState> migrationProgress, - const UUID& migrationId, - int threadCount, - SemaphoreTicketHolder* secondaryThrottleTicket) - : _outerOpCtx{outerOpCtx}, - _innerOpCtx{innerOpCtx}, - _batch{batch}, - _nss{nss}, - _range{range}, - _writeConcern{writeConcern}, - _collectionUuid{collectionUuid}, - _migrationProgress{migrationProgress}, - _migrationId{migrationId}, - _threadCount{threadCount}, - _secondaryThrottleTicket{secondaryThrottleTicket} {} - - static void onCreateThread(const std::string& threadName); - -private: - OperationContext* _outerOpCtx; - OperationContext* _innerOpCtx; - BSONObj _batch; - NamespaceString _nss; - ChunkRange _range; - WriteConcernOptions _writeConcern; - UUID _collectionUuid; - std::shared_ptr<MigrationCloningProgressSharedState> _migrationProgress; - UUID _migrationId; - int _threadCount; - SemaphoreTicketHolder* _secondaryThrottleTicket; -}; - -} // namespace mongo diff --git a/src/mongo/db/s/migration_batch_mock_inserter.h b/src/mongo/db/s/migration_batch_mock_inserter.h deleted file mode 100644 index f53b33345d9..00000000000 --- a/src/mongo/db/s/migration_batch_mock_inserter.h +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/s/migration_batch_inserter.h" -#include "mongo/db/write_concern_options.h" -#include "mongo/s/catalog/type_chunk.h" - -#pragma once - -namespace mongo { - -class MigrationBatchMockInserter { -public: - void run(Status status) const { - // Run is passed in a non-ok status if this function runs inline. - // That happens if we schedule this task on a ThreadPool that is - // already shutdown. We should never do that. Therefore, - // we assert that here. - invariant(status.isOK()); - } - MigrationBatchMockInserter(OperationContext*, - OperationContext*, - BSONObj, - NamespaceString, - ChunkRange, - WriteConcernOptions, - UUID, - std::shared_ptr<MigrationCloningProgressSharedState>, - UUID, - int, - SemaphoreTicketHolder*) {} - - static void onCreateThread(const std::string& threadName) {} - -private: - BSONObj _batch; -}; - -} // namespace mongo diff --git a/src/mongo/db/s/migration_chunk_cloner_source.cpp b/src/mongo/db/s/migration_chunk_cloner_source.cpp index b2bbb68cdb1..6df6ebadd6f 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source.cpp @@ -27,87 +27,14 @@ * it in the license file. */ - -#include "mongo/bson/bsonobj.h" #include "mongo/platform/basic.h" #include "mongo/db/s/migration_chunk_cloner_source.h" - -#include "mongo/db/concurrency/locker.h" -#include "mongo/db/s/collection_sharding_runtime.h" -#include "mongo/db/s/migration_source_manager.h" - namespace mongo { MigrationChunkClonerSource::MigrationChunkClonerSource() = default; MigrationChunkClonerSource::~MigrationChunkClonerSource() = default; -LogInsertForShardingHandler::LogInsertForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - BSONObj doc, - repl::OpTime opTime) - : _opCtx(opCtx), _nss(std::move(nss)), _doc(doc.getOwned()), _opTime(std::move(opTime)) {} - -void LogInsertForShardingHandler::commit(boost::optional<Timestamp>) { - // TODO (SERVER-71444): Fix to be interruptible or document exception. - UninterruptibleLockGuard noInterrupt(_opCtx->lockState()); // NOLINT. - - auto csr = CollectionShardingRuntime::get(_opCtx, _nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(_opCtx, csr); - - if (auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock)) { - cloner->onInsertOp(_opCtx, _doc, _opTime); - } -} - -LogUpdateForShardingHandler::LogUpdateForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - boost::optional<BSONObj> preImageDoc, - BSONObj postImageDoc, - repl::OpTime opTime, - repl::OpTime prePostImageOpTime) - : _opCtx(opCtx), - _nss(std::move(nss)), - _preImageDoc(preImageDoc ? preImageDoc->getOwned() : boost::optional<BSONObj>(boost::none)), - _postImageDoc(postImageDoc.getOwned()), - _opTime(std::move(opTime)), - _prePostImageOpTime(std::move(prePostImageOpTime)) {} - -void LogUpdateForShardingHandler::commit(boost::optional<Timestamp>) { - // TODO (SERVER-71444): Fix to be interruptible or document exception. - UninterruptibleLockGuard noInterrupt(_opCtx->lockState()); // NOLINT. - - auto csr = CollectionShardingRuntime::get(_opCtx, _nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(_opCtx, csr); - - if (auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock)) { - cloner->onUpdateOp(_opCtx, _preImageDoc, _postImageDoc, _opTime, _prePostImageOpTime); - } -} - -LogDeleteForShardingHandler::LogDeleteForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - repl::DocumentKey documentKey, - repl::OpTime opTime, - repl::OpTime prePostImageOpTime) - : _opCtx(opCtx), - _nss(std::move(nss)), - _documentKey(std::move(documentKey)), - _opTime(std::move(opTime)), - _prePostImageOpTime(std::move(prePostImageOpTime)) {} - -void LogDeleteForShardingHandler::commit(boost::optional<Timestamp>) { - // TODO (SERVER-71444): Fix to be interruptible or document exception. - UninterruptibleLockGuard noInterrupt(_opCtx->lockState()); // NOLINT. - - auto csr = CollectionShardingRuntime::get(_opCtx, _nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(_opCtx, csr); - - if (auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock)) { - cloner->onDeleteOp(_opCtx, _documentKey, _opTime, _prePostImageOpTime); - } -} - } // namespace mongo diff --git a/src/mongo/db/s/migration_chunk_cloner_source.h b/src/mongo/db/s/migration_chunk_cloner_source.h index 199421d1327..9c1a2e908f4 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source.h +++ b/src/mongo/db/s/migration_chunk_cloner_source.h @@ -30,7 +30,6 @@ #pragma once #include "mongo/db/logical_session_id.h" -#include "mongo/db/op_observer_util.h" #include "mongo/util/time_support.h" namespace mongo { @@ -47,75 +46,6 @@ class OpTime; } // namespace repl /** - * Used to keep track of inserts that can be potentially added as xferMods of a migration. - */ -class LogInsertForShardingHandler final : public RecoveryUnit::Change { -public: - LogInsertForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - BSONObj doc, - repl::OpTime opTime); - - void commit(boost::optional<Timestamp>) override; - - void rollback() override {} - -private: - OperationContext* _opCtx; - const NamespaceString _nss; - const BSONObj _doc; - const repl::OpTime _opTime; -}; - -/** - * Used to keep track of updates that can be potentially added as xferMods of a migration. - */ -class LogUpdateForShardingHandler final : public RecoveryUnit::Change { -public: - LogUpdateForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - boost::optional<BSONObj> preImageDoc, - BSONObj postImageDoc, - repl::OpTime opTime, - repl::OpTime prePostImageOpTime); - - void commit(boost::optional<Timestamp>) override; - - void rollback() override {} - -private: - OperationContext* _opCtx; - const NamespaceString _nss; - const boost::optional<BSONObj> _preImageDoc; - const BSONObj _postImageDoc; - const repl::OpTime _opTime; - const repl::OpTime _prePostImageOpTime; -}; - -/** - * Used to keep track of deletes that can be potentially added as xferMods of a migration. - */ -class LogDeleteForShardingHandler final : public RecoveryUnit::Change { -public: - LogDeleteForShardingHandler(OperationContext* opCtx, - NamespaceString nss, - repl::DocumentKey documentKey, - repl::OpTime opTime, - repl::OpTime prePostImageOpTime); - - void commit(boost::optional<Timestamp>) override; - - void rollback() override {} - -private: - OperationContext* _opCtx; - const NamespaceString _nss; - const repl::DocumentKey _documentKey; - const repl::OpTime _opTime; - const repl::OpTime _prePostImageOpTime; -}; - -/** * This class is responsible for producing chunk documents to be moved from donor to a recipient * shard and its methods represent cloning stages. Its lifetime is owned and controlled by a single * migration source manager which registers it for notifications from the replication subsystem @@ -189,6 +119,14 @@ public: // running list of changes, which need to be fetched. /** + * Checks whether the specified document is within the bounds of the chunk, which this cloner + * is responsible for. + * + * NOTE: Must be called with at least IS lock held on the collection. + */ + virtual bool isDocumentInMigratingChunk(const BSONObj& doc) = 0; + + /** * Notifies this cloner that an insert happened to the collection, which it owns. It is up to * the cloner's implementation to decide what to do with this information and it is valid for * the implementation to ignore it. @@ -220,7 +158,7 @@ public: * NOTE: Must be called with at least IX lock held on the collection. */ virtual void onDeleteOp(OperationContext* opCtx, - const repl::DocumentKey& documentKey, + const BSONObj& deletedDocId, const repl::OpTime& opTime, const repl::OpTime& preImageOpTime) = 0; diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp index 910c00aaf0d..547080ff4ea 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy.cpp @@ -33,26 +33,19 @@ #include "mongo/db/s/migration_chunk_cloner_source_legacy.h" -#include <fmt/format.h> - #include "mongo/base/status.h" -#include "mongo/bson/bsonobj.h" #include "mongo/client/read_preference.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/db_raii.h" -#include "mongo/db/dbdirectclient.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/exec/working_set_common.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/ops/write_ops_retryability.h" -#include "mongo/db/query/get_executor.h" #include "mongo/db/repl/optime.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/migration_source_manager.h" -#include "mongo/db/s/operation_sharding_state.h" #include "mongo/db/s/shard_key_index_util.h" #include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/sharding_statistics.h" @@ -75,8 +68,6 @@ namespace mongo { namespace { -using namespace fmt::literals; - const char kRecvChunkStatus[] = "_recvChunkStatus"; const char kRecvChunkCommit[] = "_recvChunkCommit"; const char kRecvChunkAbort[] = "_recvChunkAbort"; @@ -85,24 +76,13 @@ const int kMaxObjectPerChunk{250000}; const Hours kMaxWaitToCommitCloneForJumboChunk(6); MONGO_FAIL_POINT_DEFINE(failTooMuchMemoryUsed); -MONGO_FAIL_POINT_DEFINE(hangAfterProcessingDeferredXferMods); -/** - * Returns true if the given BSON object in the shard key value pair format is within the given - * range. - */ -bool isShardKeyValueInRange(const BSONObj& shardKeyValue, const BSONObj& min, const BSONObj& max) { - return shardKeyValue.woCompare(min) >= 0 && shardKeyValue.woCompare(max) < 0; -} - -/** - * Returns true if the given BSON document is within the given chunk range. - */ -bool isDocInRange(const BSONObj& obj, - const BSONObj& min, - const BSONObj& max, - const ShardKeyPattern& shardKeyPattern) { - return isShardKeyValueInRange(shardKeyPattern.extractShardKeyFromDoc(obj), min, max); +bool isInRange(const BSONObj& obj, + const BSONObj& min, + const BSONObj& max, + const ShardKeyPattern& shardKeyPattern) { + BSONObj k = shardKeyPattern.extractShardKeyFromDoc(obj); + return k.woCompare(min) >= 0 && k.woCompare(max) < 0; } BSONObj createRequestWithSessionId(StringData commandName, @@ -116,8 +96,9 @@ BSONObj createRequestWithSessionId(StringData commandName, return builder.obj(); } -BSONObj getDocumentKeyFromReplOperation(repl::ReplOperation replOperation) { - switch (replOperation.getOpType()) { +BSONObj getDocumentKeyFromReplOperation(repl::ReplOperation replOperation, + repl::OpTypeEnum opType) { + switch (opType) { case repl::OpTypeEnum::kInsert: case repl::OpTypeEnum::kDelete: return replOperation.getObject(); @@ -145,29 +126,36 @@ char getOpCharForCrudOpType(repl::OpTypeEnum opType) { } // namespace -LogTransactionOperationsForShardingHandler::LogTransactionOperationsForShardingHandler( - LogicalSessionId lsid, - const std::vector<repl::OplogEntry>& stmts, - repl::OpTime prepareOrCommitOpTime) - : _lsid(std::move(lsid)), _prepareOrCommitOpTime(std::move(prepareOrCommitOpTime)) { - _stmts.reserve(stmts.size()); - _ownedReplBSONObj.reserve(stmts.size()); - - for (const auto& op : stmts) { - auto ownedBSON = op.getDurableReplOperation().toBSON().getOwned(); - _ownedReplBSONObj.push_back(ownedBSON); - _stmts.push_back( - repl::ReplOperation::parse({"MigrationChunkClonerSource_toReplOperation"}, ownedBSON)); +/** + * Used to commit work for LogOpForSharding. Used to keep track of changes in documents that are + * part of a chunk being migrated. + */ +class LogOpForShardingHandler final : public RecoveryUnit::Change { +public: + /** + * Invariant: idObj should belong to a document that is part of the active chunk being migrated + */ + LogOpForShardingHandler(MigrationChunkClonerSourceLegacy* cloner, + const BSONObj& idObj, + const char op, + const repl::OpTime& opTime) + : _cloner(cloner), _idObj(idObj.getOwned()), _op(op), _opTime(opTime) {} + + void commit(boost::optional<Timestamp>) override { + _cloner->_addToTransferModsQueue(_idObj, _op, _opTime); + _cloner->_decrementOutstandingOperationTrackRequests(); + } + + void rollback() override { + _cloner->_decrementOutstandingOperationTrackRequests(); } -} -LogTransactionOperationsForShardingHandler::LogTransactionOperationsForShardingHandler( - LogicalSessionId lsid, - const std::vector<repl::ReplOperation>& stmts, - repl::OpTime prepareOrCommitOpTime) - : _lsid(std::move(lsid)), - _stmts(stmts), - _prepareOrCommitOpTime(std::move(prepareOrCommitOpTime)) {} +private: + MigrationChunkClonerSourceLegacy* const _cloner; + const BSONObj _idObj; + const char _op; + const repl::OpTime _opTime; +}; void LogTransactionOperationsForShardingHandler::commit(boost::optional<Timestamp>) { std::set<NamespaceString> namespacesTouchedByTransaction; @@ -226,45 +214,39 @@ void LogTransactionOperationsForShardingHandler::commit(boost::optional<Timestam continue; } - auto preImageDocKey = getDocumentKeyFromReplOperation(stmt); + auto documentKey = getDocumentKeyFromReplOperation(stmt, opType); - auto idElement = preImageDocKey["_id"]; + auto idElement = documentKey["_id"]; if (idElement.eoo()) { LOGV2_WARNING(21994, + "Received a document without an _id field, ignoring: {documentKey}", "Received a document without an _id and will ignore that document", - "documentKey"_attr = redact(preImageDocKey)); + "documentKey"_attr = redact(documentKey)); continue; } - if (opType == repl::OpTypeEnum::kUpdate) { - auto const& shardKeyPattern = cloner->_shardKeyPattern; - auto preImageShardKeyValues = - shardKeyPattern.extractShardKeyFromDocumentKey(preImageDocKey); - - // If prepare was performed from another term, we will not have the post image doc key - // since it is not persisted in the oplog. - auto postImageDocKey = stmt.getPostImageDocumentKey(); - if (!postImageDocKey.isEmpty()) { - if (!cloner->_processUpdateForXferMod(preImageDocKey, postImageDocKey)) { - // We don't need to add this op to session migration if neither post or pre - // image doc falls within the chunk range. - continue; - } + auto const& minKey = cloner->_args.getMin().get(); + auto const& maxKey = cloner->_args.getMax().get(); + auto const& shardKeyPattern = cloner->_shardKeyPattern; + + if (!isInRange(documentKey, minKey, maxKey, shardKeyPattern)) { + // If the preImageDoc is not in range but the postImageDoc was, we know that the + // document has changed shard keys and no longer belongs in the chunk being cloned. + // We will model the deletion of the preImage document so that the destination chunk + // does not receive an outdated version of this document. + if (opType == repl::OpTypeEnum::kUpdate && + isInRange(stmt.getPreImageDocumentKey(), minKey, maxKey, shardKeyPattern) && + !stmt.getPreImageDocumentKey()["_id"].eoo()) { + opType = repl::OpTypeEnum::kDelete; + idElement = stmt.getPreImageDocumentKey()["id"]; } else { - // We can't perform reads here using the same recovery unit because the transaction - // is already committed. We instead defer performing the reads when xferMods command - // is called. Also allow this op to be added to session migration since we can't - // tell whether post image doc will fall within the chunk range. If it turns out - // both preImage and postImage doc don't fall into the chunk range, it is not wrong - // for this op to be added to session migration, but it will result in wasted work - // and unneccesary extra oplog storage on the destination. - cloner->_deferProcessingForXferMod(preImageDocKey); + continue; } - } else { - cloner->_addToTransferModsQueue(idElement.wrap(), getOpCharForCrudOpType(opType), {}); } addToSessionMigrationOptimeQueueIfNeeded(cloner, nss, _prepareOrCommitOpTime); + + cloner->_addToTransferModsQueue(idElement.wrap(), getOpCharForCrudOpType(opType), {}); } } @@ -397,7 +379,7 @@ StatusWith<BSONObj> MigrationChunkClonerSourceLegacy::commitClone(OperationConte } } else { invariant(PlanExecutor::IS_EOF == _jumboChunkCloneState->clonerState); - invariant(!_cloneList.hasMore()); + invariant(_cloneLocs.empty()); } } @@ -460,6 +442,10 @@ void MigrationChunkClonerSourceLegacy::cancelClone(OperationContext* opCtx) noex } } +bool MigrationChunkClonerSourceLegacy::isDocumentInMigratingChunk(const BSONObj& doc) { + return isInRange(doc, getMin(), getMax(), _shardKeyPattern); +} + void MigrationChunkClonerSourceLegacy::onInsertOp(OperationContext* opCtx, const BSONObj& insertedDoc, const repl::OpTime& opTime) { @@ -476,7 +462,7 @@ void MigrationChunkClonerSourceLegacy::onInsertOp(OperationContext* opCtx, return; } - if (!isDocInRange(insertedDoc, getMin(), getMax(), _shardKeyPattern)) { + if (!isInRange(insertedDoc, getMin(), getMax(), _shardKeyPattern)) { return; } @@ -484,8 +470,13 @@ void MigrationChunkClonerSourceLegacy::onInsertOp(OperationContext* opCtx, return; } - _addToTransferModsQueue(idElement.wrap(), 'i', opCtx->getTxnNumber() ? opTime : repl::OpTime()); - _decrementOutstandingOperationTrackRequests(); + if (opCtx->getTxnNumber()) { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'i', opTime)); + } else { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'i', repl::OpTime())); + } } void MigrationChunkClonerSourceLegacy::onUpdateOp(OperationContext* opCtx, @@ -506,16 +497,13 @@ void MigrationChunkClonerSourceLegacy::onUpdateOp(OperationContext* opCtx, return; } - if (!isDocInRange(postImageDoc, getMin(), getMax(), _shardKeyPattern)) { + if (!isInRange(postImageDoc, getMin(), getMax(), _shardKeyPattern)) { // If the preImageDoc is not in range but the postImageDoc was, we know that the document // has changed shard keys and no longer belongs in the chunk being cloned. We will model // the deletion of the preImage document so that the destination chunk does not receive an // outdated version of this document. - if (preImageDoc && isDocInRange(*preImageDoc, getMin(), getMax(), _shardKeyPattern)) { - onDeleteOp(opCtx, - repl::getDocumentKey(_shardKeyPattern, *preImageDoc), - opTime, - prePostImageOpTime); + if (preImageDoc && isInRange(*preImageDoc, getMin(), getMax(), _shardKeyPattern)) { + onDeleteOp(opCtx, *preImageDoc, opTime, prePostImageOpTime); } return; } @@ -524,40 +512,29 @@ void MigrationChunkClonerSourceLegacy::onUpdateOp(OperationContext* opCtx, return; } - _addToTransferModsQueue(idElement.wrap(), 'u', opCtx->getTxnNumber() ? opTime : repl::OpTime()); - _decrementOutstandingOperationTrackRequests(); + if (opCtx->getTxnNumber()) { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'u', opTime)); + } else { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'u', repl::OpTime())); + } } void MigrationChunkClonerSourceLegacy::onDeleteOp(OperationContext* opCtx, - const repl::DocumentKey& documentKey, + const BSONObj& deletedDocId, const repl::OpTime& opTime, const repl::OpTime&) { dassert(opCtx->lockState()->isCollectionLockedForMode(nss(), MODE_IX)); - const auto shardKeyAndId = documentKey.getShardKeyAndId(); - - BSONElement idElement = documentKey.getId()["_id"]; + BSONElement idElement = deletedDocId["_id"]; if (idElement.eoo()) { LOGV2_WARNING( 21997, "logDeleteOp received a document without an _id field, ignoring deleted doc: " - "{shardKeyAndId}", + "{deletedDocId}", "logDeleteOp received a document without an _id field and will ignore that document", - "deletedDocShardKeyAndId"_attr = redact(shardKeyAndId)); - return; - } - - if (!documentKey.getShardKey()) { - LOGV2_WARNING(8023600, - "logDeleteOp received a document without the shard key field and will ignore " - "that document", - "deletedDocShardKeyAndId"_attr = redact(shardKeyAndId)); - return; - } - - const auto shardKeyValue = - _shardKeyPattern.extractShardKeyFromDocumentKey(*documentKey.getShardKey()); - if (!isShardKeyValueInRange(shardKeyValue, getMin(), getMax())) { + "deletedDocId"_attr = redact(deletedDocId)); return; } @@ -565,9 +542,13 @@ void MigrationChunkClonerSourceLegacy::onDeleteOp(OperationContext* opCtx, return; } - _addToTransferModsQueue( - documentKey.getId(), 'd', opCtx->getTxnNumber() ? opTime : repl::OpTime()); - _decrementOutstandingOperationTrackRequests(); + if (opCtx->getTxnNumber()) { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'd', opTime)); + } else { + opCtx->recoveryUnit()->registerChange( + std::make_unique<LogOpForShardingHandler>(this, idElement.wrap(), 'd', repl::OpTime())); + } } void MigrationChunkClonerSourceLegacy::_addToSessionMigrationOptimeQueue( @@ -683,7 +664,6 @@ void MigrationChunkClonerSourceLegacy::_nextCloneBatchFromIndexScan(OperationCon lk.unlock(); ShardingStatistics::get(opCtx).countDocsClonedOnDonor.addAndFetch(1); - ShardingStatistics::get(opCtx).countBytesClonedOnDonor.addAndFetch(obj.objsize()); } } catch (DBException& exception) { exception.addContext("Executor error while scanning for documents belonging to chunk"); @@ -705,57 +685,38 @@ void MigrationChunkClonerSourceLegacy::_nextCloneBatchFromCloneLocs(OperationCon internalQueryExecYieldIterations.load(), Milliseconds(internalQueryExecYieldPeriodMS.load())); - while (true) { - int recordsNoLongerExist = 0; - auto docInFlight = _cloneList.getNextDoc(opCtx, collection, &recordsNoLongerExist); - - if (recordsNoLongerExist) { - stdx::lock_guard lk(_mutex); - _numRecordsPassedOver += recordsNoLongerExist; - } - - const auto& doc = docInFlight->getDoc(); - if (!doc) { - break; - } + stdx::unique_lock<Latch> lk(_mutex); + auto iter = _cloneLocs.begin(); + for (; iter != _cloneLocs.end(); ++iter) { // We must always make progress in this method by at least one document because empty // return indicates there is no more initial clone data. if (arrBuilder->arrSize() && tracker.intervalHasElapsed()) { - _cloneList.insertOverflowDoc(*doc); break; } - // Do not send documents that are no longer in the chunk range being moved. This can - // happen when document shard key value of the document changed after the initial - // index scan during cloning. This is needed because the destination is very - // conservative in processing xferMod deletes and won't delete docs that are not in - // the range of the chunk being migrated. - if (!isDocInRange( - doc->value(), _args.getMin().value(), _args.getMax().value(), _shardKeyPattern)) { - { - stdx::lock_guard lk(_mutex); - _numRecordsPassedOver++; + auto nextRecordId = *iter; + + lk.unlock(); + + Snapshotted<BSONObj> doc; + if (collection->findDoc(opCtx, nextRecordId, &doc)) { + // Use the builder size instead of accumulating the document sizes directly so + // that we take into consideration the overhead of BSONArray indices. + if (arrBuilder->arrSize() && + (arrBuilder->len() + doc.value().objsize() + 1024) > BSONObjMaxUserSize) { + + break; } - continue; - } - // Use the builder size instead of accumulating the document sizes directly so - // that we take into consideration the overhead of BSONArray indices. - if (arrBuilder->arrSize() && - (arrBuilder->len() + doc->value().objsize() + 1024) > BSONObjMaxUserSize) { - _cloneList.insertOverflowDoc(*doc); - break; + arrBuilder->append(doc.value()); + ShardingStatistics::get(opCtx).countDocsClonedOnDonor.addAndFetch(1); } - { - stdx::lock_guard lk(_mutex); - _numRecordsCloned++; - } - arrBuilder->append(doc->value()); - ShardingStatistics::get(opCtx).countDocsClonedOnDonor.addAndFetch(1); - ShardingStatistics::get(opCtx).countBytesClonedOnDonor.addAndFetch(doc->value().objsize()); + lk.lock(); } + + _cloneLocs.erase(_cloneLocs.begin(), iter); } uint64_t MigrationChunkClonerSourceLegacy::getCloneBatchBufferAllocationSize() { @@ -764,7 +725,7 @@ uint64_t MigrationChunkClonerSourceLegacy::getCloneBatchBufferAllocationSize() { return static_cast<uint64_t>(BSONObjMaxUserSize); return std::min(static_cast<uint64_t>(BSONObjMaxUserSize), - _averageObjectSizeForCloneLocs * _cloneList.size()); + _averageObjectSizeForCloneLocs * _cloneLocs.size()); } Status MigrationChunkClonerSourceLegacy::nextCloneBatch(OperationContext* opCtx, @@ -787,91 +748,18 @@ Status MigrationChunkClonerSourceLegacy::nextCloneBatch(OperationContext* opCtx, return Status::OK(); } -bool MigrationChunkClonerSourceLegacy::_processUpdateForXferMod(const BSONObj& preImageDocKey, - const BSONObj& postImageDocKey) { - auto const& minKey = _args.getMin().value(); - auto const& maxKey = _args.getMax().value(); - - auto postShardKeyValues = _shardKeyPattern.extractShardKeyFromDocumentKey(postImageDocKey); - fassert(6836100, !postShardKeyValues.isEmpty()); - - auto opType = repl::OpTypeEnum::kUpdate; - auto idElement = preImageDocKey["_id"]; - - if (!isShardKeyValueInRange(postShardKeyValues, minKey, maxKey)) { - // If the preImageDoc is not in range but the postImageDoc was, we know that the - // document has changed shard keys and no longer belongs in the chunk being cloned. - // We will model the deletion of the preImage document so that the destination chunk - // does not receive an outdated version of this document. - - auto preImageShardKeyValues = - _shardKeyPattern.extractShardKeyFromDocumentKey(preImageDocKey); - fassert(6836101, !preImageShardKeyValues.isEmpty()); - - if (!isShardKeyValueInRange(preImageShardKeyValues, minKey, maxKey)) { - return false; - } - - opType = repl::OpTypeEnum::kDelete; - idElement = postImageDocKey["_id"]; - } - - _addToTransferModsQueue(idElement.wrap(), getOpCharForCrudOpType(opType), {}); - - return true; -} - -void MigrationChunkClonerSourceLegacy::_deferProcessingForXferMod(const BSONObj& preImageDocKey) { - stdx::lock_guard<Latch> sl(_mutex); - _deferredReloadOrDeletePreImageDocKeys.push_back(preImageDocKey.getOwned()); - _deferredUntransferredOpsCounter++; -} - -void MigrationChunkClonerSourceLegacy::_processDeferredXferMods(OperationContext* opCtx, - Database* db) { - std::vector<BSONObj> deferredReloadOrDeletePreImageDocKeys; - - { - stdx::unique_lock lk(_mutex); - deferredReloadOrDeletePreImageDocKeys.swap(_deferredReloadOrDeletePreImageDocKeys); - } - - for (const auto& preImageDocKey : deferredReloadOrDeletePreImageDocKeys) { - auto idElement = preImageDocKey["_id"]; - BSONObj newerVersionDoc; - if (!Helpers::findById(opCtx, db, nss().ns(), BSON("_id" << idElement), newerVersionDoc)) { - // If the document can no longer be found, this means that another later op must have - // deleted it. That delete would have been captured by the xferMods so nothing else to - // do here. - continue; - } - - auto postImageDocKey = - CollectionMetadata::extractDocumentKey(&_shardKeyPattern, newerVersionDoc); - static_cast<void>(_processUpdateForXferMod(preImageDocKey, postImageDocKey)); - } - - hangAfterProcessingDeferredXferMods.execute([&](const auto& data) { - if (!deferredReloadOrDeletePreImageDocKeys.empty()) { - hangAfterProcessingDeferredXferMods.pauseWhileSet(); - } - }); -} - Status MigrationChunkClonerSourceLegacy::nextModsBatch(OperationContext* opCtx, Database* db, BSONObjBuilder* builder) { dassert(opCtx->lockState()->isCollectionLockedForMode(nss(), MODE_IS)); - _processDeferredXferMods(opCtx, db); - std::list<BSONObj> deleteList; std::list<BSONObj> updateList; { // All clone data must have been drained before starting to fetch the incremental changes. stdx::unique_lock<Latch> lk(_mutex); - invariant(!_cloneList.hasMore()); + invariant(_cloneLocs.empty()); // The "snapshot" for delete and update list must be taken under a single lock. This is to // ensure that we will preserve the causal order of writes. Always consume the delete @@ -883,11 +771,6 @@ Status MigrationChunkClonerSourceLegacy::nextModsBatch(OperationContext* opCtx, updateList.splice(updateList.cbegin(), _reload); } - // It's important to abandon any open snapshots before processing updates so that we are sure - // that our snapshot is at least as new as those updates. It's possible for a stale snapshot to - // still be open from reads performed by _processDeferredXferMods(), above. - opCtx->recoveryUnit()->abandonSnapshot(); - StringData ns = nss().ns().c_str(); BSONArrayBuilder arrDel(builder->subarrayStart("deleted")); auto noopFn = [](BSONObj idDoc, BSONObj* fullDoc) { @@ -914,7 +797,6 @@ Status MigrationChunkClonerSourceLegacy::nextModsBatch(OperationContext* opCtx, _untransferredDeletesCounter = _deleted.size(); _reload.splice(_reload.cbegin(), updateList); _untransferredUpsertsCounter = _reload.size(); - _deferredUntransferredOpsCounter = _deferredReloadOrDeletePreImageDocKeys.size(); return Status::OK(); } @@ -929,8 +811,6 @@ void MigrationChunkClonerSourceLegacy::_cleanup() { _untransferredUpsertsCounter = 0; _deleted.clear(); _untransferredDeletesCounter = 0; - _deferredReloadOrDeletePreImageDocKeys.clear(); - _deferredUntransferredOpsCounter = 0; } StatusWith<BSONObj> MigrationChunkClonerSourceLegacy::_callRecipient(OperationContext* opCtx, @@ -1058,8 +938,6 @@ Status MigrationChunkClonerSourceLegacy::_storeCurrentLocs(OperationContext* opC try { BSONObj obj; RecordId recordId; - RecordIdSet recordIdSet; - while (PlanExecutor::ADVANCED == exec->getNext(&obj, &recordId)) { Status interruptStatus = opCtx->checkForInterruptNoAssert(); if (!interruptStatus.isOK()) { @@ -1067,20 +945,19 @@ Status MigrationChunkClonerSourceLegacy::_storeCurrentLocs(OperationContext* opC } if (!isLargeChunk) { - recordIdSet.insert(recordId); + stdx::lock_guard<Latch> lk(_mutex); + _cloneLocs.insert(recordId); } if (++recCount > maxRecsWhenFull) { isLargeChunk = true; if (_forceJumbo) { - recordIdSet.clear(); + _cloneLocs.clear(); break; } } } - - _cloneList.populateList(std::move(recordIdSet)); } catch (DBException& exception) { exception.addContext("Executor error while scanning for documents belonging to chunk"); throw; @@ -1178,9 +1055,9 @@ Status MigrationChunkClonerSourceLegacy::_checkRecipientCloningStatus(OperationC stdx::lock_guard<Latch> sl(_mutex); + const std::size_t cloneLocsRemaining = _cloneLocs.size(); int64_t untransferredModsSizeBytes = _untransferredDeletesCounter * _averageObjectIdSize + - (_untransferredUpsertsCounter + _deferredUntransferredOpsCounter) * - _averageObjectSizeForCloneLocs; + _untransferredUpsertsCounter * _averageObjectSizeForCloneLocs; if (_forceJumbo && _jumboChunkCloneState) { LOGV2(21992, @@ -1200,14 +1077,13 @@ Status MigrationChunkClonerSourceLegacy::_checkRecipientCloningStatus(OperationC "moveChunk data transfer progress", "response"_attr = redact(res), "memoryUsedBytes"_attr = _memoryUsed, - "docsRemainingToClone"_attr = - _cloneList.size() - _numRecordsCloned - _numRecordsPassedOver, + "docsRemainingToClone"_attr = cloneLocsRemaining, "untransferredModsSizeBytes"_attr = untransferredModsSizeBytes); } if (res["state"].String() == "steady" && sessionCatalogSourceInCatchupPhase && estimateUntransferredSessionsSize == 0) { - if (_cloneList.hasMore() || + if (cloneLocsRemaining != 0 || (_jumboChunkCloneState && _forceJumbo && PlanExecutor::IS_EOF != _jumboChunkCloneState->clonerState)) { return {ErrorCodes::OperationIncomplete, @@ -1246,7 +1122,6 @@ Status MigrationChunkClonerSourceLegacy::_checkRecipientCloningStatus(OperationC "moveChunk data transfer within threshold to allow write blocking", "_untransferredUpsertsCounter"_attr = _untransferredUpsertsCounter, "_untransferredDeletesCounter"_attr = _untransferredDeletesCounter, - "_deferredUntransferredOpsCounter"_attr = _deferredUntransferredOpsCounter, "_averageObjectSizeForCloneLocs"_attr = _averageObjectSizeForCloneLocs, "_averageObjectIdSize"_attr = _averageObjectIdSize, "untransferredModsSizeBytes"_attr = untransferredModsSizeBytes, @@ -1352,135 +1227,4 @@ MigrationChunkClonerSourceLegacy::getNotificationForNextSessionMigrationBatch() return _sessionCatalogSource->getNotificationForNewOplog(); } -MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWithLock::DocumentInFlightWithLock( - WithLock lock, MigrationChunkClonerSourceLegacy::CloneList& clonerList) - : _inProgressReadToken( - std::make_unique<MigrationChunkClonerSourceLegacy::CloneList::InProgressReadToken>( - lock, clonerList)) {} - -void MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWithLock::setDoc( - boost::optional<Snapshotted<BSONObj>> doc) { - _doc = std::move(doc); -} - -std::unique_ptr<MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock> -MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWithLock::release() { - invariant(_inProgressReadToken); - - return std::make_unique< - MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock>( - std::move(_inProgressReadToken), std::move(_doc)); -} - -MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock:: - DocumentInFlightWhileNotInLock( - std::unique_ptr<CloneList::InProgressReadToken> inProgressReadToken, - boost::optional<Snapshotted<BSONObj>> doc) - : _inProgressReadToken(std::move(inProgressReadToken)), _doc(std::move(doc)) {} - -void MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock::setDoc( - boost::optional<Snapshotted<BSONObj>> doc) { - _doc = std::move(doc); -} - -const boost::optional<Snapshotted<BSONObj>>& -MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock::getDoc() { - return _doc; -} - -MigrationChunkClonerSourceLegacy::CloneList::InProgressReadToken::InProgressReadToken( - WithLock withLock, CloneList& cloneList) - : _cloneList(cloneList) { - _cloneList._startedOneInProgressRead(withLock); -} - -MigrationChunkClonerSourceLegacy::CloneList::InProgressReadToken::~InProgressReadToken() { - _cloneList._finishedOneInProgressRead(); -} - -MigrationChunkClonerSourceLegacy::CloneList::CloneList() { - _recordIdsIter = _recordIds.begin(); -} - -void MigrationChunkClonerSourceLegacy::CloneList::populateList(RecordIdSet recordIds) { - stdx::lock_guard lk(_mutex); - _recordIds = std::move(recordIds); - _recordIdsIter = _recordIds.begin(); -} - -void MigrationChunkClonerSourceLegacy::CloneList::insertOverflowDoc(Snapshotted<BSONObj> doc) { - stdx::lock_guard lk(_mutex); - invariant(_inProgressReads >= 1); - _overflowDocs.push_back(std::move(doc)); -} - -bool MigrationChunkClonerSourceLegacy::CloneList::hasMore() const { - stdx::lock_guard lk(_mutex); - return _recordIdsIter != _recordIds.cend() && _inProgressReads > 0; -} - -std::unique_ptr<MigrationChunkClonerSourceLegacy::CloneList::DocumentInFlightWhileNotInLock> -MigrationChunkClonerSourceLegacy::CloneList::getNextDoc(OperationContext* opCtx, - const CollectionPtr& collection, - int* numRecordsNoLongerExist) { - while (true) { - stdx::unique_lock lk(_mutex); - invariant(_inProgressReads >= 0); - RecordId nextRecordId; - - opCtx->waitForConditionOrInterrupt(_moreDocsCV, lk, [&]() { - return _recordIdsIter != _recordIds.end() || !_overflowDocs.empty() || - _inProgressReads == 0; - }); - - DocumentInFlightWithLock docInFlight(lk, *this); - - // One of the following must now be true (corresponding to the three if conditions): - // 1. There is a document in the overflow set - // 2. The iterator has not reached the end of the record id set - // 3. The overflow set is empty, the iterator is at the end, and - // no threads are holding a document. This condition indicates - // that there are no more docs to return for the cloning phase. - if (!_overflowDocs.empty()) { - docInFlight.setDoc(std::move(_overflowDocs.front())); - _overflowDocs.pop_front(); - return docInFlight.release(); - } else if (_recordIdsIter != _recordIds.end()) { - nextRecordId = *_recordIdsIter; - ++_recordIdsIter; - } else { - return docInFlight.release(); - } - - lk.unlock(); - - auto docInFlightWhileNotLocked = docInFlight.release(); - - Snapshotted<BSONObj> doc; - if (collection->findDoc(opCtx, nextRecordId, &doc)) { - docInFlightWhileNotLocked->setDoc(std::move(doc)); - return docInFlightWhileNotLocked; - } - - if (numRecordsNoLongerExist) { - (*numRecordsNoLongerExist)++; - } - } -} - -size_t MigrationChunkClonerSourceLegacy::CloneList::size() const { - stdx::unique_lock lk(_mutex); - return _recordIds.size(); -} - -void MigrationChunkClonerSourceLegacy::CloneList::_startedOneInProgressRead(WithLock) { - _inProgressReads++; -} - -void MigrationChunkClonerSourceLegacy::CloneList::_finishedOneInProgressRead() { - stdx::lock_guard lk(_mutex); - _inProgressReads--; - _moreDocsCV.notify_one(); -} - } // namespace mongo diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy.h b/src/mongo/db/s/migration_chunk_cloner_source_legacy.h index 22ea6623d30..8c15fa7a0cb 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy.h +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy.h @@ -29,10 +29,8 @@ #pragma once -#include <deque> #include <list> #include <memory> -#include <mutex> #include <set> #include "mongo/bson/bsonobj.h" @@ -67,13 +65,13 @@ const long long kFixedCommandOverhead = 32 * 1024; */ class LogTransactionOperationsForShardingHandler final : public RecoveryUnit::Change { public: - LogTransactionOperationsForShardingHandler(LogicalSessionId lsid, - const std::vector<repl::OplogEntry>& stmts, - repl::OpTime prepareOrCommitOpTime); - - LogTransactionOperationsForShardingHandler(LogicalSessionId lsid, + /** + * Invariant: idObj should belong to a document that is part of the active chunk being migrated + */ + LogTransactionOperationsForShardingHandler(const LogicalSessionId lsid, const std::vector<repl::ReplOperation>& stmts, - repl::OpTime prepareOrCommitOpTime); + const repl::OpTime& prepareOrCommitOpTime) + : _lsid(lsid), _stmts(stmts), _prepareOrCommitOpTime(prepareOrCommitOpTime) {} void commit(boost::optional<Timestamp>) override; @@ -81,8 +79,6 @@ public: private: const LogicalSessionId _lsid; - // Use to keep BSON obj alive for the lifetime of this object. - std::vector<BSONObj> _ownedReplBSONObj; std::vector<repl::ReplOperation> _stmts; const repl::OpTime _prepareOrCommitOpTime; }; @@ -111,6 +107,8 @@ public: void cancelClone(OperationContext* opCtx) noexcept override; + bool isDocumentInMigratingChunk(const BSONObj& doc) override; + void onInsertOp(OperationContext* opCtx, const BSONObj& insertedDoc, const repl::OpTime& opTime) override; @@ -122,7 +120,7 @@ public: const repl::OpTime& prePostImageOpTime) override; void onDeleteOp(OperationContext* opCtx, - const repl::DocumentKey& documentKey, + const BSONObj& deletedDocId, const repl::OpTime& opTime, const repl::OpTime& preImageOpTime) override; @@ -226,168 +224,6 @@ private: friend class LogOpForShardingHandler; friend class LogTransactionOperationsForShardingHandler; - using RecordIdSet = std::set<RecordId>; - - /** - * This is responsible for all the logic revolving around handling documents that needs to be - * cloned. - * - * This class is multithread-safe. - */ - class CloneList { - public: - /** - * Simple container that increments the given counter when this is constructed and - * decrements it when it is destroyed. User of this class is responsible for holding - * necessary mutexes when counter is being modified. - */ - class InProgressReadToken { - public: - InProgressReadToken(WithLock, CloneList& cloneList); - InProgressReadToken(const InProgressReadToken&) = delete; - InProgressReadToken(InProgressReadToken&&) = default; - - ~InProgressReadToken(); - - private: - CloneList& _cloneList; - }; - - /** - * Container for a document that can be added to the nextCloneBatch call. As long as - * instances of this object exist, it will prevent getNextDoc from prematurely returning - * an empty response (which means there are no more docs left to clone). - * - * This assumes that _mutex is not being held when it is destroyed. - */ - class DocumentInFlightWhileNotInLock { - public: - DocumentInFlightWhileNotInLock(std::unique_ptr<InProgressReadToken> inProgressReadToken, - boost::optional<Snapshotted<BSONObj>> doc); - DocumentInFlightWhileNotInLock(const DocumentInFlightWhileNotInLock&) = delete; - DocumentInFlightWhileNotInLock(DocumentInFlightWhileNotInLock&&) = default; - - void setDoc(boost::optional<Snapshotted<BSONObj>> doc); - const boost::optional<Snapshotted<BSONObj>>& getDoc(); - - private: - std::unique_ptr<InProgressReadToken> _inProgressReadToken; - boost::optional<Snapshotted<BSONObj>> _doc; - }; - - /** - * A variant of the DocumentInFlightWhileNotInLock where the _mutex should be held while it - * has a document contained within it. - */ - class DocumentInFlightWithLock { - public: - DocumentInFlightWithLock(WithLock, CloneList& clonerList); - DocumentInFlightWithLock(const DocumentInFlightWithLock&) = delete; - DocumentInFlightWithLock(DocumentInFlightWithLock&&) = default; - - void setDoc(boost::optional<Snapshotted<BSONObj>> doc); - - /** - * Releases the contained document. Can only be called once for the entire lifetime - * of this object. - */ - std::unique_ptr<DocumentInFlightWhileNotInLock> release(); - - private: - std::unique_ptr<InProgressReadToken> _inProgressReadToken; - boost::optional<Snapshotted<BSONObj>> _doc; - }; - - CloneList(); - - /** - * Overwrites the list of record ids to clone. - */ - void populateList(RecordIdSet recordIds); - - /** - * Returns a document to clone. If there are no more documents left to clone, - * DocumentInFlightWhileNotInLock::getDoc will return boost::none. - * - * numRecordsNoLonger exists is an optional parameter that can be used to track - * the number of recordIds encountered that refers to a document that no longer - * exists. - */ - std::unique_ptr<DocumentInFlightWhileNotInLock> getNextDoc(OperationContext* opCtx, - const CollectionPtr& collection, - int* numRecordsNoLongerExist); - - /** - * Put back a document previously obtained from this CloneList instance to the overflow - * pool. - */ - void insertOverflowDoc(Snapshotted<BSONObj> doc); - - /** - * Returns true if there are more documents to clone. - */ - bool hasMore() const; - - /** - * Returns the size of the populated record ids. - */ - size_t size() const; - - private: - /** - * Increments the counter for inProgressReads. - */ - void _startedOneInProgressRead(WithLock); - - /** - * Decrements the counter for inProgressReads. - */ - void _finishedOneInProgressRead(); - - mutable Mutex _mutex = MONGO_MAKE_LATCH("MigrationChunkClonerSource::CloneList::_mutex"); - - RecordIdSet _recordIds; - - // This iterator is a pointer into the _recordIds set. It allows concurrent access to - // the _recordIds set by allowing threads servicing _migrateClone requests to do the - // following: - // 1. Acquire mutex "_mutex" above. - // 2. Copy *_recordIdsIter into its local stack frame. - // 3. Increment _recordIdsIter - // 4. Unlock "_mutex." - // 5. Do the I/O to fetch the document corresponding to this record Id. - // - // The purpose of this algorithm, is to allow different threads to concurrently start I/O - // jobs in order to more fully saturate the disk. - // - // One issue with this algorithm, is that only 16MB worth of documents can be returned in - // response to a _migrateClone request. But, the thread does not know the size of a - // document until it does the I/O. At which point, if the document does not fit in the - // response to _migrateClone request the document must be made available to a different - // thread servicing a _migrateClone request. To solve this problem, the thread adds the - // document to the below _overflowDocs deque. - RecordIdSet::iterator _recordIdsIter; - - // This deque stores all documents that must be sent to the destination, but could not fit - // in the response to a particular _migrateClone request. - std::deque<Snapshotted<BSONObj>> _overflowDocs; - - // This integer represents how many documents are being "held" by threads servicing - // _migrateClone requests. Any document that is "held" by a thread may be added to the - // _overflowDocs deque if it doesn't fit in the response to a _migrateClone request. - // This integer is necessary because it gives us a condition on when all documents to be - // sent to the destination have been exhausted. - // - // If (_recordIdsIter == _recordIds.end() && _overflowDocs.empty() && - // _inProgressReads == 0) then all documents have been returned to the destination. - RecordIdSet::size_type _inProgressReads = 0; - - // This condition variable allows us to wait on the following condition: - // Either we're done and the above condition is satisfied, or there is some document to - // return. - stdx::condition_variable _moreDocsCV; - }; - // Represents the states in which the cloner can be enum State { kNew, kCloning, kDone }; @@ -486,23 +322,6 @@ private: */ Status _checkRecipientCloningStatus(OperationContext* opCtx, Milliseconds maxTimeToWait); - /** - * Inspects the pre and post image document keys and determines which xferMods bucket to - * add a new entry. Returns false if neither pre or post image document keys fall into - * the chunk boundaries being migrated. - */ - bool _processUpdateForXferMod(const BSONObj& preImageDocKey, const BSONObj& postImageDocKey); - - /** - * Defer processing of update ops into xferMods entries to when nextModsBatch is called. - */ - void _deferProcessingForXferMod(const BSONObj& preImageDocKey); - - /** - * Converts all deferred update ops captured by the op observer into xferMods entries. - */ - void _processDeferredXferMods(OperationContext* opCtx, Database* database); - // The original move range request const ShardsvrMoveRange _args; @@ -529,10 +348,8 @@ private: // The current state of the cloner State _state{kNew}; - CloneList _cloneList; - - RecordIdSet::size_type _numRecordsCloned{0}; - RecordIdSet::size_type _numRecordsPassedOver{0}; + // List of record ids that needs to be transferred (initial clone) + std::set<RecordId> _cloneLocs; // The estimated average object size during the clone phase. Used for buffer size // pre-allocation (initial clone). @@ -564,13 +381,6 @@ private: // Amount of delete xfer mods that have not yet reached the recipient. size_t _untransferredDeletesCounter{0}; - // Amount of ops that are yet to be converted to update/delete xferMods. - size_t _deferredUntransferredOpsCounter{0}; - - // Stores document keys of document that needs to be examined if we need to put in to xferMods - // list later. - std::vector<BSONObj> _deferredReloadOrDeletePreImageDocKeys; - // Total bytes in _reload + _deleted (xfer mods) uint64_t _memoryUsed{0}; diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp index 0e7e082f448..713303693e5 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp @@ -37,7 +37,7 @@ #include "mongo/db/auth/privilege.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/commands.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/s/active_migrations_registry.h" #include "mongo/db/s/collection_sharding_runtime.h" @@ -74,11 +74,6 @@ public: str::stream() << "Collection " << nss->ns() << " does not exist", _autoColl->getCollection()); - uassert(ErrorCodes::NotWritablePrimary, - "No longer primary when trying to acquire active migrate cloner", - opCtx->writesAreReplicated() && - repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(opCtx, *nss)); - { auto csr = CollectionShardingRuntime::get(opCtx, *nss); auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp index 776e620f072..91e1b4a21bc 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp @@ -66,477 +66,6 @@ const ConnectionString kRecipientConnStr = HostAndPort("RecipientHost2:1234"), HostAndPort("RecipientHost3:1234")}); -class CollectionWithFault : public Collection { -public: - CollectionWithFault(const Collection* originalCollection) : _coll(originalCollection) {} - - void setFindDocStatus(Status newStatus) { - _findDocStatus = newStatus; - } - - ////////////////////////////////////////////////////////////////////////////////// - // Collection overrides - - std::shared_ptr<Collection> clone() const override { - return _coll->clone(); - } - - SharedCollectionDecorations* getSharedDecorations() const override { - return _coll->getSharedDecorations(); - } - - const NamespaceString& ns() const override { - return _coll->ns(); - } - - Status rename(OperationContext* opCtx, const NamespaceString& nss, bool stayTemp) override { - MONGO_UNREACHABLE; - } - - RecordId getCatalogId() const override { - return _coll->getCatalogId(); - } - - UUID uuid() const override { - return _coll->uuid(); - } - - const IndexCatalog* getIndexCatalog() const override { - return _coll->getIndexCatalog(); - } - - IndexCatalog* getIndexCatalog() override { - MONGO_UNREACHABLE; - } - - RecordStore* getRecordStore() const { - return _coll->getRecordStore(); - } - - std::shared_ptr<Ident> getSharedIdent() const override { - return _coll->getSharedIdent(); - } - - const BSONObj getValidatorDoc() const override { - return _coll->getValidatorDoc(); - } - - std::pair<SchemaValidationResult, Status> checkValidation( - OperationContext* opCtx, const BSONObj& document) const override { - return _coll->checkValidation(opCtx, document); - } - - bool requiresIdIndex() const override { - return _coll->requiresIdIndex(); - } - - Snapshotted<BSONObj> docFor(OperationContext* opCtx, RecordId loc) const override { - return _coll->docFor(opCtx, loc); - } - - bool findDoc(OperationContext* opCtx, RecordId loc, Snapshotted<BSONObj>* out) const override { - uassertStatusOK(_findDocStatus); - return _coll->findDoc(opCtx, loc, out); - } - - std::unique_ptr<SeekableRecordCursor> getCursor(OperationContext* opCtx, - bool forward = true) const override { - return _coll->getCursor(opCtx, forward); - } - - bool updateWithDamagesSupported() const override { - return _coll->updateWithDamagesSupported(); - } - - Status truncate(OperationContext* opCtx) override { - MONGO_UNREACHABLE; - } - - Validator parseValidator(OperationContext* opCtx, - const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures, - boost::optional<multiversion::FeatureCompatibilityVersion> - maxFeatureCompatibilityVersion) const override { - return _coll->parseValidator( - opCtx, validator, allowedFeatures, maxFeatureCompatibilityVersion); - } - - void setValidator(OperationContext* opCtx, Validator validator) override { - MONGO_UNREACHABLE; - } - - Status setValidationLevel(OperationContext* opCtx, ValidationLevelEnum newLevel) override { - MONGO_UNREACHABLE; - } - - Status setValidationAction(OperationContext* opCtx, ValidationActionEnum newAction) override { - MONGO_UNREACHABLE; - } - - boost::optional<ValidationLevelEnum> getValidationLevel() const override { - return _coll->getValidationLevel(); - } - - boost::optional<ValidationActionEnum> getValidationAction() const override { - return _coll->getValidationAction(); - } - - Status updateValidator(OperationContext* opCtx, - BSONObj newValidator, - boost::optional<ValidationLevelEnum> newLevel, - boost::optional<ValidationActionEnum> newAction) override { - MONGO_UNREACHABLE; - } - - Status checkValidatorAPIVersionCompatability(OperationContext* opCtx) const override { - return _coll->checkValidatorAPIVersionCompatability(opCtx); - } - - bool isChangeStreamPreAndPostImagesEnabled() const override { - return _coll->isChangeStreamPreAndPostImagesEnabled(); - } - - void setChangeStreamPreAndPostImages(OperationContext* opCtx, - ChangeStreamPreAndPostImagesOptions val) override { - MONGO_UNREACHABLE; - } - - bool isTemporary() const override { - return _coll->isTemporary(); - } - - boost::optional<bool> getTimeseriesBucketsMayHaveMixedSchemaData() const override { - return _coll->getTimeseriesBucketsMayHaveMixedSchemaData(); - } - - void setTimeseriesBucketsMayHaveMixedSchemaData(OperationContext* opCtx, - boost::optional<bool> setting) override { - MONGO_UNREACHABLE; - } - - StatusWith<bool> doesTimeseriesBucketsDocContainMixedSchemaData( - const BSONObj& bucketsDoc) const override { - return _coll->doesTimeseriesBucketsDocContainMixedSchemaData(bucketsDoc); - } - - bool getRequiresTimeseriesExtendedRangeSupport() const override { - return _coll->getRequiresTimeseriesExtendedRangeSupport(); - } - - void setRequiresTimeseriesExtendedRangeSupport(OperationContext* opCtx) const override { - return _coll->setRequiresTimeseriesExtendedRangeSupport(opCtx); - } - - bool isClustered() const override { - return _coll->isClustered(); - } - - boost::optional<ClusteredCollectionInfo> getClusteredInfo() const override { - return _coll->getClusteredInfo(); - } - - void updateClusteredIndexTTLSetting(OperationContext* opCtx, - boost::optional<int64_t> expireAfterSeconds) override { - MONGO_UNREACHABLE; - } - - Status updateCappedSize(OperationContext* opCtx, - boost::optional<long long> newCappedSize, - boost::optional<long long> newCappedMax) override { - MONGO_UNREACHABLE; - } - - StatusWith<int> checkMetaDataForIndex(const std::string& indexName, - const BSONObj& spec) const override { - return _coll->checkMetaDataForIndex(indexName, spec); - } - - void updateTTLSetting(OperationContext* opCtx, - StringData idxName, - long long newExpireSeconds) override { - MONGO_UNREACHABLE; - } - - void updateHiddenSetting(OperationContext* opCtx, StringData idxName, bool hidden) override { - MONGO_UNREACHABLE; - } - - void updateUniqueSetting(OperationContext* opCtx, StringData idxName, bool unique) override { - MONGO_UNREACHABLE; - } - - void updatePrepareUniqueSetting(OperationContext* opCtx, - StringData idxName, - bool prepareUnique) override { - MONGO_UNREACHABLE; - } - - void setIsTemp(OperationContext* opCtx, bool isTemp) override { - MONGO_UNREACHABLE; - } - - void removeIndex(OperationContext* opCtx, StringData indexName) override { - MONGO_UNREACHABLE; - } - - Status prepareForIndexBuild(OperationContext* opCtx, - const IndexDescriptor* spec, - boost::optional<UUID> buildUUID, - bool isBackgroundSecondaryBuild) override { - MONGO_UNREACHABLE; - } - - boost::optional<UUID> getIndexBuildUUID(StringData indexName) const override { - return _coll->getIndexBuildUUID(indexName); - } - - bool isIndexMultikey(OperationContext* opCtx, - StringData indexName, - MultikeyPaths* multikeyPaths, - int indexOffset = -1) const override { - return _coll->isIndexMultikey(opCtx, indexName, multikeyPaths, indexOffset); - } - - bool setIndexIsMultikey(OperationContext* opCtx, - StringData indexName, - const MultikeyPaths& multikeyPaths, - int indexOffset = -1) const override { - return _coll->setIndexIsMultikey(opCtx, indexName, multikeyPaths, indexOffset); - } - - void forceSetIndexIsMultikey(OperationContext* opCtx, - const IndexDescriptor* desc, - bool isMultikey, - const MultikeyPaths& multikeyPaths) const override { - return _coll->forceSetIndexIsMultikey(opCtx, desc, isMultikey, multikeyPaths); - } - - int getTotalIndexCount() const override { - return _coll->getTotalIndexCount(); - } - - int getCompletedIndexCount() const override { - return _coll->getCompletedIndexCount(); - } - - BSONObj getIndexSpec(StringData indexName) const override { - return _coll->getIndexSpec(indexName); - } - - void getAllIndexes(std::vector<std::string>* names) const override { - return _coll->getAllIndexes(names); - } - - void getReadyIndexes(std::vector<std::string>* names) const override { - return _coll->getReadyIndexes(names); - } - - bool isIndexPresent(StringData indexName) const override { - return _coll->isIndexPresent(indexName); - } - - bool isIndexReady(StringData indexName) const override { - return _coll->isIndexReady(indexName); - } - - void replaceMetadata(OperationContext* opCtx, - std::shared_ptr<BSONCollectionCatalogEntry::MetaData> md) override { - MONGO_UNREACHABLE; - } - - bool isCapped() const override { - return _coll->isCapped(); - } - - long long getCappedMaxDocs() const override { - return _coll->getCappedMaxDocs(); - } - - long long getCappedMaxSize() const override { - return _coll->getCappedMaxSize(); - } - - long long numRecords(OperationContext* opCtx) const override { - return _coll->numRecords(opCtx); - } - - long long dataSize(OperationContext* opCtx) const override { - return _coll->dataSize(opCtx); - } - - bool isEmpty(OperationContext* opCtx) const override { - return _coll->isEmpty(opCtx); - } - - int averageObjectSize(OperationContext* opCtx) const override { - return _coll->averageObjectSize(opCtx); - } - - uint64_t getIndexSize(OperationContext* opCtx, - BSONObjBuilder* details = nullptr, - int scale = 1) const { - return _coll->getIndexSize(opCtx, details, scale); - } - - uint64_t getIndexFreeStorageBytes(OperationContext* opCtx) const override { - return _coll->getIndexFreeStorageBytes(opCtx); - } - - boost::optional<Timestamp> getMinimumVisibleSnapshot() const override { - return _coll->getMinimumVisibleSnapshot(); - } - - void setMinimumVisibleSnapshot(Timestamp name) override { - MONGO_UNREACHABLE; - } - - boost::optional<TimeseriesOptions> getTimeseriesOptions() const override { - return _coll->getTimeseriesOptions(); - } - - void setTimeseriesOptions(OperationContext* opCtx, - const TimeseriesOptions& tsOptions) override { - MONGO_UNREACHABLE; - } - - const CollatorInterface* getDefaultCollator() const override { - return _coll->getDefaultCollator(); - } - - const CollectionOptions& getCollectionOptions() const override { - return _coll->getCollectionOptions(); - } - - StatusWith<std::vector<BSONObj>> addCollationDefaultsToIndexSpecsForCreate( - OperationContext* opCtx, const std::vector<BSONObj>& indexSpecs) const { - return _coll->addCollationDefaultsToIndexSpecsForCreate(opCtx, indexSpecs); - } - - void indexBuildSuccess(OperationContext* opCtx, IndexCatalogEntry* index) override { - MONGO_UNREACHABLE; - } - - void onDeregisterFromCatalog(OperationContext* opCtx) override { - MONGO_UNREACHABLE; - } - - void deleteDocument(OperationContext* opCtx, - StmtId stmtId, - RecordId loc, - OpDebug* opDebug, - bool fromMigrate = false, - bool noWarn = false, - StoreDeletedDoc storeDeletedDoc = StoreDeletedDoc::Off, - CheckRecordId checkRecordId = CheckRecordId::Off) const override { - MONGO_UNREACHABLE; - } - - void deleteDocument(OperationContext* opCtx, - Snapshotted<BSONObj> doc, - StmtId stmtId, - RecordId loc, - OpDebug* opDebug, - bool fromMigrate = false, - bool noWarn = false, - StoreDeletedDoc storeDeletedDoc = StoreDeletedDoc::Off, - CheckRecordId checkRecordId = CheckRecordId::Off) const override { - MONGO_UNREACHABLE; - } - - Status insertDocuments(OperationContext* opCtx, - std::vector<InsertStatement>::const_iterator begin, - std::vector<InsertStatement>::const_iterator end, - OpDebug* opDebug, - bool fromMigrate = false) const override { - MONGO_UNREACHABLE; - } - - Status insertDocument(OperationContext* opCtx, - const InsertStatement& doc, - OpDebug* opDebug, - bool fromMigrate = false) const override { - MONGO_UNREACHABLE; - } - - Status insertDocumentsForOplog(OperationContext* opCtx, - std::vector<Record>* records, - const std::vector<Timestamp>& timestamps) const override { - MONGO_UNREACHABLE; - } - - Status insertDocumentForBulkLoader(OperationContext* opCtx, - const BSONObj& doc, - const OnRecordInsertedFn& onRecordInserted) const override { - MONGO_UNREACHABLE; - } - - RecordId updateDocument(OperationContext* opCtx, - RecordId oldLocation, - const Snapshotted<BSONObj>& oldDoc, - const BSONObj& newDoc, - bool indexesAffected, - OpDebug* opDebug, - CollectionUpdateArgs* args) const override { - MONGO_UNREACHABLE; - } - - StatusWith<RecordData> updateDocumentWithDamages(OperationContext* opCtx, - RecordId loc, - const Snapshotted<RecordData>& oldRec, - const char* damageSource, - const mutablebson::DamageVector& damages, - CollectionUpdateArgs* args) const { - MONGO_UNREACHABLE; - } - - void cappedTruncateAfter(OperationContext* opCtx, RecordId end, bool inclusive) const override { - MONGO_UNREACHABLE; - } - - bool getRecordPreImages() const override { - return _coll->getRecordPreImages(); - } - - void setRecordPreImages(OperationContext* opCtx, bool val) override { - MONGO_UNREACHABLE; - } - - std::vector<std::string> repairInvalidIndexOptions(OperationContext* opCtx) override { - MONGO_UNREACHABLE; - } - - CappedCallback* getCappedCallback() override { - MONGO_UNREACHABLE; - } - - const CappedCallback* getCappedCallback() const override { - return _coll->getCappedCallback(); - } - - std::shared_ptr<CappedInsertNotifier> getCappedInsertNotifier() const override { - return _coll->getCappedInsertNotifier(); - } - - std::unique_ptr<PlanExecutor, PlanExecutor::Deleter> makePlanExecutor( - OperationContext* opCtx, - const CollectionPtr& yieldableCollection, - PlanYieldPolicy::YieldPolicy yieldPolicy, - ScanDirection scanDirection, - boost::optional<RecordId> resumeAfterRecordId = boost::none) const override { - return _coll->makePlanExecutor( - opCtx, yieldableCollection, yieldPolicy, scanDirection, resumeAfterRecordId); - } - - void establishOplogCollectionForLogging(OperationContext* opCtx) const override { - return _coll->establishOplogCollectionForLogging(opCtx); - } - -private: - const Collection* _coll; - - Status _findDocStatus{Status::OK()}; -}; - class MigrationChunkClonerSourceLegacyTest : public ShardServerTestFixture { protected: MigrationChunkClonerSourceLegacyTest() : ShardServerTestFixture(Options{}.useMockClock(true)) {} @@ -598,29 +127,9 @@ protected: if (docs.empty()) return; - std::deque<BSONObj> docsToInsert; - std::copy(docs.cbegin(), docs.cend(), std::back_inserter(docsToInsert)); - - while (!docsToInsert.empty()) { - std::vector<BSONObj> batchToInsert; - - size_t sizeInBatch = 0; - while (!docsToInsert.empty()) { - auto next = docsToInsert.front(); - sizeInBatch += next.objsize(); - - if (sizeInBatch > BSONObjMaxUserSize) { - break; - } - - batchToInsert.push_back(next); - docsToInsert.pop_front(); - } - - auto response = client()->insertAcknowledged(kNss.ns(), batchToInsert); - ASSERT_OK(getStatusFromWriteCommandReply(response)); - ASSERT_GT(response["n"].Int(), 0); - } + auto response = client()->insertAcknowledged(kNss.ns(), docs); + ASSERT_OK(getStatusFromWriteCommandReply(response)); + ASSERT_GT(response["n"].Int(), 0); } void deleteDocsInShardedCollection(BSONObj query) { @@ -738,9 +247,7 @@ private: StaticCatalogClient() = default; StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { ShardType donorShard; donorShard.setName(kDonorConnStr.getSetName()); @@ -765,7 +272,6 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetched) { createCollectionDocument(100), createCollectionDocument(199), createCollectionDocument(200)}; - const ShardKeyPattern shardKeyPattern(kShardKeyPattern); createShardedCollection(contents); @@ -830,18 +336,9 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetched) { cloner.onInsertOp(operationContext(), createCollectionDocument(151), {}); cloner.onInsertOp(operationContext(), createCollectionDocument(210), {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(shardKeyPattern, createCollectionDocument(80)), - {}, - {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(shardKeyPattern, createCollectionDocument(199)), - {}, - {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(shardKeyPattern, createCollectionDocument(220)), - {}, - {}); + cloner.onDeleteOp(operationContext(), createCollectionDocument(80), {}, {}); + cloner.onDeleteOp(operationContext(), createCollectionDocument(199), {}, {}); + cloner.onDeleteOp(operationContext(), createCollectionDocument(220), {}, {}); wuow.commit(); } @@ -865,8 +362,12 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetched) { ASSERT_BSONOBJ_EQ(createCollectionDocument(150), modsObj["reload"].Array()[0].Obj()); ASSERT_BSONOBJ_EQ(createCollectionDocument(151), modsObj["reload"].Array()[1].Obj()); - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 199), modsObj["deleted"].Array()[0].Obj()); + // The legacy chunk cloner cannot filter out deletes because we don't preserve the shard + // key on delete + ASSERT_EQ(3U, modsObj["deleted"].Array().size()); + ASSERT_BSONOBJ_EQ(BSON("_id" << 80), modsObj["deleted"].Array()[0].Obj()); + ASSERT_BSONOBJ_EQ(BSON("_id" << 199), modsObj["deleted"].Array()[1].Obj()); + ASSERT_BSONOBJ_EQ(BSON("_id" << 220), modsObj["deleted"].Array()[2].Obj()); } } @@ -882,7 +383,6 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetched) { TEST_F(MigrationChunkClonerSourceLegacyTest, RemoveDuplicateDocuments) { const std::vector<BSONObj> contents = {createCollectionDocument(100), createCollectionDocument(199)}; - const ShardKeyPattern shardKeyPattern(kShardKeyPattern); createShardedCollection(contents); @@ -933,15 +433,9 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, RemoveDuplicateDocuments) { WriteUnitOfWork wuow(operationContext()); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(shardKeyPattern, createCollectionDocument(100)), - {}, - {}); + cloner.onDeleteOp(operationContext(), createCollectionDocument(100), {}, {}); cloner.onInsertOp(operationContext(), createCollectionDocument(100), {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(shardKeyPattern, createCollectionDocument(100)), - {}, - {}); + cloner.onDeleteOp(operationContext(), createCollectionDocument(100), {}, {}); cloner.onUpdateOp(operationContext(), createCollectionDocument(199), @@ -1207,510 +701,5 @@ TEST_F(MigrationChunkClonerSourceLegacyTest, FailedToEngageRecipientShard) { cloner.cancelClone(operationContext()); } -TEST_F(MigrationChunkClonerSourceLegacyTest, CloneFetchThatOverflows) { - const auto kBigSize = 10 * 1024 * 1024; - const std::vector<BSONObj> contents = {createSizedCollectionDocument(100, kBigSize), - createSizedCollectionDocument(120, kBigSize), - createSizedCollectionDocument(199, kBigSize)}; - - createShardedCollection(contents); - - ShardsvrMoveRange req = createMoveRangeRequest(ChunkRange(BSON("X" << 100), BSON("X" << 200))); - req.setMaxChunkSizeBytes(64 * 1024 * 1024); - - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - kShardKeyPattern, - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - { - auto futureStartClone = launchAsync([&]() { - onCommand([&](const RemoteCommandRequest& request) { return BSON("ok" << true); }); - }); - - ASSERT_OK(cloner.startClone(operationContext(), UUID::gen(), _lsid, _txnNumber)); - futureStartClone.default_timed_get(); - } - - // Ensure the initial clone documents are available - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(1, arrBuilder.arrSize()); - - const auto arr = arrBuilder.arr(); - ASSERT_BSONOBJ_EQ(contents[0], arr[0].Obj()); - } - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(1, arrBuilder.arrSize()); - - const auto arr = arrBuilder.arr(); - ASSERT_BSONOBJ_EQ(contents[1], arr[0].Obj()); - } - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(1, arrBuilder.arrSize()); - - const auto arr = arrBuilder.arr(); - ASSERT_BSONOBJ_EQ(contents[2], arr[0].Obj()); - } - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - } - - auto futureCommit = launchAsync([&]() { - onCommand([&](const RemoteCommandRequest& request) { return BSON("ok" << true); }); - }); - - ASSERT_OK(cloner.commitClone(operationContext(), true /* acquireCSOnRecipient */)); - futureCommit.default_timed_get(); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, CloneShouldNotCrashWhenNextCloneBatchThrows) { - const std::vector<BSONObj> contents = {createCollectionDocument(100), - createCollectionDocument(150), - createCollectionDocument(199)}; - - createShardedCollection(contents); - - const ShardsvrMoveRange req = - createMoveRangeRequest(ChunkRange(BSON("X" << 100), BSON("X" << 200))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - kShardKeyPattern, - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - { - auto futureStartClone = launchAsync([&]() { - onCommand([&](const RemoteCommandRequest& request) { return BSON("ok" << true); }); - }); - - ASSERT_OK(cloner.startClone(operationContext(), UUID::gen(), _lsid, _txnNumber)); - futureStartClone.default_timed_get(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - auto collWithFault = - std::make_unique<CollectionWithFault>(autoColl.getCollection().get()); - CollectionPtr collPtrWithFault(collWithFault.get(), CollectionPtr::NoYieldTag()); - - // Note: findDoc currently doesn't have any interruption points, this test simulates - // an exception being thrown while it is being called. - collWithFault->setFindDocStatus({ErrorCodes::Interrupted, "fake interrupt"}); - - BSONArrayBuilder arrBuilder; - - ASSERT_THROWS_CODE( - cloner.nextCloneBatch(operationContext(), collPtrWithFault, &arrBuilder), - DBException, - ErrorCodes::Interrupted); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - // The first document was lost and returned an error during nextCloneBatch. This would - // cause the migration destination to abort, but it is still possible for other - // threads to be in the middle of calling nextCloneBatch and the next nextCloneBatch - // calls simulate calls from other threads after the first call threw. - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - - const auto arr = arrBuilder.arr(); - ASSERT_EQ(2, arrBuilder.arrSize()); - - ASSERT_BSONOBJ_EQ(contents[1], arr[0].Obj()); - ASSERT_BSONOBJ_EQ(contents[2], arr[1].Obj()); - } - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - - const auto arr = arrBuilder.arr(); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - } - - auto futureCommit = launchAsync([&]() { - // Simulate destination returning an error. - onCommand([&](const RemoteCommandRequest& request) { return BSON("ok" << false); }); - - // This is the return response for recvChunkAbort. - onCommand([&](const RemoteCommandRequest& request) { return BSON("ok" << true); }); - }); - - ASSERT_NOT_OK(cloner.commitClone(operationContext(), true /* acquireCSOnRecipient */)); - futureCommit.default_timed_get(); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetchedWithDottedShardKeyPattern) { - const ShardKeyPattern dottedShardKeyPattern(BSON("x.a" << 1 << "x.b" << 1)); - - auto createDoc = - ([](int val) { return BSON("_id" << val << "x" << BSON("a" << val << "b" << val)); }); - - const ShardsvrMoveRange req = createMoveRangeRequest( - ChunkRange(BSON("x.a" << 100 << "x.b" << 100), BSON("x.a" << 200 << "x.b" << 200))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - dottedShardKeyPattern.toBSON(), - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - // Insert some documents in the chunk range to be included for migration - insertDocsInShardedCollection({createDoc(150)}); - insertDocsInShardedCollection({createDoc(151)}); - - // Insert some documents which are outside of the chunk range and should not be included for - // migration - insertDocsInShardedCollection({createDoc(90)}); - insertDocsInShardedCollection({createDoc(210)}); - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IX); - - WriteUnitOfWork wuow(operationContext()); - - cloner.onInsertOp(operationContext(), createDoc(90), {}); - cloner.onInsertOp(operationContext(), createDoc(150), {}); - cloner.onInsertOp(operationContext(), createDoc(151), {}); - cloner.onInsertOp(operationContext(), createDoc(210), {}); - - cloner.onDeleteOp( - operationContext(), repl::getDocumentKey(dottedShardKeyPattern, createDoc(80)), {}, {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(dottedShardKeyPattern, createDoc(199)), - {}, - {}); - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(dottedShardKeyPattern, createDoc(220)), - {}, - {}); - - wuow.commit(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - { - BSONObjBuilder modsBuilder; - ASSERT_OK(cloner.nextModsBatch(operationContext(), autoColl.getDb(), &modsBuilder)); - - const auto modsObj = modsBuilder.obj(); - ASSERT_EQ(2U, modsObj["reload"].Array().size()); - ASSERT_BSONOBJ_EQ(createDoc(150), modsObj["reload"].Array()[0].Obj()); - ASSERT_BSONOBJ_EQ(createDoc(151), modsObj["reload"].Array()[1].Obj()); - - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 199), modsObj["deleted"].Array()[0].Obj()); - } - } - - cloner.cancelClone(operationContext()); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, CorrectDocumentsFetchedWithHasheddShardKeyPattern) { - const ShardKeyPattern hashedShardKeyPattern(BSON("X" - << "hashed")); - - const ShardsvrMoveRange req = createMoveRangeRequest( - ChunkRange(BSON("X" << 6000000000000000000ll), BSON("X" << 9003000000000000000ll))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - hashedShardKeyPattern.toBSON(), - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - // Insert some documents in the chunk range to be included for migration - insertDocsInShardedCollection({createCollectionDocument(150)}); - insertDocsInShardedCollection({createCollectionDocument(151)}); - - // Insert some documents which are outside of the chunk range and should not be included for - // migration - insertDocsInShardedCollection({createCollectionDocument(90)}); - insertDocsInShardedCollection({createCollectionDocument(210)}); - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IX); - - WriteUnitOfWork wuow(operationContext()); - - cloner.onInsertOp( - operationContext(), createCollectionDocument(90), {}); // hashed = 1348713528393582036 - cloner.onInsertOp( - operationContext(), createCollectionDocument(150), {}); // hashed = 9002984030040611364 - cloner.onInsertOp( - operationContext(), createCollectionDocument(151), {}); // hashed = 6186237390842619770 - cloner.onInsertOp( - operationContext(), createCollectionDocument(210), {}); // hashed = 4420792252088815836 - - cloner.onDeleteOp(operationContext(), - repl::getDocumentKey(hashedShardKeyPattern, createCollectionDocument(80)), - {}, - {}); // hashed = 6910253216116676730 - cloner.onDeleteOp( - operationContext(), - repl::getDocumentKey(hashedShardKeyPattern, createCollectionDocument(199)), - {}, - {}); // hashed = 3000073935277689405 - cloner.onDeleteOp( - operationContext(), - repl::getDocumentKey(hashedShardKeyPattern, createCollectionDocument(220)), - {}, - {}); // hashed = -6432749148213749320 - - wuow.commit(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - { - BSONObjBuilder modsBuilder; - ASSERT_OK(cloner.nextModsBatch(operationContext(), autoColl.getDb(), &modsBuilder)); - - const auto modsObj = modsBuilder.obj(); - ASSERT_EQ(2U, modsObj["reload"].Array().size()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(150), modsObj["reload"].Array()[0].Obj()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(151), modsObj["reload"].Array()[1].Obj()); - - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 80), modsObj["deleted"].Array()[0].Obj()); - } - } - - cloner.cancelClone(operationContext()); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, UpdatedDocumentsFetched) { - const ShardKeyPattern shardKeyPattern(kShardKeyPattern); - - const ShardsvrMoveRange req = - createMoveRangeRequest(ChunkRange(BSON("X" << 100), BSON("X" << 200))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - kShardKeyPattern, - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - // Insert some documents in the chunk range to be included for migration - insertDocsInShardedCollection({createCollectionDocument(150)}); - insertDocsInShardedCollection({createCollectionDocument(151)}); - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IX); - - WriteUnitOfWork wuow(operationContext()); - - cloner.onUpdateOp(operationContext(), boost::none, createCollectionDocument(150), {}, {}); - cloner.onUpdateOp(operationContext(), - createCollectionDocument(80), - createCollectionDocument(151), - {}, - {}); - - // From in doc in chunk range to outside of range will be converted to a delete xferMods. - cloner.onUpdateOp(operationContext(), - createCollectionDocument(199), - createCollectionDocument(90), - {}, - {}); - - wuow.commit(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - { - BSONObjBuilder modsBuilder; - ASSERT_OK(cloner.nextModsBatch(operationContext(), autoColl.getDb(), &modsBuilder)); - - const auto modsObj = modsBuilder.obj(); - ASSERT_EQ(2U, modsObj["reload"].Array().size()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(150), modsObj["reload"].Array()[0].Obj()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(151), modsObj["reload"].Array()[1].Obj()); - - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 199), modsObj["deleted"].Array()[0].Obj()); - } - } - - cloner.cancelClone(operationContext()); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, UpdatedDocumentsFetchedWithHashedShardKey) { - const ShardKeyPattern shardKeyPattern(BSON("X" - << "hashed")); - - const ShardsvrMoveRange req = createMoveRangeRequest( - ChunkRange(BSON("X" << 6000000000000000000ll), BSON("X" << 9003000000000000000ll))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - shardKeyPattern.toBSON(), - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - // Insert some documents in the chunk range to be included for migration - insertDocsInShardedCollection({createCollectionDocument(150)}); - insertDocsInShardedCollection({createCollectionDocument(151)}); - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IX); - - WriteUnitOfWork wuow(operationContext()); - - cloner.onUpdateOp(operationContext(), - boost::none, - createCollectionDocument(150), - {}, - {}); // hashed = 9002984030040611364 - cloner.onUpdateOp(operationContext(), - createCollectionDocument(90), - createCollectionDocument(151), - {}, - {}); // hashed = 1348713528393582036 -> 6186237390842619770 - - // From in doc in chunk range to outside of range will be converted to a delete xferMods. - cloner.onUpdateOp(operationContext(), - createCollectionDocument(80), - createCollectionDocument(199), - {}, - {}); // hashed = 6910253216116676730 -> 3000073935277689405 - - wuow.commit(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - { - BSONObjBuilder modsBuilder; - ASSERT_OK(cloner.nextModsBatch(operationContext(), autoColl.getDb(), &modsBuilder)); - - const auto modsObj = modsBuilder.obj(); - ASSERT_EQ(2U, modsObj["reload"].Array().size()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(150), modsObj["reload"].Array()[0].Obj()); - ASSERT_BSONOBJ_EQ(createCollectionDocument(151), modsObj["reload"].Array()[1].Obj()); - - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 80), modsObj["deleted"].Array()[0].Obj()); - } - } - - cloner.cancelClone(operationContext()); -} - -TEST_F(MigrationChunkClonerSourceLegacyTest, UpdatedDocumentsFetchedWithDottedShardKeyPattern) { - const ShardKeyPattern dottedShardKeyPattern(BSON("x.a" << 1 << "x.b" << 1)); - - auto createDoc = - ([](int val) { return BSON("_id" << val << "x" << BSON("a" << val << "b" << val)); }); - - const ShardsvrMoveRange req = createMoveRangeRequest( - ChunkRange(BSON("x.a" << 100 << "x.b" << 100), BSON("x.a" << 200 << "x.b" << 200))); - MigrationChunkClonerSourceLegacy cloner(req, - WriteConcernOptions(), - dottedShardKeyPattern.toBSON(), - kDonorConnStr, - kRecipientConnStr.getServers()[0]); - - // Insert some documents in the chunk range to be included for migration - insertDocsInShardedCollection({createDoc(150)}); - insertDocsInShardedCollection({createDoc(151)}); - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IX); - - WriteUnitOfWork wuow(operationContext()); - - cloner.onUpdateOp(operationContext(), boost::none, createDoc(150), {}, {}); - cloner.onUpdateOp(operationContext(), createDoc(80), createDoc(151), {}, {}); - - // From in doc in chunk range to outside of range will be converted to a delete xferMods. - cloner.onUpdateOp(operationContext(), createDoc(199), createDoc(90), {}, {}); - - wuow.commit(); - } - - { - AutoGetCollection autoColl(operationContext(), kNss, MODE_IS); - - { - BSONArrayBuilder arrBuilder; - ASSERT_OK( - cloner.nextCloneBatch(operationContext(), autoColl.getCollection(), &arrBuilder)); - ASSERT_EQ(0, arrBuilder.arrSize()); - } - - { - BSONObjBuilder modsBuilder; - ASSERT_OK(cloner.nextModsBatch(operationContext(), autoColl.getDb(), &modsBuilder)); - - const auto modsObj = modsBuilder.obj(); - ASSERT_EQ(2U, modsObj["reload"].Array().size()); - ASSERT_BSONOBJ_EQ(createDoc(150), modsObj["reload"].Array()[0].Obj()); - ASSERT_BSONOBJ_EQ(createDoc(151), modsObj["reload"].Array()[1].Obj()); - - ASSERT_EQ(1U, modsObj["deleted"].Array().size()); - ASSERT_BSONOBJ_EQ(BSON("_id" << 199), modsObj["deleted"].Array()[0].Obj()); - } - } - - cloner.cancelClone(operationContext()); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/migration_coordinator.cpp b/src/mongo/db/s/migration_coordinator.cpp index d59de326210..b6ff4b7abd1 100644 --- a/src/mongo/db/s/migration_coordinator.cpp +++ b/src/mongo/db/s/migration_coordinator.cpp @@ -37,7 +37,6 @@ #include "mongo/db/s/migration_util.h" #include "mongo/db/s/range_deletion_task_gen.h" #include "mongo/db/vector_clock.h" -#include "mongo/db/vector_clock_mutable.h" #include "mongo/logv2/log.h" #include "mongo/platform/atomic_word.h" #include "mongo/s/pm2423_feature_flags_gen.h" @@ -173,11 +172,6 @@ boost::optional<SemiFuture<void>> MigrationCoordinator::completeMigration( boost::optional<SemiFuture<void>> cleanupCompleteFuture = boost::none; - // Persist the config time before the migration decision to ensure that in case of stepdown - // next filtering metadata refresh on the new primary will always include the effect of this - // migration. - VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); - switch (*decision) { case DecisionEnum::kAborted: _abortMigrationOnDonorAndRecipient(opCtx, acquireCSOnRecipient); diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp index bbe4f34bf19..04399595453 100644 --- a/src/mongo/db/s/migration_destination_manager.cpp +++ b/src/mongo/db/s/migration_destination_manager.cpp @@ -29,7 +29,6 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kShardingMigration -#include "mongo/db/s/migration_batch_fetcher.h" #include "mongo/platform/basic.h" #include "mongo/db/s/migration_destination_manager.h" @@ -40,7 +39,7 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/cancelable_operation_context.h" #include "mongo/db/catalog/document_validation.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/index/index_descriptor.h" @@ -296,7 +295,6 @@ MONGO_FAIL_POINT_DEFINE(migrateThreadHangAtStep3); MONGO_FAIL_POINT_DEFINE(migrateThreadHangAtStep4); MONGO_FAIL_POINT_DEFINE(migrateThreadHangAtStep5); MONGO_FAIL_POINT_DEFINE(migrateThreadHangAtStep6); -MONGO_FAIL_POINT_DEFINE(migrateThreadHangAfterSteadyTransition); MONGO_FAIL_POINT_DEFINE(migrateThreadHangAtStep7); MONGO_FAIL_POINT_DEFINE(failMigrationOnRecipient); @@ -413,8 +411,8 @@ void MigrationDestinationManager::report(BSONObjBuilder& b, } BSONObjBuilder bb(b.subobjStart("counts")); - bb.append("cloned", _getNumCloned()); - bb.append("clonedBytes", _getNumBytesCloned()); + bb.append("cloned", _numCloned); + bb.append("clonedBytes", _clonedBytes); bb.append("catchup", _numCatchup); bb.append("steady", _numSteady); bb.done(); @@ -448,8 +446,6 @@ Status MigrationDestinationManager::start(OperationContext* opCtx, _lsid = cloneRequest.getLsid(); _txnNumber = cloneRequest.getTxnNumber(); - _parallelFetchersSupported = cloneRequest.parallelFetchingSupported(); - _nss = nss; _fromShard = cloneRequest.getFromShardId(); _fromShardConnString = @@ -466,8 +462,8 @@ Status MigrationDestinationManager::start(OperationContext* opCtx, _chunkMarkedPending = false; - _migrationCloningProgress = std::make_shared<MigrationCloningProgressSharedState>(); - + _numCloned = 0; + _clonedBytes = 0; _numCatchup = 0; _numSteady = 0; @@ -490,9 +486,6 @@ Status MigrationDestinationManager::start(OperationContext* opCtx, _sessionMigration = std::make_unique<SessionCatalogMigrationDestination>( _nss, _fromShard, *_sessionId, _cancellationSource.token()); ShardingStatistics::get(opCtx).countRecipientMoveChunkStarted.addAndFetch(1); - if (mongo::feature_flags::gConcurrencyInChunkMigration.isEnabledAndIgnoreFCV()) - ShardingStatistics::get(opCtx).chunkMigrationConcurrencyCnt.store( - chunkMigrationConcurrency.load()); _migrateThreadHandle = stdx::thread([this, cancellationToken = _cancellationSource.token()]() { _migrateThread(cancellationToken); @@ -840,22 +833,11 @@ MigrationDestinationManager::IndexesAndIdIndex MigrationDestinationManager::getC return {donorIndexSpecs, donorIdIndexSpec}; } - -MigrationDestinationManager::CollectionOptionsAndUUID -MigrationDestinationManager::getCollectionOptions(OperationContext* opCtx, - const NamespaceStringOrUUID& nssOrUUID, - boost::optional<Timestamp> afterClusterTime) { - const auto dbInfo = uassertStatusOK( - Grid::get(opCtx)->catalogCache()->getDatabase(opCtx, nssOrUUID.db().toString())); - return getCollectionOptions( - opCtx, nssOrUUID, dbInfo->getPrimary(), dbInfo->getVersion(), afterClusterTime); -} - MigrationDestinationManager::CollectionOptionsAndUUID MigrationDestinationManager::getCollectionOptions(OperationContext* opCtx, const NamespaceStringOrUUID& nssOrUUID, const ShardId& fromShardId, - const boost::optional<DatabaseVersion>& dbVersion, + const boost::optional<ChunkManager>& cm, boost::optional<Timestamp> afterClusterTime) { auto fromShard = uassertStatusOK(Grid::get(opCtx)->shardRegistry()->getShard(opCtx, fromShardId)); @@ -865,10 +847,9 @@ MigrationDestinationManager::getCollectionOptions(OperationContext* opCtx, auto cmd = nssOrUUID.nss() ? BSON("listCollections" << 1 << "filter" << BSON("name" << nssOrUUID.nss()->coll())) : BSON("listCollections" << 1 << "filter" << BSON("info.uuid" << *nssOrUUID.uuid())); - if (dbVersion) { - cmd = appendDbVersionIfPresent(cmd, *dbVersion); + if (cm) { + cmd = appendDbVersionIfPresent(cmd, cm->dbVersion()); } - if (afterClusterTime) { cmd = cmd.addFields(makeLocalReadConcernWithAfterClusterTime(*afterClusterTime)); } @@ -931,8 +912,11 @@ void MigrationDestinationManager::_dropLocalIndexesIfNecessary( // collection is sharded. if (optMetadata) { const auto& metadata = *optMetadata; - if (metadata.isSharded() && !metadata.currentShardHasAnyChunks()) { - return true; + if (metadata.isSharded()) { + auto chunks = metadata.getChunks(); + if (chunks.empty()) { + return true; + } } } return false; @@ -1047,9 +1031,10 @@ void MigrationDestinationManager::cloneCollectionIndexesAndOptions( << collectionByUUID->ns()); } - // We do not have a collection by this name. Create it with the donor's options. + // We do not have a collection by this name. Create the collection with the donor's + // options. OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE - unsafeCreateCollection(opCtx, /* forceCSRAsUnknownAfterCollectionCreation */ true); + unsafeCreateCollection(opCtx); WriteUnitOfWork wuow(opCtx); CollectionOptions collectionOptions = uassertStatusOK( CollectionOptions::parse(collectionOptionsAndIndexes.options, @@ -1168,20 +1153,12 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, invariant(!_min.isEmpty()); invariant(!_max.isEmpty()); - boost::optional<Timer> timeInCriticalSection; boost::optional<MoveTimingHelper> timing; - mongo::ScopeGuard timingSetMsgGuard{[this, &timing] { - // Set the error message to MoveTimingHelper just before it is destroyed. The destructor - // sends that message (among other things) to the ShardingLogging. - if (timing) { - stdx::lock_guard<Latch> sl(_mutex); - timing->setCmdErrMsg(_errmsg); - } - }}; + boost::optional<Timer> timeInCriticalSection; if (!skipToCritSecTaken) { timing.emplace( - outerOpCtx, "to", _nss.ns(), _min, _max, 8 /* steps */, _toShard, _fromShard); + outerOpCtx, "to", _nss.ns(), _min, _max, 8 /* steps */, &_errmsg, _toShard, _fromShard); LOGV2( 22000, @@ -1361,34 +1338,122 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, _sessionMigration->start(opCtx->getServiceContext()); + const BSONObj migrateCloneRequest = createMigrateCloneRequest(_nss, *_sessionId); + _chunkMarkedPending = true; // no lock needed, only the migrate thread looks. - { - // Destructor of MigrationBatchFetcher is non-trivial. Therefore, - // this scope has semantic significance. - MigrationBatchFetcher<MigrationBatchInserter> fetcher{ - outerOpCtx, - opCtx, - _nss, - *_sessionId, - _writeConcern, - _fromShard, - range, - *_migrationId, - *_collectionUuid, - _migrationCloningProgress, - _parallelFetchersSupported, - chunkMigrationFetcherMaxBufferedSizeBytesPerThread.load()}; - fetcher.fetchAndScheduleInsertion(); - } - opCtx->checkForInterrupt(); - lastOpApplied = _migrationCloningProgress->getMaxOptime(); + auto assertNotAborted = [&](OperationContext* opCtx) { + opCtx->checkForInterrupt(); + outerOpCtx->checkForInterrupt(); + uassert(50748, "Migration aborted while copying documents", getState() != kAbort); + }; + + auto insertBatchFn = [&](OperationContext* opCtx, BSONObj nextBatch) { + auto arr = nextBatch["objects"].Obj(); + if (arr.isEmpty()) { + return false; + } + auto it = arr.begin(); + while (it != arr.end()) { + int batchNumCloned = 0; + int batchClonedBytes = 0; + const int batchMaxCloned = migrateCloneInsertionBatchSize.load(); + + assertNotAborted(opCtx); + + write_ops::InsertCommandRequest insertOp(_nss); + insertOp.getWriteCommandRequestBase().setOrdered(true); + insertOp.setDocuments([&] { + std::vector<BSONObj> toInsert; + while (it != arr.end() && + (batchMaxCloned <= 0 || batchNumCloned < batchMaxCloned)) { + const auto& doc = *it; + BSONObj docToClone = doc.Obj(); + toInsert.push_back(docToClone); + batchNumCloned++; + batchClonedBytes += docToClone.objsize(); + ++it; + } + return toInsert; + }()); + + { + // Disable the schema validation (during document inserts and updates) + // and any internal validation for opCtx for performInserts() + DisableDocumentValidation documentValidationDisabler( + opCtx, + DocumentValidationSettings::kDisableSchemaValidation | + DocumentValidationSettings::kDisableInternalValidation); + const auto reply = write_ops_exec::performInserts( + opCtx, insertOp, OperationSource::kFromMigrate); + for (unsigned long i = 0; i < reply.results.size(); ++i) { + uassertStatusOKWithContext(reply.results[i], + str::stream() << "Insert of " + << insertOp.getDocuments()[i] + << " failed."); + } + // Revert to the original DocumentValidationSettings for opCtx + } + + migrationutil::persistUpdatedNumOrphans( + opCtx, _migrationId.get(), *_collectionUuid, batchNumCloned); + + { + stdx::lock_guard<Latch> statsLock(_mutex); + _numCloned += batchNumCloned; + ShardingStatistics::get(opCtx).countDocsClonedOnRecipient.addAndFetch( + batchNumCloned); + _clonedBytes += batchClonedBytes; + } + if (_writeConcern.needToWaitForOtherNodes()) { + runWithoutSession(outerOpCtx, [&] { + repl::ReplicationCoordinator::StatusAndDuration replStatus = + repl::ReplicationCoordinator::get(opCtx)->awaitReplication( + opCtx, + repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(), + _writeConcern); + if (replStatus.status.code() == ErrorCodes::WriteConcernFailed) { + LOGV2_WARNING( + 22011, + "secondaryThrottle on, but doc insert timed out; continuing", + "migrationId"_attr = _migrationId->toBSON()); + } else { + uassertStatusOK(replStatus.status); + } + }); + } + + sleepmillis(migrateCloneInsertionBatchDelayMS.load()); + } + return true; + }; + + auto fetchBatchFn = [&](OperationContext* opCtx, BSONObj* nextBatch) { + auto commandResponse = uassertStatusOKWithContext( + fromShard->runCommand(opCtx, + ReadPreferenceSetting(ReadPreference::PrimaryOnly), + "admin", + migrateCloneRequest, + Shard::RetryPolicy::kNoRetry), + "_migrateClone failed: "); + + uassertStatusOKWithContext( + Shard::CommandResponse::getEffectiveStatus(commandResponse), + "_migrateClone failed: "); + + *nextBatch = commandResponse.response; + return nextBatch->getField("objects").Obj().isEmpty(); + }; + + // If running on a replicated system, we'll need to flush the docs we cloned to the + // secondaries + lastOpApplied = fetchAndApplyBatch(opCtx, insertBatchFn, fetchBatchFn); timing->done(4); migrateThreadHangAtStep4.pauseWhileSet(); if (MONGO_unlikely(failMigrationOnRecipient.shouldFail())) { - _setStateFail(str::stream() << "failing migration after cloning " << _getNumCloned() + _setStateFail(str::stream() << "failing migration after cloning " << _numCloned << " docs due to failMigrationOnRecipient failpoint"); return; } @@ -1426,8 +1491,6 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, if (!_applyMigrateOp(opCtx, nextBatch)) { return true; } - ShardingStatistics::get(opCtx).countBytesClonedOnCatchUpOnRecipient.addAndFetch( - nextBatch["size"].number()); const int maxIterations = 3600 * 50; @@ -1488,7 +1551,7 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, runWithoutSession(outerOpCtx, [&] { auto awaitReplicationResult = repl::ReplicationCoordinator::get(opCtx)->awaitReplication( - opCtx, lastOpApplied, WriteConcerns::kMajorityWriteConcernShardingTimeout); + opCtx, lastOpApplied, _writeConcern); uassertStatusOKWithContext(awaitReplicationResult.status, awaitReplicationResult.status.codeString()); }); @@ -1506,7 +1569,6 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, { // 6. Wait for commit _setState(kSteady); - migrateThreadHangAfterSteadyTransition.pauseWhileSet(); bool transferAfterCommit = false; while (getState() == kSteady || getState() == kCommitStart) { @@ -1534,8 +1596,7 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, auto mods = res.response; - if (mods["size"].number() > 0) { - (void)_applyMigrateOp(opCtx, mods); + if (mods["size"].number() > 0 && _applyMigrateOp(opCtx, mods)) { lastOpApplied = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); continue; } @@ -1694,7 +1755,6 @@ void MigrationDestinationManager::_migrateDriver(OperationContext* outerOpCtx, bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx, const BSONObj& xfer) { bool didAnything = false; long long changeInOrphans = 0; - long long totalDocs = 0; // Deleted documents if (xfer["deleted"].isABSONObj()) { @@ -1705,7 +1765,6 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx, const BSONObjIterator i(xfer["deleted"].Obj()); while (i.more()) { - totalDocs++; AutoGetCollection autoColl(opCtx, _nss, MODE_IX); uassert(ErrorCodes::ConflictingOperationInProgress, str::stream() << "Collection " << _nss.ns() @@ -1748,7 +1807,6 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx, const if (xfer["reload"].isABSONObj()) { BSONObjIterator i(xfer["reload"].Obj()); while (i.more()) { - totalDocs++; AutoGetCollection autoColl(opCtx, _nss, MODE_IX); uassert(ErrorCodes::ConflictingOperationInProgress, str::stream() << "Collection " << _nss.ns() @@ -1803,9 +1861,6 @@ bool MigrationDestinationManager::_applyMigrateOp(OperationContext* opCtx, const migrationutil::persistUpdatedNumOrphans( opCtx, _migrationId.get(), *_collectionUuid, changeInOrphans); } - - ShardingStatistics::get(opCtx).countDocsClonedOnCatchUpOnRecipient.addAndFetch(totalDocs); - return didAnything; } diff --git a/src/mongo/db/s/migration_destination_manager.h b/src/mongo/db/s/migration_destination_manager.h index 6caa4c2a1b3..925296091cb 100644 --- a/src/mongo/db/s/migration_destination_manager.h +++ b/src/mongo/db/s/migration_destination_manager.h @@ -29,7 +29,6 @@ #pragma once -#include <memory> #include <string> #include "mongo/base/string_data.h" @@ -40,8 +39,6 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/repl/replica_set_aware_service.h" #include "mongo/db/s/active_migrations_registry.h" -#include "mongo/db/s/collection_sharding_runtime.h" -#include "mongo/db/s/migration_batch_fetcher.h" #include "mongo/db/s/migration_recipient_recovery_document_gen.h" #include "mongo/db/s/migration_session_id.h" #include "mongo/db/s/session_catalog_migration_destination.h" @@ -181,11 +178,6 @@ public: const boost::optional<ChunkManager>& cm, boost::optional<Timestamp> afterClusterTime); - - bool isParallelFetchingSupported() { - return _parallelFetchersSupported; - } - /** * Gets the collection uuid and options from fromShardId. If given a chunk manager, will fetch * the collection options using the database version protocol. @@ -194,17 +186,11 @@ public: BSONObj options; UUID uuid; }; - - static CollectionOptionsAndUUID getCollectionOptions( - OperationContext* opCtx, - const NamespaceStringOrUUID& nssOrUUID, - boost::optional<Timestamp> afterClusterTime); - static CollectionOptionsAndUUID getCollectionOptions( OperationContext* opCtx, const NamespaceStringOrUUID& nssOrUUID, const ShardId& fromShardId, - const boost::optional<DatabaseVersion>& dbVersion, + const boost::optional<ChunkManager>& cm, boost::optional<Timestamp> afterClusterTime); @@ -297,22 +283,8 @@ private: stdx::thread _migrateThreadHandle; - long long _getNumCloned() { - return _migrationCloningProgress ? _migrationCloningProgress->getNumCloned() : 0; - } - - long long _getNumBytesCloned() { - return _migrationCloningProgress ? _migrationCloningProgress->getNumBytes() : 0; - } - boost::optional<UUID> _migrationId; boost::optional<UUID> _collectionUuid; - - // State that is shared among all inserter threads. - std::shared_ptr<MigrationCloningProgressSharedState> _migrationCloningProgress; - - bool _parallelFetchersSupported; - LogicalSessionId _lsid; TxnNumber _txnNumber{kUninitializedTxnNumber}; NamespaceString _nss; @@ -332,6 +304,8 @@ private: // failure we can perform the appropriate cleanup. bool _chunkMarkedPending{false}; + long long _numCloned{0}; + long long _clonedBytes{0}; long long _numCatchup{0}; long long _numSteady{0}; diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp index cac5a1696db..8233113e409 100644 --- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp +++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp @@ -94,7 +94,6 @@ public: const BSONObj& cmdObj, std::string& errmsg, BSONObjBuilder& result) override { - opCtx->setAlwaysInterruptAtStepDownOrUp(); uassertStatusOK(ShardingState::get(opCtx)->canAcceptShardedCommands()); auto nss = NamespaceString(parseNs(dbname, cmdObj)); @@ -110,19 +109,13 @@ public: // Ensure this shard is not currently receiving or donating any chunks. auto scopedReceiveChunk( uassertStatusOK(ActiveMigrationsRegistry::get(opCtx).registerReceiveChunk( - opCtx, nss, chunkRange, cloneRequest.getFromShardId(), false))); + opCtx, nss, chunkRange, cloneRequest.getFromShardId()))); // We force a refresh immediately after registering this migration to guarantee that this // shard will not receive a chunk after refreshing. onShardVersionMismatch(opCtx, nss, boost::none); const auto shardId = ShardingState::get(opCtx)->shardId(); - // Wait for the ShardServerCatalogCacheLoader to finish flushing the metadata to the - // storage. This is not required for correctness, but helps mitigate stalls on secondaries - // when a shard receives the first chunk for a collection with a large routing table. - CatalogCacheLoader::get(opCtx).waitForCollectionFlush(opCtx, nss); - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - const auto collectionEpoch = [&] { AutoGetCollection autoColl(opCtx, nss, MODE_IS); auto const optMetadata = diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp index 9ca15fe9fad..e4ed5e9df6d 100644 --- a/src/mongo/db/s/migration_source_manager.cpp +++ b/src/mongo/db/s/migration_source_manager.cpp @@ -33,6 +33,7 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/catalog_raii.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/logical_session_cache.h" #include "mongo/db/logical_session_id_helpers.h" #include "mongo/db/op_observer.h" @@ -81,6 +82,8 @@ const WriteConcernOptions kMajorityWriteConcern(WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, WriteConcernOptions::kWriteConcernTimeoutMigration); +std::string kEmptyErrMsgForMoveTimingHelper; + /** * Best-effort attempt to ensure the recipient shard has refreshed its routing table to * 'newCollVersion'. Fires and forgets an asychronous remote setShardVersion command. @@ -177,6 +180,7 @@ MigrationSourceManager::MigrationSourceManager(OperationContext* opCtx, _args.getMin(), _args.getMax(), 6, // Total number of steps + &kEmptyErrMsgForMoveTimingHelper, _args.getToShard(), _args.getFromShard()) { invariant(!_opCtx->lockState()->isLocked()); @@ -362,17 +366,7 @@ MigrationSourceManager::~MigrationSourceManager() { invariant(!_cloneDriver); _stats.totalDonorMoveChunkTimeMillis.addAndFetch(_entireOpTimer.millis()); - if (_state == kDone) { - _completion.emplaceValue(); - } else { - std::string errMsg = "Migration not completed"; - if (_coordinator) { - const auto& migrationId = _coordinator->getMigrationId(); - errMsg = str::stream() << "Migration " << migrationId << " not completed"; - } - auto status = Status{ErrorCodes::Interrupted, errMsg}; - _completion.setError(status); - } + _completion.emplaceValue(); } void MigrationSourceManager::startClone() { @@ -426,12 +420,6 @@ void MigrationSourceManager::startClone() { _state = kCloning; } - // Refreshing the collection routing information after starting the clone driver will give us a - // stable view on whether the recipient is owning other chunks of the collection (a condition - // that will be later evaluated). - uassertStatusOK( - Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfoWithRefresh(_opCtx, nss())); - if (replEnabled) { auto const readConcernArgs = repl::ReadConcernArgs( replCoord->getMyLastAppliedOpTime(), repl::ReadConcernLevel::kLocalReadConcern); @@ -477,12 +465,11 @@ void MigrationSourceManager::enterCriticalSection() { _stats.totalDonorChunkCloneTimeMillis.addAndFetch(_cloneAndCommitTimer.millis()); _cloneAndCommitTimer.reset(); - const auto cm = - uassertStatusOK(Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfo(_opCtx, nss())); + const auto& metadata = _getCurrentMetadataAndCheckEpoch(); // Check that there are no chunks on the recepient shard. Write an oplog event for change // streams if this is the first migration to the recipient. - if (!cm.getVersion(_args.getToShard()).isSet()) { + if (!metadata.getChunkManager()->getVersion(_args.getToShard()).isSet()) { migrationutil::notifyChangeStreamsOnRecipientFirstChunk( _opCtx, nss(), _args.getFromShard(), _args.getToShard(), _collectionUUID); } @@ -492,7 +479,7 @@ void MigrationSourceManager::enterCriticalSection() { // NOTE: The 'migrateChunkToNewShard' oplog message written by the above call to // 'notifyChangeStreamsOnRecipientFirstChunk' depends on this majority write to carry its // local write to majority committed. - uassertStatusOKWithContext(ShardingStateRecovery::startMetadataOp(_opCtx), "Start metadata op"); + uassertStatusOK(ShardingStateRecovery::startMetadataOp(_opCtx)); LOGV2_DEBUG_OPTIONS(4817402, 2, @@ -510,13 +497,17 @@ void MigrationSourceManager::enterCriticalSection() { // time inclusive of the migration config commit update from accessing secondary data. // Note: this write must occur after the critSec flag is set, to ensure the secondary refresh // will stall behind the flag. - uassertStatusOKWithContext( - shardmetadatautil::updateShardCollectionsEntry( - _opCtx, - BSON(ShardCollectionType::kNssFieldName << nss().ns()), - BSON("$inc" << BSON(ShardCollectionType::kEnterCriticalSectionCounterFieldName << 1)), - false /*upsert*/), - "Persist critical section signal for secondaries"); + Status signalStatus = shardmetadatautil::updateShardCollectionsEntry( + _opCtx, + BSON(ShardCollectionType::kNssFieldName << nss().ns()), + BSON("$inc" << BSON(ShardCollectionType::kEnterCriticalSectionCounterFieldName << 1)), + false /*upsert*/); + if (!signalStatus.isOK()) { + uasserted( + ErrorCodes::OperationFailed, + str::stream() << "Failed to persist critical section signal for secondaries due to: " + << signalStatus.toString()); + } LOGV2(22017, "Migration successfully entered critical section", @@ -693,7 +684,7 @@ void MigrationSourceManager::commitChunkMetadataOnConfig() { _stats.totalCriticalSectionCommitTimeMillis.addAndFetch(t.millis()); - LOGV2(6107801, + LOGV2(4817403, "Exiting commit critical section", "migrationId"_attr = _coordinator->getMigrationId(), "durationMillis"_attr = t.millis()); @@ -814,12 +805,6 @@ void MigrationSourceManager::_cleanup(bool completeMigration) noexcept { }(); if (_state == kCriticalSection || _state == kCloneCompleted || _state == kCommittingOnConfig) { - LOGV2_DEBUG_OPTIONS(4817403, - 2, - {logv2::LogComponent::kShardMigrationPerf}, - "Finished critical section", - "migrationId"_attr = _coordinator->getMigrationId()); - LOGV2(6107802, "Finished critical section", "migrationId"_attr = _coordinator->getMigrationId(), diff --git a/src/mongo/db/s/migration_source_manager.h b/src/mongo/db/s/migration_source_manager.h index fed10b56272..d93c701f3d5 100644 --- a/src/mongo/db/s/migration_source_manager.h +++ b/src/mongo/db/s/migration_source_manager.h @@ -97,8 +97,8 @@ public: * * May throw any exception. Known exceptions are: * - InvalidOptions if the operation context is missing shard version - * - StaleConfig if the expected collection version does not match what we find it to be after - * acquiring the distributed lock. + * - StaleConfigException if the expected collection version does not match what we find it + * to be after acquiring the distributed lock. */ MigrationSourceManager(OperationContext* opCtx, ShardsvrMoveRange&& request, @@ -176,17 +176,6 @@ public: return _args.getCommandParameter(); } - boost::optional<UUID> getMigrationId() { - if (_coordinator) { - return _coordinator->getMigrationId(); - } - return boost::none; - } - - long long getOpTimeMillis() { - return _entireOpTimer.millis(); - } - private: // Used to track the current state of the source manager. See the methods above, which have // comments explaining the various state transitions. diff --git a/src/mongo/db/s/migration_util.cpp b/src/mongo/db/s/migration_util.cpp index f0058b60318..ec63e7477af 100644 --- a/src/mongo/db/s/migration_util.cpp +++ b/src/mongo/db/s/migration_util.cpp @@ -42,7 +42,7 @@ #include "mongo/db/catalog/collection_catalog_helper.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/commands.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/logical_session_cache.h" #include "mongo/db/namespace_string.h" @@ -521,27 +521,14 @@ void resubmitRangeDeletionsOnStepUp(ServiceContext* serviceContext) { FindCommandRequest findCommand(NamespaceString::kRangeDeletionNamespace); findCommand.setFilter(BSON(RangeDeletionTask::kProcessingFieldName << true)); auto cursor = client.find(std::move(findCommand)); - - auto retFuture = ExecutorFuture<void>(getMigrationUtilExecutor(serviceContext)); - - int rangeDeletionsMarkedAsProcessing = 0; - while (cursor->more()) { - retFuture = migrationutil::submitRangeDeletionTask( + if (cursor->more()) { + return migrationutil::submitRangeDeletionTask( opCtx.get(), RangeDeletionTask::parse(IDLParserErrorContext("rangeDeletionRecovery"), cursor->next())); - rangeDeletionsMarkedAsProcessing++; - } - - if (rangeDeletionsMarkedAsProcessing > 1) { - LOGV2_WARNING( - 6695800, - "Rescheduling several range deletions marked as processing. Orphans count " - "may be off while they are not drained", - "numRangeDeletionsMarkedAsProcessing"_attr = rangeDeletionsMarkedAsProcessing); + } else { + return ExecutorFuture<void>(getMigrationUtilExecutor(serviceContext)); } - - return retFuture; }) .then([serviceContext] { ThreadClient tc("ResubmitRangeDeletions", serviceContext); @@ -661,8 +648,9 @@ void submitOrphanRangesForCleanup(OperationContext* opCtx) { if (tenantDbName.dbName() == NamespaceString::kLocalDb) continue; - for (auto&& coll : catalog->range(tenantDbName)) { - auto uuid = coll->uuid(); + for (auto collIt = catalog->begin(opCtx, tenantDbName); collIt != catalog->end(opCtx); + ++collIt) { + auto uuid = collIt.uuid().get(); auto nss = catalog->lookupNSSByUUID(opCtx, uuid).get(); LOGV2_DEBUG(22034, 2, @@ -718,8 +706,8 @@ void persistUpdatedNumOrphans(OperationContext* opCtx, << BSON("$exists" << true)); try { PersistentTaskStore<RangeDeletionTask> store(NamespaceString::kRangeDeletionNamespace); - ScopedRangeDeleterLock rangeDeleterLock(opCtx, MODE_IX); - // TODO (SERVER-65996) Remove writeConflictRetry loop + ScopedRangeDeleterLock rangeDeleterLock(opCtx, collectionUuid); + // TODO (SERVER-54284) Remove writeConflictRetry loop writeConflictRetry( opCtx, "updateOrphanCount", NamespaceString::kRangeDeletionNamespace.ns(), [&] { store.update(opCtx, @@ -843,7 +831,6 @@ void persistCommitDecision(OperationContext* opCtx, store.upsert(opCtx, BSON(MigrationCoordinatorDocument::kIdFieldName << migrationDoc.getId()), migrationDoc.toBSON()); - ShardingStatistics::get(opCtx).countDonorMoveChunkCommitted.addAndFetch(1); if (hangInPersistMigrateCommitDecisionThenSimulateErrorUninterruptible.shouldFail()) { hangInPersistMigrateCommitDecisionThenSimulateErrorUninterruptible.pauseWhileSet(opCtx); @@ -863,7 +850,6 @@ void persistAbortDecision(OperationContext* opCtx, store.upsert(opCtx, BSON(MigrationCoordinatorDocument::kIdFieldName << migrationDoc.getId()), migrationDoc.toBSON()); - ShardingStatistics::get(opCtx).countDonorMoveChunkAborted.addAndFetch(1); if (hangInPersistMigrateAbortDecisionThenSimulateErrorUninterruptible.shouldFail()) { hangInPersistMigrateAbortDecisionThenSimulateErrorUninterruptible.pauseWhileSet(opCtx); @@ -1000,6 +986,11 @@ void markAsReadyRangeDeletionTaskLocally(OperationContext* opCtx, const UUID& mi } void deleteMigrationCoordinatorDocumentLocally(OperationContext* opCtx, const UUID& migrationId) { + // Before deleting the migration coordinator document, ensure that in the case of a crash, the + // node will start-up from at least the configTime, which it obtained as part of recovery of the + // shardVersion, which will ensure that it will see at least the same shardVersion. + VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); + PersistentTaskStore<MigrationCoordinatorDocument> store( NamespaceString::kMigrationCoordinatorsNamespace); store.remove(opCtx, @@ -1135,7 +1126,7 @@ void recoverMigrationCoordinations(OperationContext* opCtx, hangInRefreshFilteringMetadataUntilSuccessThenSimulateErrorUninterruptible .pauseWhileSet(); uasserted(ErrorCodes::InternalError, - "simulate an error response for forceGetCurrentMetadata"); + "simulate an error response for forceShardFilteringMetadataRefresh"); } auto setFilteringMetadata = [&opCtx, ¤tMetadata, &doc, &cancellationToken]() { @@ -1175,15 +1166,6 @@ void recoverMigrationCoordinations(OperationContext* opCtx, "coordinatorDocumentUUID"_attr = doc.getCollectionUuid()); } - // TODO SERVER-71918 once the drop collection coordinator starts persisting the - // config time we can remove this. Since the collection has been dropped, - // persist config time inclusive of the drop collection event before deleting - // leftover migration metadata. - // This will ensure that in case of stepdown the new - // primary won't read stale data from config server and think that the sharded - // collection still exists. - VectorClockMutable::get(opCtx)->waitForDurableConfigTime().get(opCtx); - deleteRangeDeletionTaskOnRecipient(opCtx, doc.getRecipientShardId(), doc.getId()); deleteRangeDeletionTaskLocally(opCtx, doc.getId()); coordinator.forgetMigration(opCtx); @@ -1316,15 +1298,10 @@ void resumeMigrationRecipientsOnStepUp(OperationContext* opCtx) { const auto& nss = doc.getNss(); // Register this receiveChunk on the ActiveMigrationsRegistry before completing step-up - // to prevent a new migration from starting while a receiveChunk was ongoing. Wait for - // any migrations that began in a previous term to complete if there are any. + // to prevent a new migration from starting while a receiveChunk was ongoing. auto scopedReceiveChunk( uassertStatusOK(ActiveMigrationsRegistry::get(opCtx).registerReceiveChunk( - opCtx, - nss, - doc.getRange(), - doc.getDonorShardIdForLoggingPurposesOnly(), - true /* waitForOngoingMigrations */))); + opCtx, nss, doc.getRange(), doc.getDonorShardIdForLoggingPurposesOnly()))); const auto mdm = MigrationDestinationManager::get(opCtx); uassertStatusOK( diff --git a/src/mongo/db/s/migration_util_test.cpp b/src/mongo/db/s/migration_util_test.cpp index ddcf302b5d8..24fb4d3567e 100644 --- a/src/mongo/db/s/migration_util_test.cpp +++ b/src/mongo/db/s/migration_util_test.cpp @@ -417,16 +417,14 @@ public: StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { return repl::OpTimeWith<std::vector<ShardType>>(_shards); } - std::vector<CollectionType> getCollections(OperationContext* opCtx, - StringData dbName, - repl::ReadConcernLevel readConcernLevel, - const BSONObj& sort) override { + std::vector<CollectionType> getCollections( + OperationContext* opCtx, + StringData dbName, + repl::ReadConcernLevel readConcernLevel) override { return _colls; } diff --git a/src/mongo/db/s/move_primary_source_manager.cpp b/src/mongo/db/s/move_primary_source_manager.cpp index 356ce6c8f88..dea310c8430 100644 --- a/src/mongo/db/s/move_primary_source_manager.cpp +++ b/src/mongo/db/s/move_primary_source_manager.cpp @@ -34,7 +34,6 @@ #include "mongo/db/s/move_primary_source_manager.h" #include "mongo/client/connpool.h" -#include "mongo/db/catalog/drop_collection.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/commands.h" #include "mongo/db/dbdirectclient.h" @@ -47,7 +46,6 @@ #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/grid.h" -#include "mongo/s/request_types/move_primary_gen.h" #include "mongo/util/exit.h" #include "mongo/util/scopeguard.h" @@ -217,8 +215,6 @@ Status MovePrimarySourceManager::commitOnConfig(OperationContext* opCtx) { invariant(_state == kCriticalSection); ScopeGuard scopedGuard([&] { cleanupOnError(opCtx); }); - boost::optional<DatabaseVersion> expectedDbVersion; - { AutoGetDb autoDb(opCtx, getNss().toString(), MODE_X); @@ -234,13 +230,13 @@ Status MovePrimarySourceManager::commitOnConfig(OperationContext* opCtx) { // Read operations must begin to wait on the critical section just before we send the // commit operation to the config server dss->enterCriticalSectionCommitPhase(opCtx, dssLock, _critSecReason); - - expectedDbVersion = dss->getDbVersion(opCtx, dssLock); } + auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + auto commitStatus = [&]() { try { - return _commitOnConfig(opCtx, *expectedDbVersion); + return _commitOnConfig(opCtx); } catch (const DBException& ex) { return ex.toStatus(); } @@ -248,7 +244,8 @@ Status MovePrimarySourceManager::commitOnConfig(OperationContext* opCtx) { if (!commitStatus.isOK()) { // Need to get the latest optime in case the refresh request goes to a secondary -- - // otherwise the read won't wait for the write that commit on config server may have done. + // otherwise the read won't wait for the write that _commitOnConfig may have + // done LOGV2(22044, "Error occurred while committing the movePrimary. Performing a majority write " "against the config server to obtain its latest optime: {error}", @@ -332,121 +329,62 @@ Status MovePrimarySourceManager::commitOnConfig(OperationContext* opCtx) { return Status::OK(); } -Status MovePrimarySourceManager::_commitOnConfig(OperationContext* opCtx, - const DatabaseVersion& expectedDbVersion) { - LOGV2_DEBUG(6854100, - 3, - "Committing movePrimary", - "db"_attr = _dbname, - "fromShard"_attr = _fromShard, - "toShard"_attr = _toShard, - "expectedDbVersion"_attr = expectedDbVersion); - - const auto commitStatus = [&] { - ConfigsvrCommitMovePrimary commitRequest(_dbname.toString(), expectedDbVersion, _toShard); - commitRequest.setDbName(NamespaceString::kAdminDb); - - const auto commitResponse = - Grid::get(opCtx)->shardRegistry()->getConfigShard()->runCommandWithFixedRetryAttempts( - opCtx, - ReadPreferenceSetting(ReadPreference::PrimaryOnly), - NamespaceString::kAdminDb.toString(), - CommandHelpers::appendMajorityWriteConcern(commitRequest.toBSON({})), - Shard::RetryPolicy::kIdempotent); - - const auto status = Shard::CommandResponse::getEffectiveStatus(commitResponse); - if (status != ErrorCodes::CommandNotFound) { - return status; - } +Status MovePrimarySourceManager::_commitOnConfig(OperationContext* opCtx) { + auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + + auto findResponse = uassertStatusOK( + configShard->exhaustiveFindOnConfig(opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + repl::ReadConcernLevel::kMajorityReadConcern, + NamespaceString::kConfigDatabasesNamespace, + BSON(DatabaseType::kNameFieldName << _dbname), + BSON(DatabaseType::kNameFieldName << -1), + 1)); + + const auto databasesVector = std::move(findResponse.docs); + uassert(ErrorCodes::IncompatibleShardingMetadata, + str::stream() << "Tried to find max database version for database '" << _dbname + << "', but found no databases", + !databasesVector.empty()); + + const auto dbType = + DatabaseType::parse(IDLParserErrorContext("DatabaseType"), databasesVector.front()); + + if (dbType.getPrimary() == _toShard) { + return Status::OK(); + } - LOGV2(6854101, - "_configsvrCommitMovePrimary command not found on config server, so try to update " - "the metadata document directly", - "db"_attr = _dbname); + auto newDbType = dbType; + newDbType.setPrimary(_toShard); - // The fallback logic is not synchronized with the removeShard command and simultaneous - // invocations of movePrimary and removeShard can lead to data loss. - return _fallbackCommitOnConfig(opCtx, expectedDbVersion); - }(); + auto const currentDatabaseVersion = dbType.getVersion(); - if (!commitStatus.isOK()) { - LOGV2(6854102, + newDbType.setVersion(currentDatabaseVersion.makeUpdated()); + + auto const updateQuery = + BSON(DatabaseType::kNameFieldName << _dbname << DatabaseType::kVersionFieldName + << currentDatabaseVersion.toBSON()); + + auto updateStatus = Grid::get(opCtx)->catalogClient()->updateConfigDocument( + opCtx, + NamespaceString::kConfigDatabasesNamespace, + updateQuery, + newDbType.toBSON(), + false, + ShardingCatalogClient::kMajorityWriteConcern); + + if (!updateStatus.isOK()) { + LOGV2(5448803, + "Error committing movePrimary for {db}: {error}", "Error committing movePrimary", "db"_attr = _dbname, - "error"_attr = redact(commitStatus)); - return commitStatus; + "error"_attr = redact(updateStatus.getStatus())); + return updateStatus.getStatus(); } - const auto updatedDbType = [&]() { - const auto findResponse = uassertStatusOK( - Grid::get(opCtx)->shardRegistry()->getConfigShard()->exhaustiveFindOnConfig( - opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - repl::ReadConcernLevel::kMajorityReadConcern, - NamespaceString::kConfigDatabasesNamespace, - BSON(DatabaseType::kNameFieldName << _dbname), - BSON(DatabaseType::kNameFieldName << -1), - 1)); - - const auto databases = std::move(findResponse.docs); - uassert(ErrorCodes::IncompatibleShardingMetadata, - "Tried to find version for database {}, but found no databases"_format(_dbname), - !databases.empty()); - - return DatabaseType::parse(IDLParserErrorContext("DatabaseType"), databases.front()); - }(); - tassert(6851100, - "Error committing movePrimary: database version went backwards", - updatedDbType.getVersion() > expectedDbVersion); - uassert(6851101, - "Error committing movePrimary: update of config.databases failed", - updatedDbType.getPrimary() != _fromShard); - - LOGV2_DEBUG(6854103, - 3, - "Commited movePrimary", - "db"_attr = _dbname, - "fromShard"_attr = _fromShard, - "toShard"_attr = _toShard, - "updatedDbVersion"_attr = updatedDbType.getVersion()); - return Status::OK(); } -Status MovePrimarySourceManager::_fallbackCommitOnConfig(OperationContext* opCtx, - const DatabaseVersion& expectedDbVersion) { - const auto query = [&] { - BSONObjBuilder bsonBuilder; - bsonBuilder.append(DatabaseType::kNameFieldName, _dbname); - // Include the version in the update filter to be resilient to potential network retries and - // delayed messages. - for (const auto [fieldName, fieldValue] : expectedDbVersion.toBSON()) { - const auto dottedFieldName = DatabaseType::kVersionFieldName + "." + fieldName; - bsonBuilder.appendAs(fieldValue, dottedFieldName); - } - return bsonBuilder.obj(); - }(); - - const auto update = [&] { - const auto newDbVersion = expectedDbVersion.makeUpdated(); - - BSONObjBuilder bsonBuilder; - bsonBuilder.append(DatabaseType::kPrimaryFieldName, _toShard); - bsonBuilder.append(DatabaseType::kVersionFieldName, newDbVersion.toBSON()); - return BSON("$set" << bsonBuilder.obj()); - }(); - - return Grid::get(opCtx) - ->catalogClient() - ->updateConfigDocument(opCtx, - NamespaceString::kConfigDatabasesNamespace, - query, - update, - false, - ShardingCatalogClient::kMajorityWriteConcern) - .getStatus(); -} - Status MovePrimarySourceManager::cleanStaleData(OperationContext* opCtx) { invariant(!opCtx->lockState()->isLocked()); invariant(_state == kNeedCleanStaleData); @@ -457,21 +395,17 @@ Status MovePrimarySourceManager::cleanStaleData(OperationContext* opCtx) { } // Only drop the cloned (unsharded) collections. + DBDirectClient client(opCtx); for (auto& coll : _clonedColls) { - DropReply unusedDropReply; - try { - uassertStatusOK( - dropCollection(opCtx, - coll, - &unusedDropReply, - DropCollectionSystemCollectionMode::kAllowSystemCollectionDrops, - false /* fromMigrate */)); - } catch (const DBException& e) { + BSONObj dropCollResult; + client.runCommand(_dbname.toString(), BSON("drop" << coll.coll()), dropCollResult); + Status dropStatus = getStatusFromCommandResult(dropCollResult); + if (!dropStatus.isOK()) { LOGV2(22045, "Failed to drop cloned collection {namespace} in movePrimary: {error}", "Failed to drop cloned collection in movePrimary", "namespace"_attr = coll, - "error"_attr = redact(e)); + "error"_attr = redact(dropStatus)); } } @@ -479,6 +413,7 @@ Status MovePrimarySourceManager::cleanStaleData(OperationContext* opCtx) { return Status::OK(); } + void MovePrimarySourceManager::cleanupOnError(OperationContext* opCtx) { if (_state == kDone) { return; diff --git a/src/mongo/db/s/move_primary_source_manager.h b/src/mongo/db/s/move_primary_source_manager.h index 42586f9d102..cd678c95b19 100644 --- a/src/mongo/db/s/move_primary_source_manager.h +++ b/src/mongo/db/s/move_primary_source_manager.h @@ -70,10 +70,10 @@ public: * Instantiates a new movePrimary source manager. Must be called with the distributed lock * acquired in advance (not asserted). * - * May throw any exception. Known exceptions are: + * May throw any exception. Known exceptions (TODO) are: * - InvalidOptions if the operation context is missing database version - * - StaleConfig if the expected database version does not match what we find it to be after - * acquiring the distributed lock + * - StaleConfigException if the expected database version does not match what we find it + * to be after acquiring the distributed lock. */ MovePrimarySourceManager(OperationContext* opCtx, @@ -145,20 +145,10 @@ private: } /** - * Invokes the _configsvrCommitMovePrimary command of the config server to reassign the primary - * shard of the database. + * Updates CSRS metadata in config.databases collection to move the given primary database on + * its new shard. */ - Status _commitOnConfig(OperationContext* opCtx, const DatabaseVersion& expectedDbVersion); - - /** - * Updates the config server's metadata in config.databases collection to reassign the primary - * shard of the database. - * - * This logic is not synchronized with the removeShard command and simultaneous invocations of - * movePrimary and removeShard can lead to data loss. - */ - Status _fallbackCommitOnConfig(OperationContext* opCtx, - const DatabaseVersion& expectedDbVersion); + Status _commitOnConfig(OperationContext* opCtx); // Used to track the current state of the source manager. See the methods above, which have // comments explaining the various state transitions. diff --git a/src/mongo/db/s/move_timing_helper.cpp b/src/mongo/db/s/move_timing_helper.cpp index 8eca70419e2..f62adc83956 100644 --- a/src/mongo/db/s/move_timing_helper.cpp +++ b/src/mongo/db/s/move_timing_helper.cpp @@ -47,6 +47,7 @@ MoveTimingHelper::MoveTimingHelper(OperationContext* opCtx, const boost::optional<BSONObj>& min, const boost::optional<BSONObj>& max, int totalNumSteps, + std::string* cmdErrmsg, const ShardId& toShard, const ShardId& fromShard) : _opCtx(opCtx), @@ -57,6 +58,7 @@ MoveTimingHelper::MoveTimingHelper(OperationContext* opCtx, _min(min), _max(max), _totalNumSteps(totalNumSteps), + _cmdErrmsg(cmdErrmsg), _nextStep(0) {} MoveTimingHelper::~MoveTimingHelper() { @@ -80,8 +82,8 @@ MoveTimingHelper::~MoveTimingHelper() { _b.append("note", "success"); } - if (!_cmdErrmsg.empty()) { - _b.append("errmsg", _cmdErrmsg); + if (!_cmdErrmsg->empty()) { + _b.append("errmsg", *_cmdErrmsg); } ShardingLogging::get(_opCtx)->logChange(_opCtx, diff --git a/src/mongo/db/s/move_timing_helper.h b/src/mongo/db/s/move_timing_helper.h index ad6844ef0a1..a90dd465090 100644 --- a/src/mongo/db/s/move_timing_helper.h +++ b/src/mongo/db/s/move_timing_helper.h @@ -48,6 +48,7 @@ public: const boost::optional<BSONObj>& min, const boost::optional<BSONObj>& max, int totalNumSteps, + std::string* cmdErrmsg, const ShardId& toShard, const ShardId& fromShard); ~MoveTimingHelper(); @@ -60,10 +61,6 @@ public: _max.emplace(max); } - void setCmdErrMsg(std::string cmdErrMsg) { - _cmdErrmsg = std::move(cmdErrMsg); - } - void done(int step); private: @@ -78,7 +75,7 @@ private: boost::optional<BSONObj> _min, _max; const int _totalNumSteps; - std::string _cmdErrmsg; + const std::string* _cmdErrmsg; int _nextStep; BSONObjBuilder _b; diff --git a/src/mongo/db/s/op_observer_sharding_impl.cpp b/src/mongo/db/s/op_observer_sharding_impl.cpp index be54ad6f9e7..ab8ce8ca5e8 100644 --- a/src/mongo/db/s/op_observer_sharding_impl.cpp +++ b/src/mongo/db/s/op_observer_sharding_impl.cpp @@ -41,12 +41,13 @@ #include "mongo/db/s/migration_source_manager.h" #include "mongo/db/s/resharding/resharding_util.h" #include "mongo/db/s/sharding_write_router.h" -#include "mongo/db/transaction_participant.h" #include "mongo/logv2/log.h" namespace mongo { namespace { +const auto getIsMigrating = OperationContext::declareDecoration<bool>(); + /** * Write operations do shard version checking, but if an update operation runs as part of a * 'readConcern:snapshot' transaction, the router could have used the metadata at the snapshot @@ -69,6 +70,13 @@ void assertIntersectingChunkHasNotMoved(OperationContext* opCtx, chunk.throwIfMoved(); } +bool isMigratingWithCSRLock(CollectionShardingRuntime* csr, + CollectionShardingRuntime::CSRLock& csrLock, + BSONObj const& docToDelete) { + auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock); + return cloner && cloner->isDocumentInMigratingChunk(docToDelete); +} + void assertMovePrimaryInProgress(OperationContext* opCtx, NamespaceString const& nss) { if (!nss.isNormalCollection() && nss.coll() != "system.views" && !nss.isTimeseriesBucketsCollection()) { @@ -96,9 +104,19 @@ void assertMovePrimaryInProgress(OperationContext* opCtx, NamespaceString const& } // namespace +bool OpObserverShardingImpl::isMigrating(OperationContext* opCtx, + NamespaceString const& nss, + BSONObj const& docToDelete) { + auto csr = CollectionShardingRuntime::get(opCtx, nss); + auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); + return isMigratingWithCSRLock(csr, csrLock, docToDelete); +} + void OpObserverShardingImpl::shardObserveAboutToDelete(OperationContext* opCtx, NamespaceString const& nss, - BSONObj const& docToDelete) {} + BSONObj const& docToDelete) { + getIsMigrating(opCtx) = isMigrating(opCtx, nss, docToDelete); +} void OpObserverShardingImpl::shardObserveInsertOp(OperationContext* opCtx, const NamespaceString nss, @@ -120,10 +138,6 @@ void OpObserverShardingImpl::shardObserveInsertOp(OperationContext* opCtx, return; } - if (!opCtx->writesAreReplicated()) { - return; - } - if (inMultiDocumentTransaction) { const auto atClusterTime = repl::ReadConcernArgs::get(opCtx).getArgsAtClusterTime(); @@ -136,8 +150,11 @@ void OpObserverShardingImpl::shardObserveInsertOp(OperationContext* opCtx, return; } - opCtx->recoveryUnit()->registerChange( - std::make_unique<LogInsertForShardingHandler>(opCtx, nss, insertedDoc, opTime)); + auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); + auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock); + if (cloner) { + cloner->onInsertOp(opCtx, insertedDoc, opTime); + } } void OpObserverShardingImpl::shardObserveUpdateOp(OperationContext* opCtx, @@ -158,10 +175,6 @@ void OpObserverShardingImpl::shardObserveUpdateOp(OperationContext* opCtx, return; } - if (!opCtx->writesAreReplicated()) { - return; - } - if (inMultiDocumentTransaction) { const auto atClusterTime = repl::ReadConcernArgs::get(opCtx).getArgsAtClusterTime(); @@ -174,13 +187,16 @@ void OpObserverShardingImpl::shardObserveUpdateOp(OperationContext* opCtx, return; } - opCtx->recoveryUnit()->registerChange(std::make_unique<LogUpdateForShardingHandler>( - opCtx, nss, preImageDoc, postImageDoc, opTime, prePostImageOpTime)); + auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); + auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock); + if (cloner) { + cloner->onUpdateOp(opCtx, preImageDoc, postImageDoc, opTime, prePostImageOpTime); + } } void OpObserverShardingImpl::shardObserveDeleteOp(OperationContext* opCtx, - const NamespaceString& nss, - const repl::DocumentKey& documentKey, + const NamespaceString nss, + const BSONObj& documentKey, const repl::OpTime& opTime, const ShardingWriteRouter& shardingWriteRouter, const repl::OpTime& preImageOpTime, @@ -195,25 +211,24 @@ void OpObserverShardingImpl::shardObserveDeleteOp(OperationContext* opCtx, return; } - if (!opCtx->writesAreReplicated()) { - return; - } - if (inMultiDocumentTransaction) { const auto atClusterTime = repl::ReadConcernArgs::get(opCtx).getArgsAtClusterTime(); if (atClusterTime) { const auto shardKey = - metadata->getShardKeyPattern().extractShardKeyFromDocumentKeyThrows( - documentKey.getShardKeyAndId()); + metadata->getShardKeyPattern().extractShardKeyFromDocumentKeyThrows(documentKey); assertIntersectingChunkHasNotMoved(opCtx, *metadata, shardKey, *atClusterTime); } return; } - opCtx->recoveryUnit()->registerChange(std::make_unique<LogDeleteForShardingHandler>( - opCtx, nss, documentKey, opTime, preImageOpTime)); + auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); + auto cloner = MigrationSourceManager::getCurrentCloner(csr, csrLock); + + if (cloner && getIsMigrating(opCtx)) { + cloner->onDeleteOp(opCtx, documentKey, opTime, preImageOpTime); + } } void OpObserverShardingImpl::shardObserveTransactionPrepareOrUnpreparedCommit( @@ -226,14 +241,4 @@ void OpObserverShardingImpl::shardObserveTransactionPrepareOrUnpreparedCommit( *opCtx->getLogicalSessionId(), stmts, prepareOrCommitOptime)); } -void OpObserverShardingImpl::shardObserveNonPrimaryTransactionPrepare( - OperationContext* opCtx, - const std::vector<repl::OplogEntry>& stmts, - const repl::OpTime& prepareOrCommitOptime) { - - opCtx->recoveryUnit()->registerChange( - std::make_unique<LogTransactionOperationsForShardingHandler>( - *opCtx->getLogicalSessionId(), stmts, prepareOrCommitOptime)); -} - } // namespace mongo diff --git a/src/mongo/db/s/op_observer_sharding_impl.h b/src/mongo/db/s/op_observer_sharding_impl.h index c572cd8b64d..f9005497c57 100644 --- a/src/mongo/db/s/op_observer_sharding_impl.h +++ b/src/mongo/db/s/op_observer_sharding_impl.h @@ -36,6 +36,14 @@ namespace mongo { class ShardingWriteRouter; class OpObserverShardingImpl : public OpObserverImpl { +public: + // True if the document being deleted belongs to a chunk which, while still in the shard, + // is being migrated out. (Not to be confused with "fromMigrate", which tags operations + // that are steps in performing the migration.) + static bool isMigrating(OperationContext* opCtx, + NamespaceString const& nss, + BSONObj const& docToDelete); + protected: void shardObserveAboutToDelete(OperationContext* opCtx, NamespaceString const& nss, @@ -56,8 +64,8 @@ protected: const repl::OpTime& prePostImageOpTime, bool inMultiDocumentTransaction) override; void shardObserveDeleteOp(OperationContext* opCtx, - const NamespaceString& nss, - const repl::DocumentKey& documentKey, + NamespaceString nss, + const BSONObj& documentKey, const repl::OpTime& opTime, const ShardingWriteRouter& shardingWriteRouter, const repl::OpTime& preImageOpTime, @@ -66,10 +74,6 @@ protected: OperationContext* opCtx, const std::vector<repl::ReplOperation>& stmts, const repl::OpTime& prepareOrCommitOptime) override; - void shardObserveNonPrimaryTransactionPrepare( - OperationContext* opCtx, - const std::vector<repl::OplogEntry>& stmts, - const repl::OpTime& prepareOrCommitOptime) override; }; } // namespace mongo diff --git a/src/mongo/db/s/op_observer_sharding_test.cpp b/src/mongo/db/s/op_observer_sharding_test.cpp index 498051105ad..1ef41426f26 100644 --- a/src/mongo/db/s/op_observer_sharding_test.cpp +++ b/src/mongo/db/s/op_observer_sharding_test.cpp @@ -97,6 +97,7 @@ TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateUnsharded) { ASSERT_BSONOBJ_EQ(repl::getDocumentKey(operationContext(), kTestNss, doc).getShardKeyAndId(), BSON("_id" << "hello")); + ASSERT_FALSE(OpObserverShardingImpl::isMigrating(operationContext(), kTestNss, doc)); } TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithoutIdInShardKey) { @@ -123,6 +124,7 @@ TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithoutIdInShardKey) { << "abc" << "_id" << "hello")); + ASSERT_FALSE(OpObserverShardingImpl::isMigrating(operationContext(), kTestNss, doc)); } TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithIdInShardKey) { @@ -148,6 +150,7 @@ TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithIdInShardKey) { BSON("key" << 100 << "_id" << "hello" << "key2" << true)); + ASSERT_FALSE(OpObserverShardingImpl::isMigrating(operationContext(), kTestNss, doc)); } TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithIdHashInShardKey) { @@ -170,6 +173,7 @@ TEST_F(DocumentKeyStateTest, MakeDocumentKeyStateShardedWithIdHashInShardKey) { ASSERT_BSONOBJ_EQ(repl::getDocumentKey(operationContext(), kTestNss, doc).getShardKeyAndId(), BSON("_id" << "hello")); + ASSERT_FALSE(OpObserverShardingImpl::isMigrating(operationContext(), kTestNss, doc)); } diff --git a/src/mongo/db/s/operation_sharding_state.cpp b/src/mongo/db/s/operation_sharding_state.cpp index 6cb56a03795..0419fb1e895 100644 --- a/src/mongo/db/s/operation_sharding_state.cpp +++ b/src/mongo/db/s/operation_sharding_state.cpp @@ -60,64 +60,29 @@ void OperationShardingState::setShardRole(OperationContext* opCtx, const boost::optional<DatabaseVersion>& databaseVersion) { auto& oss = OperationShardingState::get(opCtx); - bool shardVersionInserted = false; - bool databaseVersionInserted = false; - try { - boost::optional<OperationShardingState::ShardVersionTracker&> shardVersionTracker; - if (shardVersion) { - auto emplaceResult = oss._shardVersions.try_emplace(nss.ns(), *shardVersion); - shardVersionInserted = emplaceResult.second; - shardVersionTracker = emplaceResult.first->second; - if (!shardVersionInserted) { - uassert(640570, - str::stream() - << "Illegal attempt to change the expected shard version for " << nss - << " from " << shardVersionTracker->v << " to " << *shardVersion - << " at recursion level " << shardVersionTracker->recursion, - shardVersionTracker->v == *shardVersion); - invariant(shardVersionTracker->recursion > 0); - } else { - invariant(shardVersionTracker->recursion == 0); - } - } - - boost::optional<OperationShardingState::DatabaseVersionTracker&> dbVersionTracker; - if (databaseVersion) { - auto emplaceResult = oss._databaseVersions.try_emplace(nss.db(), *databaseVersion); - databaseVersionInserted = emplaceResult.second; - dbVersionTracker = emplaceResult.first->second; - if (!databaseVersionInserted) { - uassert(640571, - str::stream() - << "Illegal attempt to change the expected database version for " - << nss.db() << " from " << dbVersionTracker->v << " to " - << *databaseVersion << " at recursion level " - << dbVersionTracker->recursion, - dbVersionTracker->v == *databaseVersion); - invariant(dbVersionTracker->recursion > 0); - } else { - invariant(dbVersionTracker->recursion == 0); - } + if (shardVersion) { + auto emplaceResult = oss._shardVersions.try_emplace(nss.ns(), *shardVersion); + auto& tracker = emplaceResult.first->second; + if (!emplaceResult.second) { + uassert(640570, + str::stream() << "Illegal attempt to change the expected shard version for " + << nss << " from " << tracker.v << " to " << *shardVersion, + tracker.v == *shardVersion); } + invariant(++tracker.recursion > 0); + } - // Update the recursion at the end to preserve the strong exception guarantee. - if (shardVersionTracker) { - shardVersionTracker->recursion++; - } - if (dbVersionTracker) { - dbVersionTracker->recursion++; + if (databaseVersion) { + auto emplaceResult = oss._databaseVersions.try_emplace(nss.db(), *databaseVersion); + auto& tracker = emplaceResult.first->second; + if (!emplaceResult.second) { + uassert(640571, + str::stream() << "Illegal attempt to change the expected database version for " + << nss.db() << " from " << tracker.v << " to " + << *databaseVersion, + tracker.v == *databaseVersion); } - } catch (const DBException&) { - // Clean any oss update done within this method on failure to get a strong exception - // guarantee on ScopedSetShardRole objects. - if (shardVersionInserted) { - oss._shardVersions.erase(nss.ns()); - } - if (databaseVersionInserted) { - oss._databaseVersions.erase(nss.db()); - } - - throw; + invariant(++tracker.recursion > 0); } } @@ -191,19 +156,17 @@ using ScopedAllowImplicitCollectionCreate_UNSAFE = OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE; ScopedAllowImplicitCollectionCreate_UNSAFE::ScopedAllowImplicitCollectionCreate_UNSAFE( - OperationContext* opCtx, bool forceCSRAsUnknownAfterCollectionCreation) + OperationContext* opCtx) : _opCtx(opCtx) { auto& oss = get(_opCtx); invariant(!oss._allowCollectionCreation); oss._allowCollectionCreation = true; - oss._forceCSRAsUnknownAfterCollectionCreation = forceCSRAsUnknownAfterCollectionCreation; } ScopedAllowImplicitCollectionCreate_UNSAFE::~ScopedAllowImplicitCollectionCreate_UNSAFE() { auto& oss = get(_opCtx); invariant(oss._allowCollectionCreation); oss._allowCollectionCreation = false; - oss._forceCSRAsUnknownAfterCollectionCreation = false; } ScopedSetShardRole::ScopedSetShardRole(OperationContext* opCtx, diff --git a/src/mongo/db/s/operation_sharding_state.h b/src/mongo/db/s/operation_sharding_state.h index e1d89bb8ff9..99bd516295e 100644 --- a/src/mongo/db/s/operation_sharding_state.h +++ b/src/mongo/db/s/operation_sharding_state.h @@ -96,17 +96,11 @@ public: * * Instantiating this object on the stack indicates to the storage execution subsystem that it * is allowed to create any collection in this context and that the caller will be responsible - * for notifying the shard Sharding subsystem of the collection creation. Note that in most of - * cases the CollectionShardingRuntime associated to that nss will be set as UNSHARDED. However, - * there are some scenarios in which it is required to set is as UNKNOWN: that's the reason why - * the constructor has the 'forceCSRAsUnknownAfterCollectionCreation' parameter. You can find - * more information about how the CSR is modified in ShardServerOpObserver::onCreateCollection. + * for notifying the shard Sharding sybsystem of the collection creation. */ class ScopedAllowImplicitCollectionCreate_UNSAFE { public: - /* Please read the comment associated to this class */ - ScopedAllowImplicitCollectionCreate_UNSAFE( - OperationContext* opCtx, bool forceCSRAsUnknownAfterCollectionCreation = false); + ScopedAllowImplicitCollectionCreate_UNSAFE(OperationContext* opCtx); ~ScopedAllowImplicitCollectionCreate_UNSAFE(); private: @@ -173,9 +167,6 @@ private: // Specifies whether the request is allowed to create database/collection implicitly bool _allowCollectionCreation{false}; - // Specifies whether the CollectionShardingRuntime should be set as unknown after collection - // creation - bool _forceCSRAsUnknownAfterCollectionCreation{false}; // Stores the shard version expected for each collection that will be accessed struct ShardVersionTracker { diff --git a/src/mongo/db/s/operation_sharding_state_test.cpp b/src/mongo/db/s/operation_sharding_state_test.cpp index ee61b75df6b..0c4732b51ab 100644 --- a/src/mongo/db/s/operation_sharding_state_test.cpp +++ b/src/mongo/db/s/operation_sharding_state_test.cpp @@ -84,29 +84,5 @@ TEST_F(OperationShardingStateTest, ScopedSetShardRoleRecursiveShardVersionDiffer ASSERT_EQ(shardVersion2, *oss.getShardVersion(kAnotherNss)); } -TEST_F(OperationShardingStateTest, ScopeSetShardRoleHasStrongExceptionGuarantee) { - ChunkVersion shardVersion(1, 0, OID::gen(), Timestamp(10, 0)); - auto uuid = UUID::gen(); - DatabaseVersion dbv1{uuid, Timestamp(1, 0)}; - DatabaseVersion dbv2{uuid, Timestamp(2, 0)}; - - auto& oss = OperationShardingState::get(operationContext()); - { - ScopedSetShardRole scopedSetShardRole1(operationContext(), kNss, shardVersion, dbv1); - ASSERT_THROWS_CODE( - [&] { - ScopedSetShardRole scopedSetShardRole2( - operationContext(), kNss, shardVersion, dbv2); - }(), - DBException, - 640571); - ASSERT_EQ(shardVersion, *oss.getShardVersion(kNss)); - ASSERT_EQ(dbv1, *oss.getDbVersion(kNss.db())); - } - - ASSERT_EQ(false, oss.getShardVersion(kNss).has_value()); - ASSERT_EQ(false, oss.getDbVersion(kNss.db()).has_value()); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/range_deletion_util.cpp b/src/mongo/db/s/range_deletion_util.cpp index 9ca0a8f2518..ee4562105d9 100644 --- a/src/mongo/db/s/range_deletion_util.cpp +++ b/src/mongo/db/s/range_deletion_util.cpp @@ -37,7 +37,7 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/exec/delete_stage.h" #include "mongo/db/exec/working_set_common.h" @@ -130,23 +130,12 @@ StatusWith<int> deleteNextBatch(OperationContext* opCtx, auto shardKeyIdx = findShardKeyPrefixedIndex( opCtx, collection, collection->getIndexCatalog(), keyPattern, /*requireSingleKey=*/false); if (!shardKeyIdx) { - // Do not log that the shard key is missing for hashed shard key patterns. - if (!ShardKeyPattern::isHashedPatternEl(keyPattern.firstElement())) { - LOGV2_ERROR(23765, - "Unable to find range shard key index", - "keyPattern"_attr = keyPattern, - logAttrs(nss)); - - // When a shard key index is not found, the range deleter moves the task to the bottom - // of the range deletion queue. This sleep is aimed at avoiding logging too aggressively - // in order to prevent log files to increase too much in size. - opCtx->sleepFor(Seconds(5)); - } - - iasserted(ErrorCodes::IndexNotFound, - str::stream() << "Unable to find shard key index" - << " for " << nss.ns() << " and key pattern `" - << keyPattern.toString() << "'"); + LOGV2_ERROR_OPTIONS(23765, + {logv2::UserAssertAfterLog(ErrorCodes::InternalError)}, + "Unable to find shard key index for {keyPattern} in {namespace}", + "Unable to find shard key index", + "keyPattern"_attr = keyPattern, + "namespace"_attr = nss.ns()); } // Extend bounds to match the index we found @@ -192,8 +181,7 @@ StatusWith<int> deleteNextBatch(OperationContext* opCtx, hangBeforeDoingDeletion.pauseWhileSet(opCtx); } - long long bytesDeleted = 0; - int numDocsDeleted = 0; + int numDeleted = 0; do { BSONObj deletedObj; @@ -228,14 +216,12 @@ StatusWith<int> deleteNextBatch(OperationContext* opCtx, break; } - bytesDeleted += deletedObj.objsize(); invariant(PlanExecutor::ADVANCED == state); - } while (++numDocsDeleted < numDocsToRemovePerBatch); + ShardingStatistics::get(opCtx).countDocsDeletedOnDonor.addAndFetch(1); - ShardingStatistics::get(opCtx).countDocsDeletedByRangeDeleter.addAndFetch(numDocsDeleted); - ShardingStatistics::get(opCtx).countBytesDeletedByRangeDeleter.addAndFetch(bytesDeleted); + } while (++numDeleted < numDocsToRemovePerBatch); - return numDocsDeleted; + return numDeleted; } template <typename Callable> @@ -295,9 +281,7 @@ void markRangeDeletionTaskAsProcessing(OperationContext* opCtx, const UUID& migr PersistentTaskStore<RangeDeletionTask> store(NamespaceString::kRangeDeletionNamespace); auto query = BSON(RangeDeletionTask::kIdFieldName << migrationId); static const auto update = - BSON("$set" << BSON(RangeDeletionTask::kProcessingFieldName - << true << RangeDeletionTask::kWhenToCleanFieldName - << CleanWhen_serializer(CleanWhenEnum::kNow))); + BSON("$set" << BSON(RangeDeletionTask::kProcessingFieldName << true)); store.update(opCtx, query, update, WriteConcerns::kLocalWriteConcern); } @@ -311,7 +295,9 @@ ExecutorFuture<void> deleteRangeInBatches(const std::shared_ptr<executor::TaskEx const UUID& collectionUuid, const BSONObj& keyPattern, const ChunkRange& range, - const UUID& migrationId) { + const UUID& migrationId, + int numDocsToRemovePerBatch, + Milliseconds delayBetweenBatches) { return ExecutorFuture<void>(executor) .then([=] { bool allDocsRemoved = false; @@ -322,13 +308,6 @@ ExecutorFuture<void> deleteRangeInBatches(const std::shared_ptr<executor::TaskEx try { allDocsRemoved = withTemporaryOperationContext( [=](OperationContext* opCtx) { - int numDocsToRemovePerBatch = rangeDeleterBatchSize.load(); - if (numDocsToRemovePerBatch <= 0) { - numDocsToRemovePerBatch = kRangeDeleterBatchSizeDefault; - } - - Milliseconds delayBetweenBatches(rangeDeleterBatchDelayMS.load()); - LOGV2_DEBUG(5346200, 1, "Starting batch deletion", @@ -342,7 +321,6 @@ ExecutorFuture<void> deleteRangeInBatches(const std::shared_ptr<executor::TaskEx int numDeleted; { - ScopedRangeDeleterLock rangeDeleterLock(opCtx, MODE_IX); AutoGetCollection collection(opCtx, nss, MODE_IX); // Ensure the collection exists and has not been dropped or dropped @@ -358,6 +336,8 @@ ExecutorFuture<void> deleteRangeInBatches(const std::shared_ptr<executor::TaskEx markRangeDeletionTaskAsProcessing(opCtx, migrationId); { + ScopedRangeDeleterLock rangeDeleterLock(opCtx, collectionUuid); + numDeleted = uassertStatusOK(deleteNextBatch(opCtx, collection.getCollection(), @@ -401,7 +381,6 @@ ExecutorFuture<void> deleteRangeInBatches(const std::shared_ptr<executor::TaskEx RangeDeletionAbandonedBecauseCollectionWithUUIDDoesNotExist || errorCode == ErrorCodes::RangeDeletionAbandonedBecauseTaskDocumentDoesNotExist || - errorCode == ErrorCodes::IndexNotFound || errorCode == ErrorCodes::KeyPatternShorterThanBound || ErrorCodes::isShutdownError(errorCode) || ErrorCodes::isNotPrimaryError(errorCode)) { @@ -548,6 +527,7 @@ SharedSemiFuture<void> removeDocumentsInRange( const BSONObj& keyPattern, const ChunkRange& range, const UUID& migrationId, + int numDocsToRemovePerBatch, Seconds delayForActiveQueriesOnSecondariesToComplete) { return std::move(waitForActiveQueriesToComplete) .thenRunOn(executor) @@ -565,14 +545,23 @@ SharedSemiFuture<void> removeDocumentsInRange( .then([=]() mutable { LOGV2_DEBUG(23772, 1, + "Beginning deletion of any documents in {namespace} range {range} with " + "numDocsToRemovePerBatch {numDocsToRemovePerBatch}", "Beginning deletion of documents", "namespace"_attr = nss.ns(), - "range"_attr = redact(range.toString())); + "range"_attr = redact(range.toString()), + "numDocsToRemovePerBatch"_attr = numDocsToRemovePerBatch); notifySecondariesThatDeletionIsOccurring(nss, collectionUuid, range); - return deleteRangeInBatches( - executor, nss, collectionUuid, keyPattern, range, migrationId) + return deleteRangeInBatches(executor, + nss, + collectionUuid, + keyPattern, + range, + migrationId, + numDocsToRemovePerBatch, + Milliseconds(rangeDeleterBatchDelayMS.load())) .onCompletion([=](Status s) { if (!s.isOK() && s.code() != @@ -604,7 +593,7 @@ SharedSemiFuture<void> removeDocumentsInRange( "Completed deletion of documents", "namespace"_attr = nss.ns(), "range"_attr = redact(range.toString())); - } else if (s.code() != ErrorCodes::IndexNotFound) { + } else { LOGV2(23774, "Failed to delete documents in {namespace} range {range} due to {error}", "Failed to delete documents", @@ -663,12 +652,12 @@ void setOrphanCountersOnRangeDeletionTasks(OperationContext* opCtx) { ShardingCatalogClient::kLocalWriteConcern); }; - ScopedRangeDeleterLock rangeDeleterLock(opCtx, MODE_X); store.forEach( opCtx, BSONObj(), [opCtx, &store, &setNumOrphansOnTask](const RangeDeletionTask& deletionTask) { AutoGetCollection collection(opCtx, deletionTask.getNss(), MODE_IX); + ScopedRangeDeleterLock rangeDeleterLock(opCtx, deletionTask.getCollectionUuid()); if (!collection || collection->uuid() != deletionTask.getCollectionUuid()) { // The deletion task is referring to a collection that has been dropped setNumOrphansOnTask(deletionTask, 0); @@ -714,7 +703,7 @@ void clearOrphanCountersFromRangeDeletionTasks(OperationContext* opCtx) { BSONObj allDocsQuery; PersistentTaskStore<RangeDeletionTask> store(NamespaceString::kRangeDeletionNamespace); try { - // TODO SERVER-65996 Remove writeConflictRetry loop + // TODO (SERVER-54284) Remove writeConflictRetry loop writeConflictRetry( opCtx, "clearOrphanCounters", NamespaceString::kRangeDeletionNamespace.ns(), [&] { store.update( @@ -728,4 +717,19 @@ void clearOrphanCountersFromRangeDeletionTasks(OperationContext* opCtx) { } } +ScopedRangeDeleterLock::ScopedRangeDeleterLock(OperationContext* opCtx) + : _configLock(opCtx, NamespaceString::kConfigDb, MODE_IX), + _rangeDeletionLock(opCtx, NamespaceString::kRangeDeletionNamespace, MODE_X) {} + +// Take DB and Collection lock in mode IX as well as collection UUID lock to serialize with +// operations that take the above version of the ScopedRangeDeleterLock such as FCV downgrade and +// BalancerStatsRegistry initialization. +ScopedRangeDeleterLock::ScopedRangeDeleterLock(OperationContext* opCtx, const UUID& collectionUuid) + : _configLock(opCtx, NamespaceString::kConfigDb, MODE_IX), + _rangeDeletionLock(opCtx, NamespaceString::kRangeDeletionNamespace, MODE_IX), + _collectionUuidLock(Lock::ResourceLock( + opCtx->lockState(), + ResourceId(RESOURCE_MUTEX, "RangeDeleterCollLock::" + collectionUuid.toString()), + MODE_X)) {} + } // namespace mongo diff --git a/src/mongo/db/s/range_deletion_util.h b/src/mongo/db/s/range_deletion_util.h index 5e5a17c652b..1c2fdffef1d 100644 --- a/src/mongo/db/s/range_deletion_util.h +++ b/src/mongo/db/s/range_deletion_util.h @@ -40,6 +40,21 @@ namespace mongo { /** + * Acquires the config db lock in IX mode and the collection lock for config.rangeDeletions in X + * mode. + */ +class ScopedRangeDeleterLock { +public: + ScopedRangeDeleterLock(OperationContext* opCtx); + ScopedRangeDeleterLock(OperationContext* opCtx, const UUID& collectionUuid); + +private: + Lock::DBLock _configLock; + Lock::CollectionLock _rangeDeletionLock; + boost::optional<Lock::ResourceLock> _collectionUuidLock; +}; + +/** * Deletes a range of orphaned documents for the given namespace and collection UUID. Returns a * future which will be resolved when the range has finished being deleted. The resulting future * will contain an error in cases where the range could not be deleted successfully. @@ -60,6 +75,7 @@ SharedSemiFuture<void> removeDocumentsInRange( const BSONObj& keyPattern, const ChunkRange& range, const UUID& migrationId, + int numDocsToRemovePerBatch, Seconds delayForActiveQueriesOnSecondariesToComplete); /** diff --git a/src/mongo/db/s/range_deletion_util_test.cpp b/src/mongo/db/s/range_deletion_util_test.cpp index 1a8154dffd8..b39e9541271 100644 --- a/src/mongo/db/s/range_deletion_util_test.cpp +++ b/src/mongo/db/s/range_deletion_util_test.cpp @@ -216,6 +216,7 @@ RangeDeletionTask insertRangeDeletionTask(OperationContext* opCtx, TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRemovesAllDocumentsInRangeWhenAllDocumentsFitInSingleBatch) { const ChunkRange range(BSON(kShardKey << 0), BSON(kShardKey << 10)); + const int numDocsToRemovePerBatch = 10; auto queriesComplete = SemiFuture<void>::makeReady(); setFilteringMetadataWithUUID(uuid()); @@ -231,6 +232,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -242,6 +244,7 @@ TEST_F(RangeDeleterTest, const ChunkRange range(BSON(kShardKey << 0), BSON(kShardKey << 10)); // More documents than the batch size. const auto numDocsToInsert = 3; + const auto numDocsToRemovePerBatch = 1; auto queriesComplete = SemiFuture<void>::makeReady(); // Insert documents in range. @@ -260,6 +263,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -268,6 +272,7 @@ TEST_F(RangeDeleterTest, TEST_F(RangeDeleterTest, RemoveDocumentsInRangeInsertsDocumentToNotifySecondariesOfRangeDeletion) { const ChunkRange range(BSON(kShardKey << 0), BSON(kShardKey << 10)); + const int numDocsToRemovePerBatch = 10; auto queriesComplete = SemiFuture<void>::makeReady(); setFilteringMetadataWithUUID(uuid()); @@ -283,6 +288,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeInsertsDocumentToNotifySecondarie kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -298,6 +304,7 @@ TEST_F( const ChunkRange range(BSON(kShardKey << 0), BSON(kShardKey << 10)); // More documents than the batch size. const auto numDocsToInsert = 3; + const auto numDocsToRemovePerBatch = 1; auto queriesComplete = SemiFuture<void>::makeReady(); // Insert documents in range. @@ -316,6 +323,7 @@ TEST_F( kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -349,6 +357,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, task.getId(), + 1 /* numDocsToRemovePerBatch */, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -381,6 +390,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, task.getId(), + 1 /* numDocsToRemovePerBatch */, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -410,6 +420,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, task.getId(), + 10 /* numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -432,6 +443,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeThrowsErrorWhenCollectionDoesNotE kShardKeyPattern, ChunkRange(BSON(kShardKey << 0), BSON(kShardKey << 10)), task.getId(), + 10 /* numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); @@ -475,6 +487,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeLeavesDocumentsWhenTaskDocumentDo kShardKeyPattern, range, UUID::gen(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -490,7 +503,6 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationAfterDeletingS const auto numDocsToInsert = 3; const auto numDocsToRemovePerBatch = 10; - rangeDeleterBatchSize.store(numDocsToRemovePerBatch); const auto numBatches = ceil((double)numDocsToInsert / numDocsToRemovePerBatch); ASSERT_EQ(numBatches, 1); // We should wait twice: Once after deleting documents in the range, and once after deleting the @@ -525,6 +537,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationAfterDeletingS kShardKeyPattern, range, t.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); cleanupComplete.get(); @@ -539,7 +552,6 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationOnlyOnceAfterS const auto numDocsToInsert = 3; const auto numDocsToRemovePerBatch = 1; - rangeDeleterBatchSize.store(numDocsToRemovePerBatch); const auto numBatches = ceil((double)numDocsToInsert / numDocsToRemovePerBatch); ASSERT_GTE(numBatches, 1); @@ -575,6 +587,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeWaitsForReplicationOnlyOnceAfterS kShardKeyPattern, range, t.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -588,6 +601,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotWaitForReplicationIfErrorD repl::ReplicationCoordinator::get(getServiceContext())); const auto numDocsToInsert = 3; + const auto numDocsToRemovePerBatch = 10; setFilteringMetadataWithUUID(uuid()); DBDirectClient dbclient(_opCtx); @@ -621,6 +635,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotWaitForReplicationIfErrorD kShardKeyPattern, range, t.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); ASSERT_THROWS_CODE(cleanupComplete.get(), DBException, ErrorCodes::PrimarySteppedDown); @@ -651,6 +666,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnWriteConflictException) kShardKeyPattern, range, t.getId(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -682,6 +698,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRetriesOnUnexpectedError) { kShardKeyPattern, range, t.getId(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -695,8 +712,6 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsDelayInBetweenBatches) { // More documents than the batch size. const auto numDocsToInsert = 3; const auto numDocsToRemovePerBatch = 1; - rangeDeleterBatchSize.store(numDocsToRemovePerBatch); - auto queriesComplete = SemiFuture<void>::makeReady(); // Insert documents in range. setFilteringMetadataWithUUID(uuid()); @@ -723,6 +738,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsDelayInBetweenBatches) { kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -737,6 +753,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsOrphanCleanupDelay) { const ChunkRange range(BSON(kShardKey << 0), BSON(kShardKey << 10)); // More documents than the batch size. const auto numDocsToInsert = 3; + const auto numDocsToRemovePerBatch = 1; const auto orphanCleanupDelay = Seconds(10); auto queriesComplete = SemiFuture<void>::makeReady(); @@ -755,6 +772,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRespectsOrphanCleanupDelay) { kShardKeyPattern, range, task.getId(), + numDocsToRemovePerBatch, orphanCleanupDelay); // A best-effort check that cleanup has not completed without advancing the clock. @@ -793,6 +811,7 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeRemovesRangeDeletionTaskOnSuccess kShardKeyPattern, range, t.getId(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); cleanupComplete.get(); @@ -823,6 +842,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, t.getId(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); ASSERT_THROWS_CODE(cleanupComplete.get(), @@ -860,6 +880,7 @@ TEST_F(RangeDeleterTest, kShardKeyPattern, range, t.getId(), + 10 /*numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); ASSERT_THROWS_CODE(cleanupComplete.get(), DBException, ErrorCodes::PrimarySteppedDown); @@ -891,6 +912,7 @@ DEATH_TEST_F(RangeDeleterTest, RemoveDocumentsInRangeCrashesIfInputFutureHasErro kShardKeyPattern, range, t.getId(), + 10 /* numDocsToRemovePerBatch */, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete */); @@ -901,14 +923,9 @@ DEATH_TEST_F(RangeDeleterTest, RemoveDocumentsInRangeCrashesIfInputFutureHasErro TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotCrashWhenShardKeyIndexDoesNotExist) { auto queriesComplete = SemiFuture<void>::makeReady(); - const std::string kNoShardKeyIndexMsg("Unable to find range shard key index"); - startCapturingLogMessages(); + const std::string kNoShardKeyIndexMsg("Unable to find shard key index for"); auto logCountBefore = countTextFormatLogLinesContaining(kNoShardKeyIndexMsg); - // Insert range deletion task for this collection and range. - const ChunkRange range(BSON("x" << 0), BSON("x" << 10)); - auto t = insertRangeDeletionTask(_opCtx, uuid(), range); - auto cleanupComplete = removeDocumentsInRange(executor(), std::move(queriesComplete), @@ -916,14 +933,20 @@ TEST_F(RangeDeleterTest, RemoveDocumentsInRangeDoesNotCrashWhenShardKeyIndexDoes uuid(), BSON("x" << 1) /* shard key pattern */, ChunkRange(BSON("x" << 0), BSON("x" << 10)), - t.getId(), + UUID::gen(), + 10 /* numDocsToRemovePerBatch*/, Seconds(0) /* delayForActiveQueriesOnSecondariesToComplete*/); - while (countTextFormatLogLinesContaining(kNoShardKeyIndexMsg) <= logCountBefore) { + // Range deleter will keep on retrying when it encounters non-stepdown errors. Make it run + // a few iterations and then create the index to make it exit the retry loop. + while (countTextFormatLogLinesContaining(kNoShardKeyIndexMsg) < logCountBefore) { sleepmicros(100); } - ASSERT_THROWS_CODE(cleanupComplete.get(), DBException, ErrorCodes::IndexNotFound); + DBDirectClient client(_opCtx); + client.createIndex(kNss.ns(), BSON("x" << 1)); + + cleanupComplete.get(); } /** diff --git a/src/mongo/db/s/read_only_catalog_cache_loader.h b/src/mongo/db/s/read_only_catalog_cache_loader.h index 24617c31197..ddd76aa8cef 100644 --- a/src/mongo/db/s/read_only_catalog_cache_loader.h +++ b/src/mongo/db/s/read_only_catalog_cache_loader.h @@ -46,7 +46,6 @@ public: void initializeReplicaSetRole(bool isPrimary) override {} void onStepDown() override {} void onStepUp() override {} - void onReplicationRollback() override {} void shutDown() override; void notifyOfCollectionVersionUpdate(const NamespaceString& nss) override {} void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override; diff --git a/src/mongo/db/s/recoverable_critical_section_service.cpp b/src/mongo/db/s/recoverable_critical_section_service.cpp index 95214c05857..39406025140 100644 --- a/src/mongo/db/s/recoverable_critical_section_service.cpp +++ b/src/mongo/db/s/recoverable_critical_section_service.cpp @@ -29,7 +29,6 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding -#include <fmt/format.h> #include <set> #include "mongo/platform/basic.h" @@ -93,12 +92,7 @@ void RecoverableCriticalSectionService::acquireRecoverableCriticalSectionBlockWr "reason"_attr = reason, "writeConcern"_attr = writeConcern); - tassert(7032360, - fmt::format("Can't acquire recoverable critical section for collection '{}' with " - "reason '{}' while holding locks", - nss.toString(), - reason.toString()), - !opCtx->lockState()->isLocked()); + invariant(!opCtx->lockState()->isLocked()); { Lock::GlobalLock lk(opCtx, MODE_IX); @@ -116,14 +110,12 @@ void RecoverableCriticalSectionService::acquireRecoverableCriticalSectionBlockWr const auto collCSDoc = CollectionCriticalSectionDocument::parse( IDLParserErrorContext("AcquireRecoverableCSBW"), bsonObj); - tassert(7032368, - fmt::format("Trying to acquire a critical section blocking writes for " - "namespace '{}' and reason '{}' but it is already taken by another " - "operation with different reason '{}'", - nss.toString(), - reason.toString(), - collCSDoc.getReason().toString()), - collCSDoc.getReason().woCompare(reason) == 0); + invariant(collCSDoc.getReason().woCompare(reason) == 0, + str::stream() + << "Trying to acquire a critical section blocking writes for namespace " + << nss << " and reason " << reason + << " but it is already taken by another operation with different reason " + << collCSDoc.getReason()); LOGV2_DEBUG( 5656601, @@ -157,13 +149,10 @@ void RecoverableCriticalSectionService::acquireRecoverableCriticalSectionBlockWr BatchedCommandResponse batchedResponse; std::string unusedErrmsg; batchedResponse.parseBSON(commandReply, &unusedErrmsg); - tassert(7032369, - fmt::format("Insert did not add any doc to collection '{}' for namespace '{}' " - "and reason '{}'", - nss.toString(), - reason.toString(), - NamespaceString::kCollectionCriticalSectionsNamespace.toString()), - batchedResponse.getN() > 0); + invariant(batchedResponse.getN() > 0, + str::stream() << "Insert did not add any doc to collection " + << NamespaceString::kCollectionCriticalSectionsNamespace + << " for namespace " << nss << " and reason " << reason); } WriteConcernResult ignoreResult; @@ -190,12 +179,7 @@ void RecoverableCriticalSectionService::promoteRecoverableCriticalSectionToBlock "reason"_attr = reason, "writeConcern"_attr = writeConcern); - tassert(7032364, - fmt::format("Can't promote recoverable critical section for collection '{}' with " - "reason '{}' while holding locks", - nss.toString(), - reason.toString()), - !opCtx->lockState()->isLocked()); + invariant(!opCtx->lockState()->isLocked()); { AutoGetCollection cCollLock(opCtx, nss, MODE_X); @@ -206,26 +190,21 @@ void RecoverableCriticalSectionService::promoteRecoverableCriticalSectionToBlock BSON(CollectionCriticalSectionDocument::kNssFieldName << nss.toString())); auto cursor = dbClient.find(std::move(findRequest)); - tassert(7032361, - fmt::format( - "Trying to acquire a critical section blocking reads for namespace '{}' and " - "reason '{}' but the critical section wasn't acquired first blocking writers.", - nss.toString(), - reason.toString()), - cursor->more()); + invariant( + cursor->more(), + str::stream() << "Trying to acquire a critical section blocking reads for namespace " + << nss << " and reason " << reason + << " but the critical section wasn't acquired first blocking writers."); BSONObj bsonObj = cursor->next(); const auto collCSDoc = CollectionCriticalSectionDocument::parse( IDLParserErrorContext("AcquireRecoverableCSBR"), bsonObj); - tassert(7032362, - fmt::format( - "Trying to acquire a critical section blocking reads for namespace '{}' and " - "reason " - "'{}' but it is already taken by another operation with different reason '{}'", - nss.toString(), - reason.toString(), - collCSDoc.getReason().toString()), - collCSDoc.getReason().woCompare(reason) == 0); + invariant( + collCSDoc.getReason().woCompare(reason) == 0, + str::stream() << "Trying to acquire a critical section blocking reads for namespace " + << nss << " and reason " << reason + << " but it is already taken by another operation with different reason " + << collCSDoc.getReason()); // if there is a document with the same nss, reason and blocking reads -> do nothing, the CS // is already taken! @@ -270,13 +249,10 @@ void RecoverableCriticalSectionService::promoteRecoverableCriticalSectionToBlock BatchedCommandResponse batchedResponse; std::string unusedErrmsg; batchedResponse.parseBSON(commandReply, &unusedErrmsg); - tassert(7032363, - fmt::format("Update did not modify any doc from collection '{}' for namespace '{}' " - "and reason '{}'", - NamespaceString::kCollectionCriticalSectionsNamespace.toString(), - nss.toString(), - reason.toString()), - batchedResponse.getNModified() > 0); + invariant(batchedResponse.getNModified() > 0, + str::stream() << "Update did not modify any doc from collection " + << NamespaceString::kCollectionCriticalSectionsNamespace + << " for namespace " << nss << " and reason " << reason); } WriteConcernResult ignoreResult; @@ -295,8 +271,7 @@ void RecoverableCriticalSectionService::releaseRecoverableCriticalSection( OperationContext* opCtx, const NamespaceString& nss, const BSONObj& reason, - const WriteConcernOptions& writeConcern, - bool throwIfReasonDiffers) { + const WriteConcernOptions& writeConcern) { LOGV2_DEBUG(5656606, 3, "Releasing recoverable critical section", @@ -304,12 +279,7 @@ void RecoverableCriticalSectionService::releaseRecoverableCriticalSection( "reason"_attr = reason, "writeConcern"_attr = writeConcern); - tassert(7032365, - fmt::format("Can't release recoverable critical section for collection '{}' with " - "reason '{}' while holding locks", - nss.toString(), - reason.toString()), - !opCtx->lockState()->isLocked()); + invariant(!opCtx->lockState()->isLocked()); { AutoGetCollection collLock(opCtx, nss, MODE_X); @@ -337,26 +307,12 @@ void RecoverableCriticalSectionService::releaseRecoverableCriticalSection( const auto collCSDoc = CollectionCriticalSectionDocument::parse( IDLParserErrorContext("ReleaseRecoverableCS"), bsonObj); - const bool isDifferentReason = collCSDoc.getReason().woCompare(reason) != 0; - if (MONGO_unlikely(!throwIfReasonDiffers && isDifferentReason)) { - LOGV2_DEBUG(7019701, - 2, - "Impossible to release recoverable critical section since it was taken by " - "another operation with different reason", - "namespace"_attr = nss, - "callerReason"_attr = reason, - "storedReason"_attr = collCSDoc.getReason(), - "writeConcern"_attr = writeConcern); - return; - } - - tassert(7032366, - fmt::format("Trying to release a critical for namespace '{}' and reason '{}' but " - "it is already taken by another operation with different reason '{}'", - nss.toString(), - reason.toString(), - collCSDoc.getReason().toString()), - !isDifferentReason); + invariant( + collCSDoc.getReason().woCompare(reason) == 0, + str::stream() << "Trying to release a critical for namespace " << nss << " and reason " + << reason + << " but it is already taken by another operation with different reason " + << collCSDoc.getReason()); // The collection critical section is taken (in any phase), try to release it. @@ -386,13 +342,10 @@ void RecoverableCriticalSectionService::releaseRecoverableCriticalSection( BatchedCommandResponse batchedResponse; std::string unusedErrmsg; batchedResponse.parseBSON(commandReply, &unusedErrmsg); - tassert(7032367, - fmt::format("Delete did not remove any doc from collection '{}' for namespace '{}' " - "and reason '{}'", - NamespaceString::kCollectionCriticalSectionsNamespace.toString(), - nss.toString(), - reason.toString()), - batchedResponse.getN() > 0); + invariant(batchedResponse.getN() > 0, + str::stream() << "Delete did not remove any doc from collection " + << NamespaceString::kCollectionCriticalSectionsNamespace + << " for namespace " << nss << " and reason " << reason); } WriteConcernResult ignoreResult; diff --git a/src/mongo/db/s/recoverable_critical_section_service.h b/src/mongo/db/s/recoverable_critical_section_service.h index 621e3b5cdde..d1953b27942 100644 --- a/src/mongo/db/s/recoverable_critical_section_service.h +++ b/src/mongo/db/s/recoverable_critical_section_service.h @@ -57,8 +57,6 @@ public: * specified namespace and reason. It works even if the namespace's current metadata are * UNKNOWN. * - * Entering into the Critical Section interrupts any ongoing filtering metadata refresh. - * * It adds a doc to config.collectionCriticalSections with with writeConcern write concern. * * Do nothing if the collection critical section is taken for that nss and reason, and will @@ -89,20 +87,14 @@ public: /** * Releases the recoverable critical section for the given nss and reason. * - * It removes a doc from config.collectionCriticalSections with writeConcern write concern. As - * part of the removal, the filtering information is cleared on secondary nodes. It is - * responsability of the caller to properly set the filtering information on the primary node. + * It removes a doc from config.collectionCriticalSections with writeConcern write concern. * * Do nothing if the collection critical section is not taken for that nss and reason. - * - * Throw an invariant in case the collection critical section is already taken by another - * operation with a different reason unless the flag 'throwIfReasonDiffers' is set to false. */ void releaseRecoverableCriticalSection(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& reason, - const WriteConcernOptions& writeConcern, - bool throwIfReasonDiffers = true); + const WriteConcernOptions& writeConcern); /** diff --git a/src/mongo/db/s/rename_collection_coordinator.cpp b/src/mongo/db/s/rename_collection_coordinator.cpp index 60293c51d89..59048ce4c0f 100644 --- a/src/mongo/db/s/rename_collection_coordinator.cpp +++ b/src/mongo/db/s/rename_collection_coordinator.cpp @@ -40,7 +40,6 @@ #include "mongo/db/ops/insert.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/s/collection_sharding_runtime.h" -#include "mongo/db/s/recoverable_critical_section_service.h" #include "mongo/db/s/sharding_ddl_util.h" #include "mongo/db/s/sharding_logging.h" #include "mongo/db/s/sharding_state.h" @@ -168,7 +167,7 @@ ExecutorFuture<void> RenameCollectionCoordinator::_runImpl( return ExecutorFuture<void>(**executor) .then(_executePhase( Phase::kCheckPreconditions, - [this, executor = executor, anchor = shared_from_this()] { + [this, anchor = shared_from_this()] { auto opCtxHolder = cc().makeOperationContext(); auto* opCtx = opCtxHolder.get(); getForwardableOpMetadata().setOn(opCtx); @@ -176,10 +175,6 @@ ExecutorFuture<void> RenameCollectionCoordinator::_runImpl( const auto& fromNss = nss(); const auto& toNss = _request.getTo(); - const auto criticalSectionReason = - sharding_ddl_util::getCriticalSectionReasonForRename(fromNss, toNss); - - bool isCriticalSectionAcquired = false; try { uassert(ErrorCodes::InvalidOptions, "Cannot provide an expected collection UUID when renaming between " @@ -190,23 +185,12 @@ ExecutorFuture<void> RenameCollectionCoordinator::_runImpl( { AutoGetCollection coll{ opCtx, fromNss, MODE_IS, AutoGetCollectionViewMode::kViewsPermitted}; - - uassert(ErrorCodes::CommandNotSupportedOnView, - str::stream() << "Can't rename source collection `" << fromNss - << "` because it is a view.", - !coll.getView()); - checkCollectionUUIDMismatch( opCtx, fromNss, *coll, _doc.getExpectedSourceUUID()); - uassert(ErrorCodes::NamespaceNotFound, - str::stream() << "Collection " << fromNss << " doesn't exist.", - coll.getCollection()); - uassert(ErrorCodes::IllegalOperation, "Cannot rename an encrypted collection", - !coll || !coll->getCollectionOptions().encryptedFieldConfig || - _doc.getAllowEncryptedCollectionRename().value_or(false)); + !coll || !coll->getCollectionOptions().encryptedFieldConfig); } // Make sure the source collection exists @@ -225,82 +209,32 @@ ExecutorFuture<void> RenameCollectionCoordinator::_runImpl( sharding_ddl_util::checkDbPrimariesOnTheSameShard(opCtx, fromNss, toNss); } + // Make sure the target namespace is not a view + { + uassert(ErrorCodes::CommandNotSupportedOnView, + str::stream() << "Can't rename to target collection `" << toNss + << "` because it is a view.", + !CollectionCatalog::get(opCtx)->lookupView(opCtx, toNss)); + } + const auto optTargetCollType = getShardedCollection(opCtx, toNss); - const bool targetIsSharded = (bool)optTargetCollType; - _doc.setTargetIsSharded(targetIsSharded); + _doc.setTargetIsSharded((bool)optTargetCollType); _doc.setTargetUUID(getCollectionUUID( opCtx, toNss, optTargetCollType, /*throwNotFound*/ false)); - if (!targetIsSharded) { - auto criticalSection = RecoverableCriticalSectionService::get(opCtx); - { - // (SERVER-67325) Acquire critical section on the target collection in - // order to disallow concurrent `createCollection`. In case the - // collection does not exist, it will be later released by the rename - // participant. In case the collection exists and is unsharded, the - // critical section can be released right away as the participant will - // re-acquire it when needed. - try { - - criticalSection->acquireRecoverableCriticalSectionBlockWrites( - opCtx, - toNss, - criticalSectionReason, - ShardingCatalogClient::kLocalWriteConcern); - isCriticalSectionAcquired = true; - criticalSection->promoteRecoverableCriticalSectionToBlockAlsoReads( - opCtx, - toNss, - criticalSectionReason, - ShardingCatalogClient::kLocalWriteConcern); - - } catch (const ExceptionFor<ErrorCodes::CommandNotSupportedOnView>&) { - - // Target namespace should never be a view - // We forcely throw NamespaceExists for compatibility with - // replicaset - uasserted(ErrorCodes::NamespaceExists, - str::stream() - << "a view already exists with that name: " << toNss); - } - } - - if (CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, - toNss)) { - // Release the critical section because the unsharded target collection - // already exists, hence no risk of concurrent `createCollection` - criticalSection->releaseRecoverableCriticalSection( - opCtx, - toNss, - criticalSectionReason, - WriteConcerns::kLocalWriteConcern); - } - } sharding_ddl_util::checkRenamePreconditions( opCtx, sourceIsSharded, toNss, _doc.getDropTarget()); - sharding_ddl_util::checkCatalogConsistencyAcrossShardsForRename( - opCtx, fromNss, toNss, _doc.getDropTarget(), executor); - { AutoGetCollection coll{opCtx, toNss, MODE_IS}; checkCollectionUUIDMismatch( opCtx, toNss, *coll, _doc.getExpectedTargetUUID()); uassert(ErrorCodes::IllegalOperation, "Cannot rename to an existing encrypted collection", - !coll || !coll->getCollectionOptions().encryptedFieldConfig || - _doc.getAllowEncryptedCollectionRename().value_or(false)); + !coll || !coll->getCollectionOptions().encryptedFieldConfig); } } catch (const DBException&) { - auto criticalSection = RecoverableCriticalSectionService::get(opCtx); - if (isCriticalSectionAcquired) - criticalSection->releaseRecoverableCriticalSection( - opCtx, - toNss, - criticalSectionReason, - WriteConcerns::kLocalWriteConcern, - false /* throwIfReasonDiffers */); _completeOnError = true; throw; } diff --git a/src/mongo/db/s/rename_collection_participant_service.cpp b/src/mongo/db/s/rename_collection_participant_service.cpp index 92341d1035f..e99c16dff44 100644 --- a/src/mongo/db/s/rename_collection_participant_service.cpp +++ b/src/mongo/db/s/rename_collection_participant_service.cpp @@ -34,7 +34,6 @@ #include "mongo/base/checked_cast.h" #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/catalog/rename_collection.h" -#include "mongo/db/catalog_raii.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/database_sharding_state.h" @@ -44,7 +43,6 @@ #include "mongo/db/s/recoverable_critical_section_service.h" #include "mongo/db/s/rename_collection_participant_service.h" #include "mongo/db/s/shard_metadata_util.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/logv2/log.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/grid.h" @@ -60,11 +58,29 @@ const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); * Drop the collection locally and clear stale metadata from cache collections. */ void dropCollectionLocally(OperationContext* opCtx, const NamespaceString& nss) { - DropCollectionCoordinator::dropCollectionLocally(opCtx, nss, false /* fromMigrate */); + bool knownNss = [&]() { + try { + DropCollectionCoordinator::dropCollectionLocally(opCtx, nss); + return true; + } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { + return false; + } + }(); + LOGV2_DEBUG(5515100, 1, - "Dropped target collection locally on renameCollection participant.", - "namespace"_attr = nss); + "Dropped target collection locally on renameCollection participant", + "namespace"_attr = nss, + "collectionExisted"_attr = knownNss); +} + +/* Clear the CollectionShardingRuntime entry for the specified namespace */ +void clearFilteringMetadata(OperationContext* opCtx, const NamespaceString& nss) { + UninterruptibleLockGuard noInterrupt(opCtx->lockState()); + Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); + Lock::CollectionLock collLock(opCtx, nss, MODE_IX); + auto* csr = CollectionShardingRuntime::get(opCtx, nss); + csr->clearFilteringMetadata(opCtx); } /* @@ -117,16 +133,6 @@ void renameOrDropTarget(OperationContext* opCtx, deleteRangeDeletionTasksForRename(opCtx, fromNss, toNss); } } - -void clearFilteringMetadataOnNss(OperationContext* opCtx, const NamespaceString& nss) { - // Set the placement version to UNKNOWN to force a future operation to refresh the metadata - // TODO (SERVER-71444): Fix to be interruptible or document exception. - UninterruptibleLockGuard noInterrupt(opCtx->lockState()); // NOLINT. - AutoGetCollection autoColl(opCtx, nss, MODE_IX); - auto* csr = CollectionShardingRuntime::get(opCtx, nss); - csr->clearFilteringMetadata(opCtx); -} - } // namespace RenameCollectionParticipantService* RenameCollectionParticipantService::getService( @@ -301,7 +307,9 @@ SemiFuture<void> RenameParticipantInstance::_runImpl( // Acquire source/target critical sections const auto reason = - sharding_ddl_util::getCriticalSectionReasonForRename(fromNss(), toNss()); + BSON("command" + << "rename" + << "from" << fromNss().toString() << "to" << toNss().toString()); auto service = RecoverableCriticalSectionService::get(opCtx); service->acquireRecoverableCriticalSectionBlockWrites( opCtx, fromNss(), reason, ShardingCatalogClient::kLocalWriteConcern); @@ -316,19 +324,8 @@ SemiFuture<void> RenameParticipantInstance::_runImpl( // recovered the next time is accessed) and to safely create new range deletion // tasks (the submission will serialize on the renamed collection's metadata // refresh). - { - Lock::DBLock dbLock(opCtx, fromNss().db(), MODE_IX); - Lock::CollectionLock collLock(opCtx, fromNss(), MODE_IX); - auto* csr = CollectionShardingRuntime::get(opCtx, fromNss()); - csr->clearFilteringMetadataForDroppedCollection(opCtx); - } - - { - Lock::DBLock dbLock(opCtx, toNss().db(), MODE_IX); - Lock::CollectionLock collLock(opCtx, toNss(), MODE_IX); - auto* csr = CollectionShardingRuntime::get(opCtx, toNss()); - csr->clearFilteringMetadata(opCtx); - } + clearFilteringMetadata(opCtx, fromNss()); + clearFilteringMetadata(opCtx, toNss()); snapshotRangeDeletionsForRename(opCtx, fromNss(), toNss()); })) @@ -380,9 +377,6 @@ SemiFuture<void> RenameParticipantInstance::_runImpl( auto opCtxHolder = cc().makeOperationContext(); auto* opCtx = opCtxHolder.get(); - clearFilteringMetadataOnNss(opCtx, fromNss()); - clearFilteringMetadataOnNss(opCtx, toNss()); - // Release source/target critical sections const auto reason = BSON("command" diff --git a/src/mongo/db/s/reshard_collection_coordinator.cpp b/src/mongo/db/s/reshard_collection_coordinator.cpp index 33477548f42..802a85cfe1d 100644 --- a/src/mongo/db/s/reshard_collection_coordinator.cpp +++ b/src/mongo/db/s/reshard_collection_coordinator.cpp @@ -32,9 +32,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/catalog/collection_uuid_mismatch.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/db_raii.h" -#include "mongo/db/op_observer.h" #include "mongo/db/s/reshard_collection_coordinator.h" #include "mongo/logv2/log.h" #include "mongo/s/grid.h" @@ -43,62 +41,6 @@ namespace mongo { -namespace { - -void notifyChangeStreamsOnReshardCollectionComplete(OperationContext* opCtx, - const NamespaceString& collNss, - const ReshardCollectionCoordinatorDocument& doc, - const UUID& reshardUUID) { - - const std::string oMessage = str::stream() - << "Reshard collection " << collNss << " with shard key " << doc.getKey().toString(); - - BSONObjBuilder cmdBuilder; - tassert(6590800, "Did not set old collectionUUID", doc.getOldCollectionUUID()); - tassert(6590801, "Did not set old ShardKey", doc.getOldShardKey()); - UUID collUUID = *doc.getOldCollectionUUID(); - cmdBuilder.append("reshardCollection", collNss.ns()); - reshardUUID.appendToBuilder(&cmdBuilder, "reshardUUID"); - cmdBuilder.append("shardKey", doc.getKey()); - cmdBuilder.append("oldShardKey", *doc.getOldShardKey()); - - cmdBuilder.append("unique", doc.getUnique().get_value_or(false)); - if (doc.getNumInitialChunks()) { - cmdBuilder.append("numInitialChunks", doc.getNumInitialChunks().get()); - } - if (doc.getCollation()) { - cmdBuilder.append("collation", doc.getCollation().get()); - } - - if (doc.getZones()) { - BSONArrayBuilder zonesBSON(cmdBuilder.subarrayStart("zones")); - for (const auto& zone : *doc.getZones()) { - zonesBSON.append(zone.toBSON()); - } - zonesBSON.doneFast(); - } - - auto const serviceContext = opCtx->getClient()->getServiceContext(); - - const auto cmd = cmdBuilder.obj(); - - writeConflictRetry(opCtx, "ReshardCollection", NamespaceString::kRsOplogNamespace.ns(), [&] { - AutoGetOplog oplogWrite(opCtx, OplogAccessMode::kWrite); - WriteUnitOfWork uow(opCtx); - serviceContext->getOpObserver()->onInternalOpMessage(opCtx, - collNss, - collUUID, - BSON("msg" << oMessage), - cmd, - boost::none, - boost::none, - boost::none, - boost::none); - uow.commit(); - }); -} -} // namespace - ReshardCollectionCoordinator::ReshardCollectionCoordinator(ShardingDDLCoordinatorService* service, const BSONObj& initialState) : ReshardCollectionCoordinator(service, initialState, true /* persistCoordinatorDocument */) {} @@ -187,20 +129,6 @@ ExecutorFuture<void> ReshardCollectionCoordinator::_runImpl( checkCollectionUUIDMismatch(opCtx, nss(), *coll, _doc.getCollectionUUID()); } - const auto cmOld = uassertStatusOK( - Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh( - opCtx, nss())); - - if (_persistCoordinatorDocument) { - StateDoc newDoc(_doc); - newDoc.setOldShardKey(cmOld.getShardKeyPattern().getKeyPattern().toBSON()); - newDoc.setOldCollectionUUID(cmOld.getUUID()); - _doc = _updateStateDocument(opCtx, std::move(newDoc)); - } else { - _doc.setOldShardKey(cmOld.getShardKeyPattern().getKeyPattern().toBSON()); - _doc.setOldCollectionUUID(cmOld.getUUID()); - } - ConfigsvrReshardCollection configsvrReshardCollection(nss(), _doc.getKey()); configsvrReshardCollection.setDbName(nss().db()); configsvrReshardCollection.setUnique(_doc.getUnique()); @@ -221,16 +149,6 @@ ExecutorFuture<void> ReshardCollectionCoordinator::_runImpl( configsvrReshardCollection.toBSON({}), opCtx->getWriteConcern()), Shard::RetryPolicy::kIdempotent)); uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(std::move(cmdResponse))); - - // Report command completion to the oplog. - const auto cm = uassertStatusOK( - Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh( - opCtx, nss())); - - if (_doc.getOldCollectionUUID() && _doc.getOldCollectionUUID() != cm.getUUID()) { - notifyChangeStreamsOnReshardCollectionComplete( - opCtx, nss(), _doc, cm.getUUID()); - } })) .onError([this, anchor = shared_from_this()](const Status& status) { LOGV2_ERROR(6206401, diff --git a/src/mongo/db/s/reshard_collection_coordinator_document.idl b/src/mongo/db/s/reshard_collection_coordinator_document.idl index e3d9134a8ac..2d3032957b6 100644 --- a/src/mongo/db/s/reshard_collection_coordinator_document.idl +++ b/src/mongo/db/s/reshard_collection_coordinator_document.idl @@ -59,11 +59,3 @@ structs: type: ReshardCollectionCoordinatorPhase description: "Coordinator phase." default: kUnset - oldShardKey: - type: object - description: "The shard key of the collection before resharding." - optional: true - oldCollectionUUID: - type: uuid - description: "The UUID of the collection before resharding." - optional: true diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp index 2695ed842c0..75cb7be7049 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.cpp @@ -101,7 +101,8 @@ StageConstraints DocumentSourceReshardingAddResumeId::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceReshardingAddResumeId::serialize(const SerializationOptions& opts) const { +Value DocumentSourceReshardingAddResumeId::serialize( + boost::optional<ExplainOptions::Verbosity> explain) const { return Value(Document{{kStageName, Value(Document{})}}); } diff --git a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h index 4fb27980c68..31cbd97c694 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h +++ b/src/mongo/db/s/resharding/document_source_resharding_add_resume_id.h @@ -53,7 +53,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp index 8260cf3e4cc..9d412147b7f 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.cpp @@ -112,7 +112,7 @@ StageConstraints DocumentSourceReshardingIterateTransaction::constraints( } Value DocumentSourceReshardingIterateTransaction::serialize( - const SerializationOptions& opts) const { + boost::optional<ExplainOptions::Verbosity> explain) const { return Value( Document{{kStageName, Value(Document{{kIncludeCommitTransactionTimestampFieldName, @@ -134,7 +134,7 @@ DepsTracker::State DocumentSourceReshardingIterateTransaction::getDependencies( DocumentSource::GetModPathsReturn DocumentSourceReshardingIterateTransaction::getModifiedPaths() const { - return {DocumentSource::GetModPathsReturn::Type::kAllPaths, OrderedPathSet{}, {}}; + return {DocumentSource::GetModPathsReturn::Type::kAllPaths, std::set<std::string>{}, {}}; } DocumentSource::GetNextResult DocumentSourceReshardingIterateTransaction::doGetNext() { diff --git a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h index 8c792116e6c..9589cb64a08 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h +++ b/src/mongo/db/s/resharding/document_source_resharding_iterate_transaction.h @@ -66,7 +66,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp index 157876391d3..f8a186e3bcd 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp +++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.cpp @@ -90,11 +90,12 @@ StageConstraints DocumentSourceReshardingOwnershipMatch::constraints( ChangeStreamRequirement::kDenylist); } -Value DocumentSourceReshardingOwnershipMatch::serialize(const SerializationOptions& opts) const { +Value DocumentSourceReshardingOwnershipMatch::serialize( + boost::optional<ExplainOptions::Verbosity> explain) const { return Value{Document{{kStageName, DocumentSourceReshardingOwnershipMatchSpec( _recipientShardId, _reshardingKey.getKeyPattern()) - .toBSON(opts)}}}; + .toBSON()}}}; } DepsTracker::State DocumentSourceReshardingOwnershipMatch::getDependencies( @@ -108,7 +109,7 @@ DepsTracker::State DocumentSourceReshardingOwnershipMatch::getDependencies( DocumentSource::GetModPathsReturn DocumentSourceReshardingOwnershipMatch::getModifiedPaths() const { // This stage does not modify or rename any paths. - return {DocumentSource::GetModPathsReturn::Type::kFiniteSet, OrderedPathSet{}, {}}; + return {DocumentSource::GetModPathsReturn::Type::kFiniteSet, std::set<std::string>{}, {}}; } DocumentSource::GetNextResult DocumentSourceReshardingOwnershipMatch::doGetNext() { diff --git a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h index b7da07a5a57..7a6db2bc125 100644 --- a/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h +++ b/src/mongo/db/s/resharding/document_source_resharding_ownership_match.h @@ -58,7 +58,7 @@ public: DocumentSource::GetModPathsReturn getModifiedPaths() const final; - Value serialize(const SerializationOptions& opts = SerializationOptions{}) const final override; + Value serialize(boost::optional<ExplainOptions::Verbosity> explain) const final; StageConstraints constraints(Pipeline::SplitState pipeState) const final; diff --git a/src/mongo/db/s/resharding/recipient_document.idl b/src/mongo/db/s/resharding/recipient_document.idl index 1eda4620b8c..e3128de0db5 100644 --- a/src/mongo/db/s/resharding/recipient_document.idl +++ b/src/mongo/db/s/resharding/recipient_document.idl @@ -82,11 +82,6 @@ structs: startConfigTxnCloneTime: type: date optional: true - approxBytesToCopy: - type: long - description: >- - Approximate number of bytes to copy during cloning - optional: true metrics: type: ReshardingRecipientMetrics description: "Metrics related to this recipient." diff --git a/src/mongo/db/s/resharding/resharding_agg_test.cpp b/src/mongo/db/s/resharding/resharding_agg_test.cpp index 99ad259c7d0..f2c5345e907 100644 --- a/src/mongo/db/s/resharding/resharding_agg_test.cpp +++ b/src/mongo/db/s/resharding/resharding_agg_test.cpp @@ -239,7 +239,6 @@ repl::DurableOplogEntry makeApplyOpsOplog(std::vector<BSONObj> operations, {}, UUID::gen(), false /* fromMigrate */, - boost::none, // checkExistenceForDiffInsert 0 /* version */, applyOpsBuilder.obj(), /* o */ boost::none, /* o2 */ @@ -1452,7 +1451,6 @@ TEST_F(ReshardingAggWithStorageTest, RetryableFindAndModifyWithImageLookup) { kCrudNs, kCrudUUID, false /* fromMigrate */, - boost::none, // checkExistenceForDiffInsert 0 /* version */, BSON("$set" << BSON("y" << 1)), /* o1 */ BSON("_id" << 2), /* o2 */ diff --git a/src/mongo/db/s/resharding/resharding_collection_cloner.cpp b/src/mongo/db/s/resharding/resharding_collection_cloner.cpp index e51d2e2ed1f..9132438053a 100644 --- a/src/mongo/db/s/resharding/resharding_collection_cloner.cpp +++ b/src/mongo/db/s/resharding/resharding_collection_cloner.cpp @@ -29,6 +29,8 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kResharding +#include "mongo/platform/basic.h" + #include "mongo/db/s/resharding/resharding_collection_cloner.h" #include <utility> @@ -37,6 +39,7 @@ #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/client.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/logical_session_id_helpers.h" @@ -274,7 +277,7 @@ bool ReshardingCollectionCloner::doOneBatch(OperationContext* opCtx, Pipeline& p // ReshardingOpObserver depends on the collection metadata being known when processing writes to // the temporary resharding collection. We attach shard version IGNORED to the insert operations - // and retry once on a StaleConfig error to allow the collection metadata information to be + // and retry once on a StaleConfig exception to allow the collection metadata information to be // recovered. ScopedSetShardRole scopedSetShardRole(opCtx, _outputNss, diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp index 5700f0326ae..374dcd6538f 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor.cpp @@ -164,21 +164,13 @@ CoordinatorCommitMonitor::queryRemainingOperationTimeForRecipients() const { uassertStatusOKWithContext(status, errorContext); const auto remainingTime = extractOperationRemainingTime(shardResponse.data); - - // If any recipient omits the "remainingMillis" field of the response then - // we cannot conclude that it is safe to begin the critical section. - // It is possible that the recipient just had a failover and - // was not able to restore its metrics before it replied to the - // _shardsvrReshardingOperationTime command. - if (!remainingTime) { - maxRemainingTime = Milliseconds::max(); - continue; - } - if (remainingTime.value() < minRemainingTime) { - minRemainingTime = remainingTime.value(); + // A recipient shard does not report the remaining operation time when there is no data + // to copy and no oplog entry to apply. + if (remainingTime && remainingTime.get() < minRemainingTime) { + minRemainingTime = remainingTime.get(); } - if (remainingTime.value() > maxRemainingTime) { - maxRemainingTime = remainingTime.value(); + if (remainingTime && remainingTime.get() > maxRemainingTime) { + maxRemainingTime = remainingTime.get(); } } @@ -211,20 +203,12 @@ ExecutorFuture<void> CoordinatorCommitMonitor::_makeFuture() const { "Encountered an error while querying recipients, will retry shortly", "error"_attr = status); - // On error we definitely cannot begin the critical section. Therefore, - // return Milliseconds::max for remainingTimes.max (remainingTimes.max is used - // for determining whether the critical section should begin). - return RemainingOperationTimes{Milliseconds(-1), Milliseconds::max()}; + return RemainingOperationTimes{Milliseconds(0), Milliseconds::max()}; }) .then([this, anchor = shared_from_this()](RemainingOperationTimes remainingTimes) { auto metrics = ReshardingMetrics::get(cc().getServiceContext()); - // If remainingTimes.max (or remainingTimes.min) is Milliseconds::max, then use -1 so - // that the scale of the y-axis is still useful when looking at FTDC metrics. - auto clampIfMax = [](Milliseconds t) { - return t != Milliseconds::max() ? t : Milliseconds(-1); - }; - metrics->setMinRemainingOperationTime(clampIfMax(remainingTimes.min)); - metrics->setMaxRemainingOperationTime(clampIfMax(remainingTimes.max)); + metrics->setMinRemainingOperationTime(remainingTimes.min); + metrics->setMaxRemainingOperationTime(remainingTimes.max); // Check if all recipient shards are within the commit threshold. if (remainingTimes.max <= _threshold) diff --git a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp index 2fe3075f1fc..e7f90cc41fa 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_commit_monitor_test.cpp @@ -92,8 +92,6 @@ protected: void tearDown() override; void mockCommandForRecipients(Milliseconds remainingOperationTime); - void mockOmitRemainingMillisForRecipients(); - void mockOmitRemainingMillisForOneRecipient(); void mockRemaingOperationTimesCommandForRecipients( CoordinatorCommitMonitor::RemainingOperationTimes remainingOperationTimes); @@ -182,31 +180,6 @@ void CoordinatorCommitMonitorTest::mockCommandForRecipients(Milliseconds remaini _recipientShards.begin(), _recipientShards.end(), [&](const ShardId&) { onCommand(func); }); } -void CoordinatorCommitMonitorTest::mockOmitRemainingMillisForRecipients() { - // Omit remainingMillis from all shard responses. - std::for_each(_recipientShards.begin(), _recipientShards.end(), [this](const ShardId&) { - onCommand([](const executor::RemoteCommandRequest& request) -> StatusWith<BSONObj> { - // Return an empty BSON object. - return BSONObj(); - }); - }); -} - -void CoordinatorCommitMonitorTest::mockOmitRemainingMillisForOneRecipient() { - // Omit remainingMillis from a single recipient. - for (const auto& shard : _recipientShards) { - onCommand([&](const executor::RemoteCommandRequest&) -> StatusWith<BSONObj> { - if (shard == _recipientShards.front()) { - // Return an empty BSON object. - return BSONObj(); - } - auto threshold = Milliseconds(gRemainingReshardingOperationTimeThresholdMillis.load()); - return BSON("remainingMillis" - << durationCount<Milliseconds>(threshold - Milliseconds(1))); - }); - } -} - void CoordinatorCommitMonitorTest::mockRemaingOperationTimesCommandForRecipients( CoordinatorCommitMonitor::RemainingOperationTimes remainingOperationTimes) { bool useMin = true; @@ -291,20 +264,6 @@ TEST_F(CoordinatorCommitMonitorTest, RetriesWhenEncountersErrorsWhileQueryingRec future.get(); } -TEST_F(CoordinatorCommitMonitorTest, BlocksWhenRemainingMillisIsOmitted) { - auto future = getCommitMonitor()->waitUntilRecipientsAreWithinCommitThreshold(); - - mockOmitRemainingMillisForRecipients(); - ASSERT(!future.isReady()); - - // If even a single shard omits remainingMillis, we cannot begin the critical section. - mockOmitRemainingMillisForOneRecipient(); - ASSERT(!future.isReady()); - - respondWithReadyToCommit(); - future.get(); -} - } // namespace } // namespace resharding } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index ea3451f1c4a..85c14b6478f 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -37,7 +37,6 @@ #include "mongo/bson/bsonobj.h" #include "mongo/bson/json.h" #include "mongo/db/auth/authorization_session_impl.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/logical_session_cache.h" #include "mongo/db/ops/write_ops.h" @@ -95,7 +94,6 @@ MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeStartingErrorFlow); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforePersistingStateTransition); MONGO_FAIL_POINT_DEFINE(pauseBeforeTellDonorToRefresh); MONGO_FAIL_POINT_DEFINE(pauseBeforeInsertCoordinatorDoc); -MONGO_FAIL_POINT_DEFINE(pauseBeforeCTHolderInitialization); const std::string kReshardingCoordinatorActiveIndexName = "ReshardingCoordinatorActiveIndex"; const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); @@ -111,14 +109,14 @@ Date_t getCurrentTime() { return svcCtx->getFastClockSource()->now(); } -void assertNumDocsMatchedEqualsExpected(const BatchedCommandRequest& request, - const BSONObj& response, - int expected) { - auto numDocsMatched = response.getIntField("n"); +void assertNumDocsModifiedMatchesExpected(const BatchedCommandRequest& request, + const BSONObj& response, + int expected) { + auto numDocsModified = response.getIntField("n"); uassert(5030401, str::stream() << "Expected to match " << expected << " docs, but only matched " - << numDocsMatched << " for write request " << request.toString(), - expected == numDocsMatched); + << numDocsModified << " for write request " << request.toString(), + expected == numDocsModified); } void appendShardEntriesToSetBuilder(const ReshardingCoordinatorDocument& coordinatorDoc, @@ -217,15 +215,15 @@ void writeToCoordinatorStateNss(OperationContext* opCtx, } }()); - auto expectedNumMatched = (request.getBatchType() == BatchedCommandRequest::BatchType_Insert) + auto expectedNumModified = (request.getBatchType() == BatchedCommandRequest::BatchType_Insert) ? boost::none : boost::make_optional(1); auto res = ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( opCtx, NamespaceString::kConfigReshardingOperationsNamespace, request, txnNumber); - if (expectedNumMatched) { - assertNumDocsMatchedEqualsExpected(request, res, *expectedNumMatched); + if (expectedNumModified) { + assertNumDocsModifiedMatchesExpected(request, res, *expectedNumModified); } } @@ -387,7 +385,7 @@ void updateConfigCollectionsForOriginalNss(OperationContext* opCtx, auto res = ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( opCtx, CollectionType::ConfigNS, request, txnNumber); - assertNumDocsMatchedEqualsExpected(request, res, 1 /* expected */); + assertNumDocsModifiedMatchesExpected(request, res, 1 /* expected */); } void writeToConfigCollectionsForTempNss(OperationContext* opCtx, @@ -478,15 +476,15 @@ void writeToConfigCollectionsForTempNss(OperationContext* opCtx, } }()); - auto expectedNumMatched = (request.getBatchType() == BatchedCommandRequest::BatchType_Insert) + auto expectedNumModified = (request.getBatchType() == BatchedCommandRequest::BatchType_Insert) ? boost::none : boost::make_optional(1); auto res = ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( opCtx, CollectionType::ConfigNS, request, txnNumber); - if (expectedNumMatched) { - assertNumDocsMatchedEqualsExpected(request, res, *expectedNumMatched); + if (expectedNumModified) { + assertNumDocsModifiedMatchesExpected(request, res, *expectedNumModified); } } @@ -507,33 +505,43 @@ void insertChunkAndTagDocsForTempNss(OperationContext* opCtx, ShardingCatalogManager::get(opCtx)->insertConfigDocuments(opCtx, TagsType::ConfigNS, newZones); } -void removeTagsDocs(OperationContext* opCtx, const BSONObj& tagsQuery, TxnNumber txnNumber) { - // Remove tag documents with the specified tagsQuery. - const auto tagDeleteOperationHint = BSON(TagsType::ns() << 1 << TagsType::min() << 1); - ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( - opCtx, - TagsType::ConfigNS, - BatchedCommandRequest::buildDeleteOp(TagsType::ConfigNS, - tagsQuery, // query - true, // multi - tagDeleteOperationHint // hint - ), - txnNumber); -} - // Requires that there be no session information on the opCtx. void removeChunkAndTagsDocs(OperationContext* opCtx, const BSONObj& tagsQuery, const UUID& collUUID) { - // Remove all chunk documents and specified tag documents. - resharding::removeChunkDocs(opCtx, collUUID); - + // Remove all chunk documents for the original nss. We do not know how many chunk docs + // currently exist, so cannot pass a value for expectedNumModified + const auto chunksQuery = BSON(ChunkType::collectionUUID() << collUUID); const auto tagDeleteOperationHint = BSON(TagsType::ns() << 1 << TagsType::min() << 1); + const auto catalogClient = Grid::get(opCtx)->catalogClient(); + + uassertStatusOK(catalogClient->removeConfigDocuments( + opCtx, ChunkType::ConfigNS, chunksQuery, kMajorityWriteConcern)); uassertStatusOK(catalogClient->removeConfigDocuments( opCtx, TagsType::ConfigNS, tagsQuery, kMajorityWriteConcern, tagDeleteOperationHint)); } +void updateChunkAndTagsDocsForTempNss(OperationContext* opCtx, + const ReshardingCoordinatorDocument& coordinatorDoc, + OID newCollectionEpoch, + TxnNumber txnNumber) { + auto hint = BSON("ns" << 1 << "min" << 1); + auto tagsRequest = BatchedCommandRequest::buildUpdateOp( + TagsType::ConfigNS, + BSON(TagsType::ns(coordinatorDoc.getTempReshardingNss().ns())), // query + BSON("$set" << BSON("ns" << coordinatorDoc.getSourceNss().ns())), // update + false, // upsert + true, // multi + hint // hint + ); + + // Update the 'ns' field to be the original collection namespace for all tags documents that + // currently have 'ns' as the temporary collection namespace + auto tagsRes = ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( + opCtx, TagsType::ConfigNS, tagsRequest, txnNumber); +} + /** * Executes metadata changes in a transaction without bumping the collection version. */ @@ -584,14 +592,28 @@ CollectionType createTempReshardingCollectionType( return collType; } -void removeChunkDocs(OperationContext* opCtx, const UUID& collUUID) { - // Remove all chunk documents for the specified collUUID. We do not know how many chunk docs - // currently exist, so cannot pass a value for expectedNumModified - const auto chunksQuery = BSON(ChunkType::collectionUUID() << collUUID); - const auto catalogClient = Grid::get(opCtx)->catalogClient(); +void cleanupSourceConfigCollections(OperationContext* opCtx, + const ReshardingCoordinatorDocument& coordinatorDoc) { + using Doc = Document; + using Arr = std::vector<Value>; + using V = Value; - uassertStatusOK(catalogClient->removeConfigDocuments( - opCtx, ChunkType::ConfigNS, chunksQuery, kMajorityWriteConcern)); + auto createTagFilter = [](const V value) { + return V{Doc{{"$map", + V{Doc{{"input", V{Doc{{"$objectToArray", value}}}}, + {"in", V{StringData("$$this.k")}}}}}}}; + }; + + + auto skipNewTagsFilter = Doc{ + {"$ne", + Arr{createTagFilter(V{StringData("$min")}), + createTagFilter(V{Doc{{"$literal", coordinatorDoc.getReshardingKey().toBSON()}}})}}}; + + const auto removeTagsQuery = + BSON(TagsType::ns(coordinatorDoc.getSourceNss().ns()) << "$expr" << skipNewTagsFilter); + + removeChunkAndTagsDocs(opCtx, removeTagsQuery, coordinatorDoc.getSourceUUID()); } void writeDecisionPersistedState(OperationContext* opCtx, @@ -600,49 +622,21 @@ void writeDecisionPersistedState(OperationContext* opCtx, Timestamp newCollectionTimestamp) { // No need to bump originalNss version because its epoch will be changed. - executeMetadataChangesInTxn( - opCtx, - [&coordinatorDoc, &newCollectionEpoch, &newCollectionTimestamp](OperationContext* opCtx, - TxnNumber txnNumber) { - // Update the config.reshardingOperations entry - writeToCoordinatorStateNss(opCtx, coordinatorDoc, txnNumber); + executeMetadataChangesInTxn(opCtx, [&](OperationContext* opCtx, TxnNumber txnNumber) { + // Update the config.reshardingOperations entry + writeToCoordinatorStateNss(opCtx, coordinatorDoc, txnNumber); - // Remove the config.collections entry for the temporary collection - writeToConfigCollectionsForTempNss( - opCtx, coordinatorDoc, boost::none, boost::none, txnNumber); - - // Update the config.collections entry for the original namespace to reflect the new - // shard key, new epoch, and new UUID - updateConfigCollectionsForOriginalNss( - opCtx, coordinatorDoc, newCollectionEpoch, newCollectionTimestamp, txnNumber); + // Remove the config.collections entry for the temporary collection + writeToConfigCollectionsForTempNss( + opCtx, coordinatorDoc, boost::none, boost::none, txnNumber); - // Delete all of the config.tags entries for the user collection namespace. - const auto removeTagsQuery = BSON(TagsType::ns(coordinatorDoc.getSourceNss().ns())); - removeTagsDocs(opCtx, removeTagsQuery, txnNumber); + // Update the config.collections entry for the original namespace to reflect the new + // shard key, new epoch, and new UUID + updateConfigCollectionsForOriginalNss( + opCtx, coordinatorDoc, newCollectionEpoch, newCollectionTimestamp, txnNumber); - // Update all of the config.tags entries for the temporary resharding namespace - // to refer to the user collection namespace. - updateTagsDocsForTempNss(opCtx, coordinatorDoc, txnNumber); - }); -} - -void updateTagsDocsForTempNss(OperationContext* opCtx, - const ReshardingCoordinatorDocument& coordinatorDoc, - TxnNumber txnNumber) { - auto hint = BSON("ns" << 1 << "min" << 1); - auto tagsRequest = BatchedCommandRequest::buildUpdateOp( - TagsType::ConfigNS, - BSON(TagsType::ns(coordinatorDoc.getTempReshardingNss().ns())), // query - BSON("$set" << BSON("ns" << coordinatorDoc.getSourceNss().ns())), // update - false, // upsert - true, // multi - hint // hint - ); - - // Update the 'ns' field to be the original collection namespace for all tags documents that - // currently have 'ns' as the temporary collection namespace. - auto tagsRes = ShardingCatalogManager::get(opCtx)->writeToConfigDocumentInTxn( - opCtx, TagsType::ConfigNS, tagsRequest, txnNumber); + updateChunkAndTagsDocsForTempNss(opCtx, coordinatorDoc, newCollectionEpoch, txnNumber); + }); } void insertCoordDocAndChangeOrigCollEntry(OperationContext* opCtx, @@ -754,21 +748,16 @@ void writeStateTransitionAndCatalogUpdatesThenBumpShardVersions( ShardingCatalogClient::kLocalWriteConcern); } -ReshardingCoordinatorDocument removeCoordinatorDocAndReshardingFields( - OperationContext* opCtx, - const ReshardingCoordinatorDocument& coordinatorDoc, - boost::optional<Status> abortReason) { +void removeCoordinatorDocAndReshardingFields(OperationContext* opCtx, + const ReshardingCoordinatorDocument& coordinatorDoc, + boost::optional<Status> abortReason) { // If the coordinator needs to abort and isn't in kInitializing, additional collections need to // be cleaned up in the final transaction. Otherwise, cleanup for abort and success are the // same. const bool wasDecisionPersisted = - coordinatorDoc.getState() >= CoordinatorStateEnum::kCommitting; + coordinatorDoc.getState() == CoordinatorStateEnum::kCommitting; invariant((wasDecisionPersisted && !abortReason) || abortReason); - if (coordinatorDoc.getState() > CoordinatorStateEnum::kCommitting) { - return coordinatorDoc; - } - ReshardingCoordinatorDocument updatedCoordinatorDoc = coordinatorDoc; updatedCoordinatorDoc.setState(CoordinatorStateEnum::kDone); emplaceTruncatedAbortReasonIfExists(updatedCoordinatorDoc, abortReason); @@ -802,7 +791,6 @@ ReshardingCoordinatorDocument removeCoordinatorDocAndReshardingFields( opCtx, updatedCoordinatorDoc, boost::none, boost::none, txnNumber); }, ShardingCatalogClient::kLocalWriteConcern); - return updatedCoordinatorDoc; } } // namespace resharding @@ -855,12 +843,7 @@ ReshardingCoordinatorExternalStateImpl::calculateParticipantShardsAndChunks( // The database primary must always be a recipient to ensure it ends up with consistent // collection metadata. - const auto dbPrimaryShard = - uassertStatusOK(Grid::get(opCtx)->catalogCache()->getDatabaseWithRefresh( - opCtx, coordinatorDoc.getSourceNss().db())) - ->getPrimary(); - - recipientShardIds.emplace(dbPrimaryShard); + recipientShardIds.emplace(cm.dbPrimary()); if (const auto& chunks = coordinatorDoc.getPresetReshardedChunks()) { auto version = calculateChunkVersionForInitialChunks(opCtx); @@ -1089,7 +1072,6 @@ ReshardingCoordinatorService::ReshardingCoordinator::_tellAllParticipantsReshard _cancelableOpCtxFactory.emplace(_ctHolder->getStepdownToken(), _markKilledExecutor); }) - .then([this] { return _waitForMajority(_ctHolder->getStepdownToken()); }) .then([this, executor]() { pauseBeforeTellDonorToRefresh.pauseWhileSet(); _establishAllDonorsAsParticipants(executor); @@ -1120,7 +1102,8 @@ ExecutorFuture<void> ReshardingCoordinatorService::ReshardingCoordinator::_initi return resharding::WithAutomaticRetry([this, executor] { return ExecutorFuture<void>(**executor) .then([this, executor] { _insertCoordDocAndChangeOrigCollEntry(); }) - .then([this, executor] { _calculateParticipantsAndChunksThenWriteToDisk(); }); + .then([this, executor] { _calculateParticipantsAndChunksThenWriteToDisk(); }) + .then([this] { return _waitForMajority(_ctHolder->getAbortToken()); }); }) .onTransientError([](const Status& status) { LOGV2(5093703, @@ -1259,11 +1242,33 @@ ReshardingCoordinatorService::ReshardingCoordinator::_commitAndFinishReshardOper const ReshardingCoordinatorDocument& updatedCoordinatorDoc) noexcept { return resharding::WithAutomaticRetry([this, executor, updatedCoordinatorDoc] { return ExecutorFuture<void>(**executor) - .then( - [this, executor, updatedCoordinatorDoc] { _commit(updatedCoordinatorDoc); }); + .then([this, executor, updatedCoordinatorDoc] { + return _commit(updatedCoordinatorDoc); + }) + .then([this] { return _waitForMajority(_ctHolder->getStepdownToken()); }) + .thenRunOn(**executor) + .then([this, executor] { + _tellAllParticipantsToCommit(_coordinatorDoc.getSourceNss(), executor); + }) + .then([this] { _updateChunkImbalanceMetrics(_coordinatorDoc.getSourceNss()); }) + .then([this, updatedCoordinatorDoc] { + auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); + resharding::cleanupSourceConfigCollections(opCtx.get(), + updatedCoordinatorDoc); + return Status::OK(); + }) + .then([this, executor] { return _awaitAllParticipantShardsDone(executor); }) + .then([this, executor] { + // Best-effort attempt to trigger a refresh on the participant shards so + // they see the collection metadata without reshardingFields and no longer + // throw ReshardCollectionInProgress. There is no guarantee this logic ever + // runs if the config server primary steps down after having removed the + // coordinator state document. + return _tellAllRecipientsToRefresh(executor); + }); }) .onTransientError([](const Status& status) { - LOGV2(7698801, + LOGV2(5093705, "Resharding coordinator encountered transient error while committing", "error"_attr = status); }) @@ -1271,83 +1276,25 @@ ReshardingCoordinatorService::ReshardingCoordinator::_commitAndFinishReshardOper .until<Status>([](const Status& status) { return status.isOK(); }) .on(**executor, _ctHolder->getStepdownToken()) .onError([this, executor](Status status) { - if (status == ErrorCodes::TransactionTooLargeForCache) { - return _onAbortCoordinatorAndParticipants(executor, status); + { + auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); + reshardingPauseCoordinatorBeforeStartingErrorFlow.pauseWhileSet(opCtx.get()); } - return ExecutorFuture<void>(**executor, status); - }) - .then([this, executor, updatedCoordinatorDoc] { - return resharding::WithAutomaticRetry([this, executor, updatedCoordinatorDoc] { - return ExecutorFuture<void>(**executor) - .then([this] { return _waitForMajority(_ctHolder->getStepdownToken()); }) - .thenRunOn(**executor) - .then([this, executor] { - _tellAllParticipantsToCommit(_coordinatorDoc.getSourceNss(), - executor); - }) - .then([this] { - _updateChunkImbalanceMetrics(_coordinatorDoc.getSourceNss()); - }) - .then([this, updatedCoordinatorDoc] { - auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - resharding::removeChunkDocs(opCtx.get(), - updatedCoordinatorDoc.getSourceUUID()); - return Status::OK(); - }) - .then([this, executor] { - return _awaitAllParticipantShardsDone(executor); - }) - .then([this, executor] { - // Best-effort attempt to trigger a refresh on the participant shards - // so they see the collection metadata without reshardingFields and - // no longer throw ReshardCollectionInProgress. There is no guarantee - // this logic ever runs if the config server primary steps down after - // having removed the coordinator state document. - return _tellAllRecipientsToRefresh(executor); - }); - }) - .onTransientError([](const Status& status) { - LOGV2(5093705, - "Resharding coordinator encountered transient error while committing", - "error"_attr = status); - }) - .onUnrecoverableError([](const Status& status) {}) - .until<Status>([](const Status& status) { return status.isOK(); }) - .on(**executor, _ctHolder->getStepdownToken()) - .onError([this, executor](Status status) { - { - auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - reshardingPauseCoordinatorBeforeStartingErrorFlow.pauseWhileSet( - opCtx.get()); - } - if (_ctHolder->isSteppingOrShuttingDown()) { - return status; - } + if (_ctHolder->isSteppingOrShuttingDown()) { + return status; + } - LOGV2_FATAL( - 5277000, + LOGV2_FATAL(5277000, "Unrecoverable error past the point resharding was guaranteed to succeed", "error"_attr = redact(status)); - }); }); } SemiFuture<void> ReshardingCoordinatorService::ReshardingCoordinator::run( std::shared_ptr<executor::ScopedTaskExecutor> executor, const CancellationToken& stepdownToken) noexcept { - pauseBeforeCTHolderInitialization.pauseWhileSet(); - - auto abortCalled = [&] { - stdx::lock_guard<Latch> lk(_abortCalledMutex); - _ctHolder = std::make_unique<CoordinatorCancellationTokenHolder>(stepdownToken); - return _abortCalled; - }(); - - if (abortCalled) { - _ctHolder->abort(); - } - + _ctHolder = std::make_unique<CoordinatorCancellationTokenHolder>(stepdownToken); _markKilledExecutor->startup(); _cancelableOpCtxFactory.emplace(_ctHolder->getAbortToken(), _markKilledExecutor); @@ -1358,15 +1305,8 @@ SemiFuture<void> ReshardingCoordinatorService::ReshardingCoordinator::run( }) .onCompletion([this, executor](Status status) { auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - reshardingPauseCoordinatorBeforeCompletion.executeIf( - [&](const BSONObj&) { - reshardingPauseCoordinatorBeforeCompletion.pauseWhileSetAndNotCanceled( - opCtx.get(), _ctHolder->getStepdownToken()); - }, - [&](const BSONObj& data) { - auto ns = data.getStringField("sourceNamespace"); - return ns.empty() ? true : ns.toString() == _coordinatorDoc.getSourceNss().ns(); - }); + reshardingPauseCoordinatorBeforeCompletion.pauseWhileSetAndNotCanceled( + opCtx.get(), _ctHolder->getStepdownToken()); { auto lg = stdx::lock_guard(_fulfillmentMutex); @@ -1407,8 +1347,6 @@ SemiFuture<void> ReshardingCoordinatorService::ReshardingCoordinator::run( ->onStepDown(ReshardingMetrics::Role::kCoordinator); } - _logStatsOnCompletion(status.isOK()); - if (!status.isOK()) { { auto lg = stdx::lock_guard(_fulfillmentMutex); @@ -1505,15 +1443,7 @@ ReshardingCoordinatorService::ReshardingCoordinator::_onAbortCoordinatorAndParti } void ReshardingCoordinatorService::ReshardingCoordinator::abort() { - auto ctHolderInitialized = [&] { - stdx::lock_guard<Latch> lk(_abortCalledMutex); - _abortCalled = true; - return !(_ctHolder == nullptr); - }(); - - if (ctHolderInitialized) { - _ctHolder->abort(); - } + _ctHolder->abort(); } boost::optional<BSONObj> ReshardingCoordinatorService::ReshardingCoordinator::reportForCurrentOp( @@ -1612,12 +1542,10 @@ void ReshardingCoordinatorService::ReshardingCoordinator:: // Remove the presetReshardedChunks and zones from the coordinator document to reduce // the possibility of the document reaching the BSONObj size constraint. - ShardKeyPattern shardKey(updatedCoordinatorDoc.getReshardingKey()); std::vector<BSONObj> zones; if (updatedCoordinatorDoc.getZones()) { zones = buildTagsDocsFromZones(updatedCoordinatorDoc.getTempReshardingNss(), - *updatedCoordinatorDoc.getZones(), - shardKey); + *updatedCoordinatorDoc.getZones()); } updatedCoordinatorDoc.setPresetReshardedChunks(boost::none); updatedCoordinatorDoc.setZones(boost::none); @@ -1811,11 +1739,11 @@ ReshardingCoordinatorService::ReshardingCoordinator::_awaitAllRecipientsInStrict .thenRunOn(**executor); } -void ReshardingCoordinatorService::ReshardingCoordinator::_commit( +Future<void> ReshardingCoordinatorService::ReshardingCoordinator::_commit( const ReshardingCoordinatorDocument& coordinatorDoc) { if (_coordinatorDoc.getState() > CoordinatorStateEnum::kBlockingWrites) { invariant(_coordinatorDoc.getState() != CoordinatorStateEnum::kAborting); - return; + return Status::OK(); } ReshardingCoordinatorDocument updatedCoordinatorDoc = coordinatorDoc; @@ -1841,6 +1769,8 @@ void ReshardingCoordinatorService::ReshardingCoordinator::_commit( // Update the in memory state installCoordinatorDoc(opCtx.get(), updatedCoordinatorDoc); + + return Status::OK(); } ExecutorFuture<void> @@ -2073,22 +2003,4 @@ void ReshardingCoordinatorService::ReshardingCoordinator::_updateChunkImbalanceM } } -void ReshardingCoordinatorService::ReshardingCoordinator::_logStatsOnCompletion(bool success) { - BSONObjBuilder builder; - BSONObjBuilder statsBuilder; - builder.append("uuid", _coordinatorDoc.getReshardingUUID().toBSON()); - builder.append("status", success ? "success" : "failed"); - statsBuilder.append("ns", _coordinatorDoc.getSourceNss().toString()); - statsBuilder.append("sourceUUID", _coordinatorDoc.getSourceUUID().toBSON()); - statsBuilder.append("newUUID", _coordinatorDoc.getReshardingUUID().toBSON()); - statsBuilder.append("newShardKey", _coordinatorDoc.getReshardingKey().toBSON()); - if (_coordinatorDoc.getStartTime()) { - statsBuilder.append("startTime", *_coordinatorDoc.getStartTime()); - } - statsBuilder.append("endTime", getCurrentTime()); - - builder.append("statistics", statsBuilder.obj()); - LOGV2(7763800, "Resharding complete", "info"_attr = builder.obj()); -} - } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.h b/src/mongo/db/s/resharding/resharding_coordinator_service.h index 694070c126c..a24569ecc44 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.h +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.h @@ -50,17 +50,14 @@ CollectionType createTempReshardingCollectionType( const ChunkVersion& chunkVersion, const BSONObj& collation); -void removeChunkDocs(OperationContext* opCtx, const UUID& collUUID); +void cleanupSourceConfigCollections(OperationContext* opCtx, + const ReshardingCoordinatorDocument& coordinatorDoc); void writeDecisionPersistedState(OperationContext* opCtx, const ReshardingCoordinatorDocument& coordinatorDoc, OID newCollectionEpoch, Timestamp newCollectionTimestamp); -void updateTagsDocsForTempNss(OperationContext* opCtx, - const ReshardingCoordinatorDocument& coordinatorDoc, - TxnNumber txnNumber); - void insertCoordDocAndChangeOrigCollEntry(OperationContext* opCtx, const ReshardingCoordinatorDocument& coordinatorDoc); @@ -72,10 +69,9 @@ void writeParticipantShardsAndTempCollInfo(OperationContext* opCtx, void writeStateTransitionAndCatalogUpdatesThenBumpShardVersions( OperationContext* opCtx, const ReshardingCoordinatorDocument& coordinatorDoc); -ReshardingCoordinatorDocument removeCoordinatorDocAndReshardingFields( - OperationContext* opCtx, - const ReshardingCoordinatorDocument& coordinatorDoc, - boost::optional<Status> abortReason = boost::none); +void removeCoordinatorDocAndReshardingFields(OperationContext* opCtx, + const ReshardingCoordinatorDocument& coordinatorDoc, + boost::optional<Status> abortReason = boost::none); } // namespace resharding class ReshardingCoordinatorExternalState { @@ -408,10 +404,11 @@ private: * Does the following writes: * 1. Updates the config.collections entry for the new sharded collection * 2. Updates config.chunks entries for the new sharded collection + * 3. Updates config.tags for the new sharded collection * * Transitions to 'kCommitting'. */ - void _commit(const ReshardingCoordinatorDocument& updatedDoc); + Future<void> _commit(const ReshardingCoordinatorDocument& updatedDoc); /** * Waits on _reshardingCoordinatorObserver to notify that: @@ -501,11 +498,6 @@ private: // Waits for majority replication of the latest opTime unless token is cancelled. SemiFuture<void> _waitForMajority(const CancellationToken& token); - /** - * Print a log containing the information of this resharding operation. - */ - void _logStatsOnCompletion(bool success); - // The unique key for a given resharding operation. InstanceID is an alias for BSONObj. The // value of this is the UUID that will be used as the collection UUID for the new sharded // collection. The object looks like: {_id: 'reshardingUUID'} @@ -542,13 +534,6 @@ private: MONGO_MAKE_LATCH("ReshardingCoordinatorService::_fulfillmentMutex"); /** - * Must be locked while the _abortCalled is being set to true. - */ - mutable Mutex _abortCalledMutex = - MONGO_MAKE_LATCH("ReshardingCoordinatorService::_abortCalledMutex"); - - - /** * Coordinator does not enter the critical section until this is fulfilled. * Can be set by "commitReshardCollection" command or by metrics determining * that it's okay to proceed. @@ -568,10 +553,6 @@ private: std::shared_ptr<resharding::CoordinatorCommitMonitor> _commitMonitor; std::shared_ptr<ReshardingCoordinatorExternalState> _reshardingCoordinatorExternalState; - - // Used to catch the case when an abort() is called but the cancellation source (_ctHolder) has - // not been initialized. - bool _abortCalled{false}; }; } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp index 33511df9fd4..6a5197b4c41 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp @@ -128,6 +128,10 @@ class ReshardingCoordinatorServiceTest : public ConfigServerTestFixture { public: using ReshardingCoordinator = ReshardingCoordinatorService::ReshardingCoordinator; + // TODO (SERVER-65302): Use wiredTiger. + ReshardingCoordinatorServiceTest() + : ConfigServerTestFixture(Options{}.engine("ephemeralForTest")) {} + std::unique_ptr<repl::PrimaryOnlyService> makeService(ServiceContext* serviceContext) { return std::make_unique<ReshardingCoordinatorServiceForTest>(serviceContext); } @@ -164,6 +168,7 @@ public: dynamic_cast<OpObserverRegistry*>(getServiceContext()->getOpObserver()); invariant(_opObserverRegistry); + _opObserverRegistry->addObserver(std::make_unique<OpObserverImpl>()); _opObserverRegistry->addObserver(std::make_unique<ReshardingOpObserver>()); _opObserverRegistry->addObserver( std::make_unique<CoordinatorOpObserverForTest>(_controller)); @@ -213,9 +218,14 @@ public: std::shared_ptr<ReshardingCoordinatorService::ReshardingCoordinator> getCoordinatorIfExists( OperationContext* opCtx, repl::PrimaryOnlyService::InstanceID instanceId) { - auto [coordinatorOpt, _] = ReshardingCoordinatorService::ReshardingCoordinator::lookup( + auto coordinatorOpt = ReshardingCoordinatorService::ReshardingCoordinator::lookup( opCtx, _service, instanceId); - return coordinatorOpt ? *coordinatorOpt : nullptr; + if (!coordinatorOpt) { + return nullptr; + } + + auto coordinator = *coordinatorOpt; + return coordinator ? coordinator : nullptr; } ReshardingCoordinatorDocument getCoordinatorDoc(OperationContext* opCtx) { @@ -913,16 +923,5 @@ TEST_F(ReshardingCoordinatorServiceTest, ReshardingCoordinatorFailsIfMigrationNo } } -TEST_F(ReshardingCoordinatorServiceTest, SuccessfullyAbortReshardOperationImmediately) { - auto pauseBeforeCTHolderInitialization = - globalFailPointRegistry().find("pauseBeforeCTHolderInitialization"); - auto timesEnteredFailPoint = pauseBeforeCTHolderInitialization->setMode(FailPoint::alwaysOn, 0); - auto coordinator = initializeAndGetCoordinator(); - coordinator->abort(); - pauseBeforeCTHolderInitialization->waitForTimesEntered(timesEnteredFailPoint + 1); - pauseBeforeCTHolderInitialization->setMode(FailPoint::off, 0); - coordinator->getCompletionFuture().wait(); -} - } // namespace } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_coordinator_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_test.cpp index 521cb84471e..a8fb4d83889 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_test.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_test.cpp @@ -584,8 +584,7 @@ protected: std::vector<BSONObj> zones; if (expectedCoordinatorDoc.getZones()) { zones = buildTagsDocsFromZones(expectedCoordinatorDoc.getTempReshardingNss(), - *expectedCoordinatorDoc.getZones(), - _newShardKey); + *expectedCoordinatorDoc.getZones()); } expectedCoordinatorDoc.setZones(boost::none); expectedCoordinatorDoc.setPresetReshardedChunks(boost::none); @@ -637,7 +636,7 @@ protected: ReshardingCoordinatorDocument expectedCoordinatorDoc, std::vector<ChunkType> expectedChunks, std::vector<TagsType> expectedZones) { - removeChunkDocs(opCtx, expectedCoordinatorDoc.getSourceUUID()); + cleanupSourceConfigCollections(opCtx, expectedCoordinatorDoc); // Check that chunks and tags entries previously under the temporary namespace have been // correctly updated to the original namespace @@ -648,13 +647,10 @@ protected: void removeCoordinatorDocAndReshardingFieldsExpectSuccess( OperationContext* opCtx, const ReshardingCoordinatorDocument& coordinatorDoc) { - auto updatedCoordinatorDoc = removeCoordinatorDocAndReshardingFields(opCtx, coordinatorDoc); - ASSERT_EQUALS(int(updatedCoordinatorDoc.getState()), int(CoordinatorStateEnum::kDone)); + removeCoordinatorDocAndReshardingFields(opCtx, coordinatorDoc); - // Check that the on disk document is same as the in memory document returned above. auto expectedCoordinatorDoc = coordinatorDoc; expectedCoordinatorDoc.setState(CoordinatorStateEnum::kDone); - ASSERT_BSONOBJ_EQ(updatedCoordinatorDoc.toBSON(), expectedCoordinatorDoc.toBSON()); // Check that the entry is removed from config.reshardingOperations DBDirectClient client(opCtx); @@ -665,12 +661,12 @@ protected: // Check that the resharding fields are removed from the config.collections entry and // allowMigrations is set back to true. auto expectedOriginalCollType = makeOriginalCollectionCatalogEntry( - updatedCoordinatorDoc, + expectedCoordinatorDoc, boost::none, _finalEpoch, opCtx->getServiceContext()->getPreciseClockSource()->now()); assertOriginalCollectionCatalogEntryMatchesExpected( - opCtx, expectedOriginalCollType, updatedCoordinatorDoc); + opCtx, expectedOriginalCollType, expectedCoordinatorDoc); } void transitionToErrorExpectSuccess(ErrorCodes::Error errorCode) { diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp index 8635e389cf4..9893b2b0f2e 100644 --- a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp +++ b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp @@ -33,7 +33,7 @@ #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/namespace_string.h" @@ -44,7 +44,6 @@ #include "mongo/db/s/resharding/resharding_txn_cloner_progress_gen.h" #include "mongo/db/s/resharding/resharding_util.h" #include "mongo/db/s/session_catalog_migration.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/db/session_catalog_mongod.h" #include "mongo/db/session_txn_record_gen.h" #include "mongo/db/storage/write_unit_of_work.h" @@ -72,6 +71,28 @@ void ensureCollectionExists(OperationContext* opCtx, }); } +void ensureCollectionDropped(OperationContext* opCtx, + const NamespaceString& nss, + const boost::optional<UUID>& uuid) { + invariant(!opCtx->lockState()->isLocked()); + invariant(!opCtx->lockState()->inAWriteUnitOfWork()); + + writeConflictRetry( + opCtx, "resharding::data_copy::ensureCollectionDropped", nss.toString(), [&] { + AutoGetCollection coll(opCtx, nss, MODE_X); + if (!coll || (uuid && coll->uuid() != uuid)) { + // If the collection doesn't exist or exists with a different UUID, then the + // requested collection has been dropped already. + return; + } + + WriteUnitOfWork wuow(opCtx); + uassertStatusOK(coll.getDb()->dropCollectionEvenIfSystem( + opCtx, nss, {} /* dropOpTime */, true /* markFromMigrate */)); + wuow.commit(); + }); +} + void ensureOplogCollectionsDropped(OperationContext* opCtx, const UUID& reshardingUUID, const UUID& sourceUUID, @@ -98,11 +119,11 @@ void ensureOplogCollectionsDropped(OperationContext* opCtx, // Drop the conflict stash collection for this donor. auto stashNss = getLocalConflictStashNamespace(sourceUUID, donor.getShardId()); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent(opCtx, stashNss); + ensureCollectionDropped(opCtx, stashNss); // Drop the oplog buffer collection for this donor. auto oplogBufferNss = getLocalOplogBufferNamespace(sourceUUID, donor.getShardId()); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent(opCtx, oplogBufferNss); + ensureCollectionDropped(opCtx, oplogBufferNss); } } @@ -282,8 +303,7 @@ void updateSessionRecord(OperationContext* opCtx, BSONObj o2Field, std::vector<StmtId> stmtIds, boost::optional<repl::OpTime> preImageOpTime, - boost::optional<repl::OpTime> postImageOpTime, - NamespaceString sourceNss) { + boost::optional<repl::OpTime> postImageOpTime) { invariant(opCtx->getLogicalSessionId()); invariant(opCtx->getTxnNumber()); @@ -297,7 +317,7 @@ void updateSessionRecord(OperationContext* opCtx, oplogEntry.setOpType(repl::OpTypeEnum::kNoop); oplogEntry.setObject(SessionCatalogMigration::kSessionOplogTag); oplogEntry.setObject2(std::move(o2Field)); - oplogEntry.setNss(std::move(sourceNss)); + oplogEntry.setNss({}); oplogEntry.setSessionId(sessionId); oplogEntry.setTxnNumber(txnNumber); oplogEntry.setStatementIds(stmtIds); diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.h b/src/mongo/db/s/resharding/resharding_data_copy_util.h index af24152aaf2..9f2a332ef6c 100644 --- a/src/mongo/db/s/resharding/resharding_data_copy_util.h +++ b/src/mongo/db/s/resharding/resharding_data_copy_util.h @@ -60,6 +60,16 @@ void ensureCollectionExists(OperationContext* opCtx, const CollectionOptions& options); /** + * Drops the specified collection or returns without error if the collection has already been + * dropped. A particular incarnation of the collection can be dropped by specifying its UUID. + * + * This functions assumes the collection being dropped doesn't have any two-phase index builds + * active on it. + */ +void ensureCollectionDropped(OperationContext* opCtx, + const NamespaceString& nss, + const boost::optional<UUID>& uuid = boost::none); +/** * Removes documents from the oplog applier progress and transaction applier progress collections * that are associated with an in-progress resharding operation. Also drops all oplog buffer * collections and conflict stash collections that are associated with the in-progress resharding @@ -144,14 +154,13 @@ void updateSessionRecord(OperationContext* opCtx, BSONObj o2Field, std::vector<StmtId> stmtIds, boost::optional<repl::OpTime> preImageOpTime, - boost::optional<repl::OpTime> postImageOpTime, - NamespaceString sourceNss); + boost::optional<repl::OpTime> postImageOpTime); /** * Calls and returns the value from the supplied lambda function. * - * If a StaleConfig error is thrown during its execution, then this function will attempt to refresh - * the collection and invoke the supplied lambda function a second time. + * If a StaleConfig exception is thrown during its execution, then this function will attempt to + * refresh the collection and invoke the supplied lambda function a second time. */ template <typename Callable> auto withOneStaleConfigRetry(OperationContext* opCtx, Callable&& callable) { diff --git a/src/mongo/db/s/resharding/resharding_data_replication.cpp b/src/mongo/db/s/resharding/resharding_data_replication.cpp index 5223b1c5c0b..a5918e3ea77 100644 --- a/src/mongo/db/s/resharding/resharding_data_replication.cpp +++ b/src/mongo/db/s/resharding/resharding_data_replication.cpp @@ -475,9 +475,4 @@ ReshardingDonorOplogId ReshardingDataReplication::getOplogApplierResumeId( : ReshardingDonorOplogId{minFetchTimestamp, minFetchTimestamp}; } -ReshardingDataReplication::~ReshardingDataReplication() { - shutdown(); - join(); -} - } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_data_replication.h b/src/mongo/db/s/resharding/resharding_data_replication.h index 61355447a40..a2705abd6f6 100644 --- a/src/mongo/db/s/resharding/resharding_data_replication.h +++ b/src/mongo/db/s/resharding/resharding_data_replication.h @@ -136,7 +136,6 @@ private: struct TrustedInitTag {}; public: - virtual ~ReshardingDataReplication(); static std::unique_ptr<ReshardingDataReplicationInterface> make( OperationContext* opCtx, ReshardingMetrics* metrics, diff --git a/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp b/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp index 0c26b0acd02..e942dcd139f 100644 --- a/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp +++ b/src/mongo/db/s/resharding/resharding_destined_recipient_test.cpp @@ -131,16 +131,14 @@ public: StaticCatalogClient(std::vector<ShardType> shards) : _shards(std::move(shards)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { return repl::OpTimeWith<std::vector<ShardType>>(_shards); } - std::vector<CollectionType> getCollections(OperationContext* opCtx, - StringData dbName, - repl::ReadConcernLevel readConcernLevel, - const BSONObj& sort) override { + std::vector<CollectionType> getCollections( + OperationContext* opCtx, + StringData dbName, + repl::ReadConcernLevel readConcernLevel) override { return _colls; } @@ -239,7 +237,7 @@ protected: createChunks(env.version.epoch(), env.sourceUuid, env.version.getTimestamp(), "y"), boost::none); - ASSERT_OK(onDbVersionMismatchNoExcept(opCtx, kNss.db(), boost::none)); + forceDatabaseRefresh(opCtx, kNss.db()); forceShardFilteringMetadataRefresh(opCtx, kNss); if (refreshTempNss) @@ -312,18 +310,22 @@ TEST_F(DestinedRecipientTest, TestGetDestinedRecipientThrowsOnBlockedRefresh) { auto opCtx = operationContext(); auto env = setupReshardingEnv(opCtx, false); - AutoGetCollection coll(opCtx, kNss, MODE_IX); - OperationShardingState::setShardRole(opCtx, kNss, env.version, env.dbVersion); + { + AutoGetCollection coll(opCtx, kNss, MODE_IX); + OperationShardingState::setShardRole(opCtx, kNss, env.version, env.dbVersion); + + FailPointEnableBlock failPoint("blockCollectionCacheLookup"); + ASSERT_THROWS_WITH_CHECK(ShardingWriteRouter(opCtx, kNss, Grid::get(opCtx)->catalogCache()), + ShardCannotRefreshDueToLocksHeldException, + [&](const ShardCannotRefreshDueToLocksHeldException& ex) { + const auto refreshInfo = + ex.extraInfo<ShardCannotRefreshDueToLocksHeldInfo>(); + ASSERT(refreshInfo); + ASSERT_EQ(refreshInfo->getNss(), env.tempNss); + }); + } - FailPointEnableBlock failPoint("blockCollectionCacheLookup"); - ASSERT_THROWS_WITH_CHECK(ShardingWriteRouter(opCtx, kNss, Grid::get(opCtx)->catalogCache()), - ShardCannotRefreshDueToLocksHeldException, - [&](const ShardCannotRefreshDueToLocksHeldException& ex) { - const auto refreshInfo = - ex.extraInfo<ShardCannotRefreshDueToLocksHeldInfo>(); - ASSERT(refreshInfo); - ASSERT_EQ(refreshInfo->getNss(), env.tempNss); - }); + auto sw = catalogCache()->getCollectionRoutingInfoWithRefresh(opCtx, env.tempNss); } TEST_F(DestinedRecipientTest, TestOpObserverSetsDestinedRecipientOnInserts) { diff --git a/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp b/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp index 99ab47fc11a..e9a466a1cb6 100644 --- a/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp @@ -51,35 +51,10 @@ using DonorStateMachine = ReshardingDonorService::DonorStateMachine; using RecipientStateMachine = ReshardingRecipientService::RecipientStateMachine; namespace { -MONGO_FAIL_POINT_DEFINE(reshardingInterruptAfterInsertStateMachineDocument); - using namespace fmt::literals; const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); -template <class StateMachine, class ReshardingDocument> -void ensureStateDocumentInserted(OperationContext* opCtx, const ReshardingDocument& doc) { - try { - StateMachine::insertStateDocument(opCtx, doc); - } catch (const ExceptionFor<ErrorCodes::DuplicateKey>& ex) { - // It's possible that the state document was already previously inserted in the following - // cases: - // 1. The document was inserted previously, but the opCtx was interrupted before the - // state machine was started in-memory with getOrCreate(), e.g. due to a chunk migration - // (see SERVER-74647) - // 2. Similar to the ErrorCategory::NotPrimaryError clause below, it is - // theoretically possible for a series of stepdowns and step-ups to lead a scenario where a - // stale but now re-elected primary attempts to insert the state document when another node - // which was primary had already done so. Again, rather than attempt to prevent replica set - // member state transitions during the shard version refresh, we instead swallow the - // DuplicateKey exception. This is safe because PrimaryOnlyService::onStepUp() will have - // constructed a new instance of the resharding state machine. - auto dupeKeyInfo = ex.extraInfo<DuplicateKeyErrorInfo>(); - invariant(dupeKeyInfo->getDuplicatedKeyValue().binaryEqual( - BSON("_id" << doc.getReshardingUUID()))); - } -} - /* * Creates a ReshardingStateMachine if this node is primary and the ReshardingStateMachine doesn't * already exist. @@ -92,10 +67,7 @@ void createReshardingStateMachine(OperationContext* opCtx, const ReshardingDocum // Inserting the resharding state document must happen synchronously with the shard version // refresh for the w:majority wait from the resharding coordinator to mean that this replica // set shard cannot forget about being a participant. - ensureStateDocumentInserted<StateMachine>(opCtx, doc); - - reshardingInterruptAfterInsertStateMachineDocument.execute( - [&opCtx](const BSONObj& data) { opCtx->markKilled(); }); + StateMachine::insertStateDocument(opCtx, doc); auto registry = repl::PrimaryOnlyServiceRegistry::get(opCtx->getServiceContext()); auto service = registry->lookupServiceByName(Service::kServiceName); @@ -110,6 +82,17 @@ void createReshardingStateMachine(OperationContext* opCtx, const ReshardingDocum // secondary (or primary which stepped down) must do for an active resharding operation upon // refreshing its shard version. The primary is solely responsible for advancing the // participant state as a result of the shard version refresh. + } catch (const ExceptionFor<ErrorCodes::DuplicateKey>& ex) { + // Similar to the ErrorCategory::NotPrimaryError clause above, it is theoretically possible + // for a series of stepdowns and step-ups to lead a scenario where a stale but now + // re-elected primary attempts to insert the state document when another node which was + // primary had already done so. Again, rather than attempt to prevent replica set member + // state transitions during the shard version refresh, we instead swallow the DuplicateKey + // exception. This is safe because PrimaryOnlyService::onStepUp() will have constructed a + // new instance of the resharding state machine. + auto dupeKeyInfo = ex.extraInfo<DuplicateKeyErrorInfo>(); + invariant(dupeKeyInfo->getDuplicatedKeyValue().binaryEqual( + BSON("_id" << doc.getReshardingUUID()))); } } @@ -155,13 +138,6 @@ void processReshardingFieldsForDonorCollection(OperationContext* opCtx, return; } - // We clear the routing information for the temporary resharding namespace to ensure this donor - // shard primary will refresh from the config server and see the chunk distribution for the new - // resharding operation. - auto* catalogCache = Grid::get(opCtx)->catalogCache(); - catalogCache->invalidateCollectionEntry_LINEARIZABLE( - reshardingFields.getDonorFields()->getTempReshardingNss()); - auto donorDoc = constructDonorDocumentFromReshardingFields(nss, metadata, reshardingFields); createReshardingStateMachine<ReshardingDonorService, DonorStateMachine, @@ -352,16 +328,7 @@ void clearFilteringMetadata(OperationContext* opCtx, bool scheduleAsyncRefresh) }); } - auto* catalogCache = Grid::get(opCtx)->catalogCache(); - for (const auto& nss : namespacesToRefresh) { - if (nss.isTemporaryReshardingCollection()) { - // We clear the routing information for the temporary resharding namespace to ensure all - // new donor shard primaries will refresh from the config server and see the chunk - // distribution for the ongoing resharding operation. - catalogCache->invalidateCollectionEntry_LINEARIZABLE(nss); - } - AutoGetCollection autoColl(opCtx, nss, MODE_IX); CollectionShardingRuntime::get(opCtx, nss)->clearFilteringMetadata(opCtx); diff --git a/src/mongo/db/s/resharding/resharding_donor_recipient_common.h b/src/mongo/db/s/resharding/resharding_donor_recipient_common.h index f6d6a259282..2efba26f659 100644 --- a/src/mongo/db/s/resharding/resharding_donor_recipient_common.h +++ b/src/mongo/db/s/resharding/resharding_donor_recipient_common.h @@ -38,15 +38,11 @@ namespace resharding { using ReshardingFields = TypeCollectionReshardingFields; /** - * Looks up the StateMachine by the 'reshardingUUID'. Returns boost::none in the following cases: - * 1. The state machine does not exist. - * 2. In certain cases when the node is shutting down. - * Additionally returns a bool indicating if the node is stepping or shutting down to disambiguate - * the two. + * Looks up the StateMachine by the 'reshardingUUID'. If it does not exist, returns boost::none. */ template <class Service, class StateMachine, class ReshardingDocument> -std::pair<boost::optional<std::shared_ptr<StateMachine>>, bool> -tryGetReshardingStateMachineAndShutdownState(OperationContext* opCtx, const UUID& reshardingUUID) { +boost::optional<std::shared_ptr<StateMachine>> tryGetReshardingStateMachine( + OperationContext* opCtx, const UUID& reshardingUUID) { auto instanceId = BSON(ReshardingDocument::kReshardingUUIDFieldName << reshardingUUID); auto registry = repl::PrimaryOnlyServiceRegistry::get(opCtx->getServiceContext()); auto service = registry->lookupServiceByName(Service::kServiceName); @@ -54,39 +50,6 @@ tryGetReshardingStateMachineAndShutdownState(OperationContext* opCtx, const UUID } /** - * Same as tryGetReshardingStateMachineAndShutdownState, except does not return the shutdown state. - */ -template <class Service, class StateMachine, class ReshardingDocument> -boost::optional<std::shared_ptr<StateMachine>> tryGetReshardingStateMachine( - OperationContext* opCtx, const UUID& reshardingUUID) { - auto [instance, _] = - tryGetReshardingStateMachineAndShutdownState<Service, StateMachine, ReshardingDocument>( - opCtx, reshardingUUID); - return instance; -} - -/** - * Same as tryGetReshardingStateMachine, except throws if we were stepping or shutting down when we - * tried to access the PrimaryOnlyService. Use this function in situations where you need to - * guarantee that a return of boost::none means that there is no state document on disk for the - * associated state machine. - */ -template <class Service, class StateMachine, class ReshardingDocument> -boost::optional<std::shared_ptr<StateMachine>> tryGetReshardingStateMachineAndThrowIfShuttingDown( - OperationContext* opCtx, const UUID& reshardingUUID) { - auto [instance, steppingOrShuttingDown] = - tryGetReshardingStateMachineAndShutdownState<Service, StateMachine, ReshardingDocument>( - opCtx, reshardingUUID); - - uassert(ErrorCodes::InterruptedDueToReplStateChange, - "Unable to get resharding state machine, if it exists, because the node is " - "stepping or shutting down.", - !steppingOrShuttingDown); - - return instance; -} - -/** * The following functions construct a ReshardingDocument from the given 'reshardingFields'. */ ReshardingDonorDocument constructDonorDocumentFromReshardingFields( diff --git a/src/mongo/db/s/resharding/resharding_donor_service.cpp b/src/mongo/db/s/resharding/resharding_donor_service.cpp index 2e75751ba01..ce60e58f0e1 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service.cpp @@ -37,7 +37,7 @@ #include "mongo/db/catalog/drop_collection.h" #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/dbhelpers.h" @@ -47,7 +47,6 @@ #include "mongo/db/persistent_task_store.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/repl/wait_for_majority_service.h" -#include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/recoverable_critical_section_service.h" #include "mongo/db/s/resharding/resharding_change_event_o2_field_gen.h" #include "mongo/db/s/resharding/resharding_data_copy_util.h" @@ -56,7 +55,6 @@ #include "mongo/db/s/resharding/resharding_metrics.h" #include "mongo/db/s/resharding/resharding_server_parameters_gen.h" #include "mongo/db/s/resharding/resharding_util.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/write_block_bypass.h" #include "mongo/db/write_concern_options.h" @@ -67,7 +65,6 @@ namespace mongo { -MONGO_FAIL_POINT_DEFINE(reshardingPauseDonorBeforeCatalogCacheRefresh); MONGO_FAIL_POINT_DEFINE(reshardingDonorFailsAfterTransitionToDonatingOplogEntries); MONGO_FAIL_POINT_DEFINE(removeDonorDocFailpoint); @@ -180,9 +177,8 @@ public: } } - void refreshCollectionPlacementInfo(OperationContext* opCtx, - const NamespaceString& sourceNss) override { - onShardVersionMismatch(opCtx, sourceNss, boost::none); + void clearFilteringMetadata(OperationContext* opCtx) { + resharding::clearFilteringMetadata(opCtx, true /* scheduleAsyncRefresh */); } }; @@ -371,15 +367,8 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin { auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - std::initializer_list<NamespaceString> namespacesToRefresh{ - _metadata.getSourceNss(), _metadata.getTempReshardingNss()}; - - // Clear filtering metadata for the source and temp resharding nss. - for (const auto& nss : namespacesToRefresh) { - AutoGetCollection autoColl(opCtx.get(), nss, MODE_IX); - CollectionShardingRuntime::get(opCtx.get(), nss) - ->clearFilteringMetadata(opCtx.get()); - } + + _externalState->clearFilteringMetadata(opCtx.get()); RecoverableCriticalSectionService::get(opCtx.get()) ->releaseRecoverableCriticalSection( @@ -389,13 +378,6 @@ ExecutorFuture<void> ReshardingDonorService::DonorStateMachine::_finishReshardin ShardingCatalogClient::kLocalWriteConcern); _metrics()->leaveCriticalSection(getCurrentTime()); - - // We force a refresh to make sure that the placement information is updated in - // cache after abort decision before the donor state document is deleted. - for (const auto& nss : namespacesToRefresh) { - _externalState->refreshCollectionPlacementInfo(opCtx.get(), nss); - _externalState->waitForCollectionFlush(opCtx.get(), nss); - } } auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); @@ -585,8 +567,6 @@ void ReshardingDonorService::DonorStateMachine:: // with a SnapshotUnavailable error response. { auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); - reshardingPauseDonorBeforeCatalogCacheRefresh.pauseWhileSet(opCtx.get()); - _externalState->refreshCatalogCache(opCtx.get(), _metadata.getTempReshardingNss()); _externalState->waitForCollectionFlush(opCtx.get(), _metadata.getTempReshardingNss()); } @@ -815,7 +795,7 @@ void ReshardingDonorService::DonorStateMachine::_dropOriginalCollectionThenTrans // Allow bypassing user write blocking. The check has already been performed on the // db-primary shard's ReshardCollectionCoordinator. WriteBlockBypass::get(opCtx.get()).set(true); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent( + resharding::data_copy::ensureCollectionDropped( opCtx.get(), _metadata.getSourceNss(), _metadata.getSourceUUID()); } diff --git a/src/mongo/db/s/resharding/resharding_donor_service.h b/src/mongo/db/s/resharding/resharding_donor_service.h index 7b5331f93ac..b50c88b6af5 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service.h +++ b/src/mongo/db/s/resharding/resharding_donor_service.h @@ -298,8 +298,7 @@ public: const BSONObj& query, const BSONObj& update) = 0; - virtual void refreshCollectionPlacementInfo(OperationContext* opCtx, - const NamespaceString& sourceNss) = 0; + virtual void clearFilteringMetadata(OperationContext* opCtx) = 0; }; } // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp index 16e1507be1c..663b8c28136 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service_test.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service_test.cpp @@ -34,6 +34,7 @@ #include <boost/optional/optional_io.hpp> #include <utility> +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/op_observer_noop.h" #include "mongo/db/op_observer_registry.h" @@ -52,7 +53,6 @@ #include "mongo/db/s/resharding/resharding_donor_service.h" #include "mongo/db/s/resharding/resharding_service_test_helpers.h" #include "mongo/db/s/resharding/resharding_util.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/logv2/log.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/unittest/death_test.h" @@ -84,8 +84,7 @@ public: const BSONObj& query, const BSONObj& update) override {} - void refreshCollectionPlacementInfo(OperationContext* opCtx, - const NamespaceString& sourceNss) override {} + void clearFilteringMetadata(OperationContext* opCtx) override {} }; class DonorOpObserverForTest : public OpObserverForTest { @@ -162,8 +161,7 @@ public: void createSourceCollection(OperationContext* opCtx, const ReshardingDonorDocument& donorDoc) { CollectionOptions options; options.uuid = donorDoc.getSourceUUID(); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent(opCtx, - donorDoc.getSourceNss()); + resharding::data_copy::ensureCollectionDropped(opCtx, donorDoc.getSourceNss()); resharding::data_copy::ensureCollectionExists(opCtx, donorDoc.getSourceNss(), options); } @@ -171,8 +169,7 @@ public: const ReshardingDonorDocument& donorDoc) { CollectionOptions options; options.uuid = donorDoc.getReshardingUUID(); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent( - opCtx, donorDoc.getTempReshardingNss()); + resharding::data_copy::ensureCollectionDropped(opCtx, donorDoc.getTempReshardingNss()); resharding::data_copy::ensureCollectionExists( opCtx, donorDoc.getTempReshardingNss(), options); } @@ -418,10 +415,8 @@ TEST_F(ReshardingDonorServiceTest, StepDownStepUpEachTransition) { DonorStateMachine::insertStateDocument(opCtx.get(), doc); return DonorStateMachine::getOrCreate(opCtx.get(), _service, doc.toBSON()); } else { - auto [maybeDonor, isPausedOrShutdown] = - DonorStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeDonor); - ASSERT_FALSE(isPausedOrShutdown); + auto maybeDonor = DonorStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeDonor)); // Allow the transition to prevState to succeed on this primary-only service // instance. @@ -482,10 +477,8 @@ TEST_F(ReshardingDonorServiceTest, StepDownStepUpEachTransition) { } // Finally complete the operation and ensure its success. - auto [maybeDonor, isPausedOrShutdown] = - DonorStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeDonor); - ASSERT_FALSE(isPausedOrShutdown); + auto maybeDonor = DonorStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeDonor)); auto donor = *maybeDonor; stateTransitionsGuard.unset(DonorStateEnum::kDone); @@ -615,10 +608,8 @@ TEST_F(ReshardingDonorServiceTest, CompletesWithStepdownAfterAbort) { donor.reset(); stepUp(opCtx.get()); - auto [maybeDonor, isPausedOrShutdown] = - DonorStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeDonor); - ASSERT_FALSE(isPausedOrShutdown); + auto maybeDonor = DonorStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeDonor)); donor = *maybeDonor; doneTransitionGuard.reset(); diff --git a/src/mongo/db/s/resharding/resharding_future_util.cpp b/src/mongo/db/s/resharding/resharding_future_util.cpp index 62c95fbaf3f..849a14bd80e 100644 --- a/src/mongo/db/s/resharding/resharding_future_util.cpp +++ b/src/mongo/db/s/resharding/resharding_future_util.cpp @@ -51,42 +51,19 @@ ExecutorFuture<void> whenAllSucceedOn(const std::vector<SharedSemiFuture<void>>& : ExecutorFuture(executor); } -std::vector<Future<void>> runAllInlineUnsafe(const std::vector<SharedSemiFuture<void>>& futures) { - std::vector<Future<void>> result; - result.reserve(futures.size()); - - for (const auto& future : futures) { - result.emplace_back(future.unsafeToInlineFuture()); - } - - return result; -} - ExecutorFuture<void> cancelWhenAnyErrorThenQuiesce( const std::vector<SharedSemiFuture<void>>& futures, ExecutorPtr executor, CancellationSource cancelSource) { - if (futures.empty()) { - return ExecutorFuture(executor); - } - // Run all futures inline so that the onError callback is called even if that error was caused - // by the executor shutting down. This causes the logic for whenAllSucceed, whenAll, and the - // onError callback to potentially run on the threads of the setters of the promises - // associated with the input futures. Since this logic is thread safe, not blocking, and does - // not acquire additional resources, this is safe, but beware if making further changes to this - // function. - return whenAllSucceed(runAllInlineUnsafe(futures)) - .unsafeToInlineFuture() - .onError([futures, cancelSource](Status originalError) mutable { + return whenAllSucceedOn(futures, executor) + .onError([futures, executor, cancelSource](Status originalError) mutable { cancelSource.cancel(); - return whenAll(runAllInlineUnsafe(futures)) + return whenAll(thenRunAllOn(futures, executor)) .ignoreValue() - .unsafeToInlineFuture() + .thenRunOn(executor) .onCompletion([originalError](auto) { return originalError; }); - }) - .thenRunOn(executor); + }); } - } // namespace mongo::resharding diff --git a/src/mongo/db/s/resharding/resharding_future_util.h b/src/mongo/db/s/resharding/resharding_future_util.h index 78847350d0c..0bb858a963c 100644 --- a/src/mongo/db/s/resharding/resharding_future_util.h +++ b/src/mongo/db/s/resharding/resharding_future_util.h @@ -131,8 +131,7 @@ public: status.isA<ErrorCategory::CursorInvalidatedError>() || status == ErrorCodes::Interrupted || status.isA<ErrorCategory::CancellationError>() || - status.isA<ErrorCategory::NotPrimaryError>() || - status.isA<ErrorCategory::NetworkTimeoutError>()) { + status.isA<ErrorCategory::NotPrimaryError>()) { // Always attempt to retry on any type of retryable error. Also retry on errors // from stray killCursors and killOp commands being run. Cancellation and // NotPrimary errors may indicate the primary-only service Instance will be shut diff --git a/src/mongo/db/s/resharding/resharding_future_util_test.cpp b/src/mongo/db/s/resharding/resharding_future_util_test.cpp deleted file mode 100644 index e37a13a314b..00000000000 --- a/src/mongo/db/s/resharding/resharding_future_util_test.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/s/resharding/resharding_future_util.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/concurrency/thread_pool.h" - -namespace mongo { -namespace { -class ReshardingFutureUtilTest : public unittest::Test { -protected: - void setUp() override { - _executor = std::make_shared<ThreadPool>([]() { - ThreadPool::Options options; - options.maxThreads = 2; - return options; - }()); - _executor->startup(); - } - - void tearDown() override { - _executor->shutdown(); - _executor->join(); - } - - std::shared_ptr<ThreadPool> getExecutor() const { - return _executor; - } - -private: - std::shared_ptr<ThreadPool> _executor; -}; - -TEST_F(ReshardingFutureUtilTest, CancelWhenAnyErrorThenQuiesceDuringExecutorShutdown) { - CancellationSource cancelSource; - auto token = cancelSource.token(); - PromiseAndFuture<void> taskThreadsReady; - AtomicWord<int> tasksRunningCount{0}; - AtomicWord<bool> taskWasCancelled{false}; - auto checkSignalReady = [&]() { - auto running = tasksRunningCount.addAndFetch(1); - if (running == 2) { - taskThreadsReady.promise.emplaceValue(); - } - }; - PromiseAndFuture<void> executorShutDownTriggered; - auto quiesced = ExecutorFuture(getExecutor()).then([&]() { - return resharding::cancelWhenAnyErrorThenQuiesce( - {ExecutorFuture(getExecutor()) - .then([&]() { - checkSignalReady(); - executorShutDownTriggered.future.wait(); - uasserted(6791600, "Executor shut down"); - }) - .share(), - ExecutorFuture(getExecutor()) - .then([&]() { - checkSignalReady(); - token.onCancel().wait(); - taskWasCancelled.store(true); - }) - .share()}, - getExecutor(), - cancelSource); - }); - taskThreadsReady.future.wait(); - getExecutor()->shutdown(); - executorShutDownTriggered.promise.emplaceValue(); - auto status = quiesced.getNoThrow(); - ASSERT_EQ(status.code(), 6791600); - ASSERT_TRUE(taskWasCancelled.load()); -} -} // namespace -} // namespace mongo diff --git a/src/mongo/db/s/resharding/resharding_metrics.cpp b/src/mongo/db/s/resharding/resharding_metrics.cpp index b42c9df5c9e..6e4e4e041e9 100644 --- a/src/mongo/db/s/resharding/resharding_metrics.cpp +++ b/src/mongo/db/s/resharding/resharding_metrics.cpp @@ -406,60 +406,6 @@ void ReshardingMetrics::onStepUp(Role role) noexcept { // instead of starting from the current time. } -void ReshardingMetrics::onStepUp(RecipientStateEnum state, - const ReshardingRecipientCountsAndMetrics& recipientMetrics) { - stdx::lock_guard<Latch> lk(_mutex); - - _emplaceCurrentOpForRole(Role::kRecipient, boost::none); - _onStepUpCalled = true; - - invariant(_currentOp, kNoOperationInProgress); - invariant(_currentOp->documentsCopied == 0, kMetricsSetBeforeRestore); - invariant(_currentOp->bytesCopied == 0, kMetricsSetBeforeRestore); - invariant(_currentOp->oplogEntriesFetched == 0, kMetricsSetBeforeRestore); - invariant(_currentOp->oplogEntriesApplied == 0, kMetricsSetBeforeRestore); - - _currentOp->recipientState = state; - _currentOp->documentsCopied = recipientMetrics.documentCountCopied; - _currentOp->bytesCopied = recipientMetrics.documentBytesCopied; - _currentOp->oplogEntriesFetched = recipientMetrics.oplogEntriesFetched; - _currentOp->oplogEntriesApplied = recipientMetrics.oplogEntriesApplied; - - if (recipientMetrics.approxBytesToCopy) - _currentOp->bytesToCopy = recipientMetrics.approxBytesToCopy.get(); - - - const auto& timeIntervals = recipientMetrics.metrics; - - // Restore in memory state of document copy metrics. - // Not calling startCopyingDocuments or endCopyingDocuments because they acquire a mutex that we - // already have. - // - // Also, note that it is possible for documentCopyInterval->getStart() to be none and for - // documentCopyInterval->getStop() to be not none. That can happen if the cluster is upgraded - // to include code for persisting time intervals during a resharding operation. - // In that case, restore neither the start nor stop time. The resharding coordinator will still - // treat this scenario as the recipient shard being completely caught up after a primary - // failover and engage the critical section too early. - const auto& documentCopyInterval = timeIntervals.getDocumentCopy(); - if (documentCopyInterval && documentCopyInterval->getStart()) { - _currentOp->copyingDocuments.start(documentCopyInterval->getStart().get()); - if (documentCopyInterval->getStop()) { - _currentOp->copyingDocuments.end(documentCopyInterval->getStop().get()); - } - } - // Restore in memory state of oplog application metrics. - // Not calling startApplyingOplogEntries or endApplyingOplogEntries because they acquire a mutex - // that we already have. - const auto& oplogApplicationInterval = timeIntervals.getOplogApplication(); - if (oplogApplicationInterval && oplogApplicationInterval->getStart()) { - _currentOp->applyingOplogEntries.start(oplogApplicationInterval->getStart().get()); - if (oplogApplicationInterval->getStop()) { - _currentOp->applyingOplogEntries.end(oplogApplicationInterval->getStop().get()); - } - } -} - void ReshardingMetrics::onStepUp(DonorStateEnum state, ReshardingDonorMetrics donorMetrics) { stdx::lock_guard<Latch> lk(_mutex); auto operationRuntime = donorMetrics.getOperationRuntime(); @@ -538,9 +484,6 @@ void ReshardingMetrics::setDonorState(DonorStateEnum state) noexcept { void ReshardingMetrics::setRecipientState(RecipientStateEnum state) noexcept { stdx::lock_guard<Latch> lk(_mutex); - if (!_currentOp && state == RecipientStateEnum::kDone) { - return; - } invariant(_currentOp, kNoOperationInProgress); const auto oldState = std::exchange(_currentOp->recipientState, state); @@ -691,10 +634,6 @@ void ReshardingMetrics::enterCriticalSection(Date_t start) { void ReshardingMetrics::leaveCriticalSection(Date_t end) { stdx::lock_guard<Latch> lk(_mutex); - if (!_currentOp) { - return; - } - _currentOp->inCriticalSection.forceEnd(end); } @@ -723,6 +662,22 @@ void ReshardingMetrics::onOplogEntriesApplied(int64_t entries) noexcept { _cumulativeOp->oplogEntriesApplied += entries; } +void ReshardingMetrics::restoreForCurrentOp(int64_t documentCountCopied, + int64_t documentBytesCopied, + int64_t oplogEntriesFetched, + int64_t oplogEntriesApplied) noexcept { + invariant(_currentOp, kNoOperationInProgress); + invariant(_currentOp->documentsCopied == 0, kMetricsSetBeforeRestore); + invariant(_currentOp->bytesCopied == 0, kMetricsSetBeforeRestore); + invariant(_currentOp->oplogEntriesFetched == 0, kMetricsSetBeforeRestore); + invariant(_currentOp->oplogEntriesApplied == 0, kMetricsSetBeforeRestore); + + _currentOp->documentsCopied = documentCountCopied; + _currentOp->bytesCopied = documentBytesCopied; + _currentOp->oplogEntriesFetched = oplogEntriesFetched; + _currentOp->oplogEntriesApplied = oplogEntriesApplied; +} + void ReshardingMetrics::onWriteDuringCriticalSection(int64_t writes) noexcept { stdx::lock_guard<Latch> lk(_mutex); if (!_currentOp) diff --git a/src/mongo/db/s/resharding/resharding_metrics.h b/src/mongo/db/s/resharding/resharding_metrics.h index 9ba80d35417..a6964c9d611 100644 --- a/src/mongo/db/s/resharding/resharding_metrics.h +++ b/src/mongo/db/s/resharding/resharding_metrics.h @@ -35,7 +35,6 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/namespace_string.h" #include "mongo/db/s/resharding/donor_document_gen.h" -#include "mongo/db/s/resharding/recipient_document_gen.h" #include "mongo/db/service_context.h" #include "mongo/platform/mutex.h" #include "mongo/s/resharding/common_types_gen.h" @@ -72,29 +71,6 @@ public: void onStepUp(DonorStateEnum state, ReshardingDonorMetrics donorMetrics); - struct ReshardingRecipientCountsAndMetrics { - ReshardingRecipientCountsAndMetrics(int64_t documentCountCopied, - int64_t documentBytesCopied, - int64_t oplogEntriesFetched, - int64_t oplogEntriesApplied, - boost::optional<int64_t> approxBytesToCopy, - ReshardingRecipientMetrics metrics) - : documentCountCopied{documentCountCopied}, - documentBytesCopied{documentBytesCopied}, - oplogEntriesFetched{oplogEntriesFetched}, - oplogEntriesApplied{oplogEntriesApplied}, - approxBytesToCopy{approxBytesToCopy}, - metrics{metrics} {} - int64_t documentCountCopied; - int64_t documentBytesCopied; - int64_t oplogEntriesFetched; - int64_t oplogEntriesApplied; - boost::optional<int64_t> approxBytesToCopy; - ReshardingRecipientMetrics metrics; - }; - - void onStepUp(RecipientStateEnum, const ReshardingRecipientCountsAndMetrics&); - // So long as a resharding operation is in progress, the following may be used to update the // state of a donor, a recipient, and a coordinator, respectively. void setDonorState(DonorStateEnum) noexcept; @@ -137,6 +113,11 @@ public: // Allows restoring "oplog entries to apply" metrics. void onOplogEntriesApplied(int64_t entries) noexcept; + void restoreForCurrentOp(int64_t documentCountCopied, + int64_t documentBytesCopied, + int64_t oplogEntriesFetched, + int64_t oplogEntriesApplied) noexcept; + // Allows tracking writes during a critical section when the donor's state is either of // "donating-oplog-entries" or "blocking-writes". void onWriteDuringCriticalSection(int64_t writes) noexcept; diff --git a/src/mongo/db/s/resharding/resharding_op_observer.cpp b/src/mongo/db/s/resharding/resharding_op_observer.cpp index cd2fb324c8e..ba1f768f552 100644 --- a/src/mongo/db/s/resharding/resharding_op_observer.cpp +++ b/src/mongo/db/s/resharding/resharding_op_observer.cpp @@ -48,7 +48,7 @@ std::shared_ptr<ReshardingCoordinatorObserver> getReshardingCoordinatorObserver( OperationContext* opCtx, const BSONObj& reshardingId) { auto registry = repl::PrimaryOnlyServiceRegistry::get(opCtx->getServiceContext()); auto service = registry->lookupServiceByName(ReshardingCoordinatorService::kServiceName); - auto [instance, _] = + auto instance = ReshardingCoordinatorService::ReshardingCoordinator::lookup(opCtx, service, reshardingId); iassert( diff --git a/src/mongo/db/s/resharding/resharding_op_observer.h b/src/mongo/db/s/resharding/resharding_op_observer.h index e8affe3ef4a..30d319a041d 100644 --- a/src/mongo/db/s/resharding/resharding_op_observer.h +++ b/src/mongo/db/s/resharding/resharding_op_observer.h @@ -228,10 +228,6 @@ public: size_t numberOfPrePostImagesToWrite, Date_t wallClockTime) override {} - void onTransactionPrepareNonPrimary(OperationContext* opCtx, - const std::vector<repl::OplogEntry>& statements, - const repl::OpTime& prepareOpTime) override {} - void onTransactionAbort(OperationContext* opCtx, boost::optional<OplogSlot> abortOplogEntryOpTime) override {} diff --git a/src/mongo/db/s/resharding/resharding_oplog_application.cpp b/src/mongo/db/s/resharding/resharding_oplog_application.cpp index 42c5a1543e5..9a2f6f2750e 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_application.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_application.cpp @@ -29,9 +29,11 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kResharding +#include "mongo/platform/basic.h" + #include "mongo/db/s/resharding/resharding_oplog_application.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/index/index_access_method.h" #include "mongo/db/logical_session_cache.h" @@ -77,8 +79,8 @@ void runWithTransaction(OperationContext* opCtx, // ReshardingOpObserver depends on the collection metadata being known when processing writes to // the temporary resharding collection. We attach shard version IGNORED to the write operations - // and leave it to ReshardingOplogBatchApplier::applyBatch() to retry on a StaleConfig error to - // allow the collection metadata information to be recovered. + // and leave it to ReshardingOplogBatchApplier::applyBatch() to retry on a StaleConfig exception + // to allow the collection metadata information to be recovered. ScopedSetShardRole scopedSetShardRole(asr.opCtx(), nss, ChunkVersion::IGNORED() /* shardVersion */, diff --git a/src/mongo/db/s/resharding/resharding_oplog_applier.cpp b/src/mongo/db/s/resharding/resharding_oplog_applier.cpp index 71d156170d4..766ac67f266 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_applier.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_applier.cpp @@ -236,8 +236,7 @@ void ReshardingOplogApplier::_clearAppliedOpsAndStoreProgress(OperationContext* store.upsert( opCtx, BSON(ReshardingOplogApplierProgress::kOplogSourceIdFieldName << _sourceId.toBSON()), - builder.obj(), - WriteConcerns::kLocalWriteConcern); + builder.obj()); _env->metrics()->onOplogEntriesApplied(_currentBatchToApply.size()); if (ShardingDataTransformMetrics::isEnabled()) { _env->metricsNew()->onOplogEntriesApplied(_currentBatchToApply.size()); diff --git a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp index 774a654ce8f..02334d1f4bf 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_applier_test.cpp @@ -249,7 +249,6 @@ public: kCrudNs, kCrudUUID, false /* fromMigrate */, - boost::none, // checkExistenceForDiffInsert 0 /* version */, obj1, obj2, diff --git a/src/mongo/db/s/resharding/resharding_oplog_batch_applier.cpp b/src/mongo/db/s/resharding/resharding_oplog_batch_applier.cpp index c7efa6d33e4..4ff29b42d30 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_batch_applier.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_batch_applier.cpp @@ -29,8 +29,12 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kResharding +#include "mongo/platform/basic.h" + #include "mongo/db/s/resharding/resharding_oplog_batch_applier.h" +#include <memory> + #include "mongo/db/s/operation_sharding_state.h" #include "mongo/db/s/resharding/resharding_data_copy_util.h" #include "mongo/db/s/resharding/resharding_future_util.h" @@ -79,8 +83,8 @@ SemiFuture<void> ReshardingOplogBatchApplier::applyBatch( // ReshardingOpObserver depends on the collection metadata being known // when processing writes to the temporary resharding collection. We // attach shard version IGNORED to the write operations and retry once - // on a StaleConfig error to allow the collection metadata information to - // be recovered. + // on a StaleConfig exception to allow the collection metadata + // information to be recovered. ScopedSetShardRole scopedSetShardRole( opCtx.get(), _crudApplication.getOutputNss(), diff --git a/src/mongo/db/s/resharding/resharding_oplog_fetcher.cpp b/src/mongo/db/s/resharding/resharding_oplog_fetcher.cpp index 064ca34c9fa..30811c8d2aa 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_fetcher.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_fetcher.cpp @@ -41,7 +41,7 @@ #include "mongo/client/dbclient_connection.h" #include "mongo/client/remote_command_targeter.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/pipeline/aggregate_command_gen.h" #include "mongo/db/repl/read_concern_args.h" #include "mongo/db/repl/read_concern_level.h" @@ -194,22 +194,23 @@ ExecutorFuture<void> ReshardingOplogFetcher::_reschedule( } bool ReshardingOplogFetcher::iterate(Client* client, CancelableOperationContextFactory factory) { - try { - std::shared_ptr<Shard> targetShard; - { - auto opCtxRaii = factory.makeOperationContext(client); - opCtxRaii->checkForInterrupt(); - - StatusWith<std::shared_ptr<Shard>> swDonor = - Grid::get(opCtxRaii.get())->shardRegistry()->getShard(opCtxRaii.get(), _donorShard); - if (!swDonor.isOK()) { - LOGV2_WARNING(5127203, - "Error finding shard in registry, retrying.", - "error"_attr = swDonor.getStatus()); - return true; - } - targetShard = swDonor.getValue(); + std::shared_ptr<Shard> targetShard; + { + auto opCtxRaii = factory.makeOperationContext(client); + opCtxRaii->checkForInterrupt(); + + StatusWith<std::shared_ptr<Shard>> swDonor = + Grid::get(opCtxRaii.get())->shardRegistry()->getShard(opCtxRaii.get(), _donorShard); + if (!swDonor.isOK()) { + LOGV2_WARNING(5127203, + "Error finding shard in registry, retrying.", + "error"_attr = swDonor.getStatus()); + return true; } + targetShard = swDonor.getValue(); + } + + try { return consume(client, factory, targetShard.get()); } catch (const ExceptionForCat<ErrorCategory::Interruption>&) { // Defer to the cancellation token for whether the Interruption exception should be retried diff --git a/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp b/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp index 99707c61b96..e7c512e2669 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_fetcher_test.cpp @@ -36,7 +36,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/dbhelpers.h" @@ -144,9 +144,7 @@ public: StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { std::vector<ShardType> shardTypes; for (const auto& shardId : _shardIds) { const ConnectionString cs = ConnectionString::forReplicaSet( @@ -668,44 +666,6 @@ TEST_F(ReshardingOplogFetcherTest, RetriesOnRemoteInterruptionError) { ASSERT_TRUE(moreToCome); } -TEST_F(ReshardingOplogFetcherTest, RetriesOnNetworkTimeoutError) { - const NamespaceString outputCollectionNss("dbtests.outputCollection"); - const NamespaceString dataCollectionNss("dbtests.runFetchIteration"); - - create(outputCollectionNss); - create(dataCollectionNss); - _fetchTimestamp = repl::StorageInterface::get(_svcCtx)->getLatestOplogTimestamp(_opCtx); - - const auto& collectionUUID = [&] { - AutoGetCollection dataColl(_opCtx, dataCollectionNss, LockMode::MODE_IX); - return dataColl->uuid(); - }(); - - auto fetcherJob = launchAsync([&, this] { - ThreadClient tc("RunnerForFetcher", _svcCtx, nullptr); - - ReshardingDonorOplogId startAt{_fetchTimestamp, _fetchTimestamp}; - ReshardingOplogFetcher fetcher(makeFetcherEnv(), - _reshardingUUID, - collectionUUID, - startAt, - _donorShard, - _destinationShard, - outputCollectionNss); - - auto factory = makeCancelableOpCtx(); - return fetcher.iterate(&cc(), factory); - }); - - onCommand([&](const executor::RemoteCommandRequest& request) -> StatusWith<BSONObj> { - // Inject network timeout error. - return {ErrorCodes::NetworkInterfaceExceededTimeLimit, "exceeded network time limit"}; - }); - - auto moreToCome = fetcherJob.timed_get(Seconds(5)); - ASSERT_TRUE(moreToCome); -} - TEST_F(ReshardingOplogFetcherTest, ImmediatelyDoneWhenFinalOpHasAlreadyBeenFetched) { const NamespaceString outputCollectionNss("dbtests.outputCollection"); const NamespaceString dataCollectionNss("dbtests.runFetchIteration"); diff --git a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp index 95c29d0825c..44251fa39dc 100644 --- a/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp +++ b/src/mongo/db/s/resharding/resharding_oplog_session_application.cpp @@ -32,7 +32,7 @@ #include "mongo/db/s/resharding/resharding_oplog_session_application.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/oplog_entry.h" @@ -108,7 +108,6 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp invariant(op.getTxnNumber()); invariant(op.get_id()); - auto sourceNss = op.getNss(); auto lsid = *op.getSessionId(); if (isInternalSessionForNonRetryableWrite(lsid)) { // Skip internal sessions for non-retryable writes since they only support transactions @@ -158,8 +157,7 @@ boost::optional<SharedSemiFuture<void>> ReshardingOplogSessionApplication::tryAp std::move(o2Field), std::move(stmtIds), std::move(preImageOpTime), - std::move(postImageOpTime), - std::move(sourceNss)); + std::move(postImageOpTime)); }); } diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.cpp b/src/mongo/db/s/resharding/resharding_recipient_service.cpp index 087e3cb603f..5d49e0cdb22 100644 --- a/src/mongo/db/s/resharding/resharding_recipient_service.cpp +++ b/src/mongo/db/s/resharding/resharding_recipient_service.cpp @@ -36,7 +36,7 @@ #include "mongo/db/cancelable_operation_context.h" #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/dbhelpers.h" #include "mongo/db/ops/delete.h" @@ -56,7 +56,6 @@ #include "mongo/db/s/resharding/resharding_recipient_service_external_state.h" #include "mongo/db/s/resharding/resharding_server_parameters_gen.h" #include "mongo/db/s/shard_key_util.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/write_block_bypass.h" #include "mongo/executor/network_interface_factory.h" @@ -146,8 +145,6 @@ ReshardingRecipientService::RecipientStateMachine::RecipientStateMachine( _recipientCtx{recipientDoc.getMutableState()}, _donorShards{recipientDoc.getDonorShards()}, _cloneTimestamp{recipientDoc.getCloneTimestamp()}, - _timeIntervals{recipientDoc.getMetrics().get_value_or({})}, - _approxBytesToCopy{recipientDoc.getApproxBytesToCopy()}, _externalState{std::move(externalState)}, _startConfigTxnCloneAt{recipientDoc.getStartConfigTxnCloneTime()}, _markKilledExecutor(std::make_shared<ThreadPool>([] { @@ -169,7 +166,6 @@ ReshardingRecipientService::RecipientStateMachine::RecipientStateMachine( return donor.getShardId() == myShardId; }) != _donorShards.end(); }()) { - invariant(_externalState); } @@ -557,11 +553,6 @@ ReshardingRecipientService::RecipientStateMachine::_makeDataReplication(Operatio bool cloningDone) { invariant(_cloneTimestamp); - // We refresh the routing information for the source collection to ensure the - // ReshardingOplogApplier is making its decisions according to the chunk distribution after the - // sharding metadata was frozen. - _externalState->refreshCatalogCache(opCtx, _metadata.getSourceNss()); - auto myShardId = _externalState->myShardId(opCtx->getServiceContext()); auto sourceChunkMgr = _externalState->getShardedCollectionRoutingInfo(opCtx, _metadata.getSourceNss()); @@ -761,7 +752,7 @@ void ReshardingRecipientService::RecipientStateMachine::_cleanupReshardingCollec opCtx.get(), _metadata.getReshardingUUID(), _metadata.getSourceUUID(), _donorShards); if (aborted) { - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent( + resharding::data_copy::ensureCollectionDropped( opCtx.get(), _metadata.getTempReshardingNss(), _metadata.getReshardingUUID()); } } @@ -817,53 +808,27 @@ void ReshardingRecipientService::RecipientStateMachine::_transitionToCloning( const CancelableOperationContextFactory& factory) { auto newRecipientCtx = _recipientCtx; newRecipientCtx.setState(RecipientStateEnum::kCloning); - auto cloningStartTime = getCurrentTime(); - - // Record cloning start time. - ReshardingMetricsTimeInterval interval; - interval.setStart(cloningStartTime); - _timeIntervals.setDocumentCopy(interval); - _transitionState(std::move(newRecipientCtx), boost::none, boost::none, factory); - _metrics()->startCopyingDocuments(cloningStartTime); + _metrics()->startCopyingDocuments(getCurrentTime()); } void ReshardingRecipientService::RecipientStateMachine::_transitionToApplying( const CancelableOperationContextFactory& factory) { auto newRecipientCtx = _recipientCtx; newRecipientCtx.setState(RecipientStateEnum::kApplying); - auto oplogApplicationStartTime = getCurrentTime(); - - // Record oplog application start time. - ReshardingMetricsTimeInterval interval; - interval.setStart(oplogApplicationStartTime); - _timeIntervals.setOplogApplication(interval); - - // Record document copy stop time. - ReshardingMetricsTimeInterval documentCopy{_timeIntervals.getDocumentCopy().get_value_or({})}; - documentCopy.setStop(oplogApplicationStartTime); - _timeIntervals.setDocumentCopy(documentCopy); - _transitionState(std::move(newRecipientCtx), boost::none, boost::none, factory); - _metrics()->endCopyingDocuments(oplogApplicationStartTime); - _metrics()->startApplyingOplogEntries(oplogApplicationStartTime); + auto currentTime = getCurrentTime(); + _metrics()->endCopyingDocuments(currentTime); + _metrics()->startApplyingOplogEntries(currentTime); } void ReshardingRecipientService::RecipientStateMachine::_transitionToStrictConsistency( const CancelableOperationContextFactory& factory) { auto newRecipientCtx = _recipientCtx; newRecipientCtx.setState(RecipientStateEnum::kStrictConsistency); - auto oplogApplicationStopTime = getCurrentTime(); - - // Record oplog application stop time - ReshardingMetricsTimeInterval oplogApplication{ - _timeIntervals.getOplogApplication().get_value_or({})}; - oplogApplication.setStop(oplogApplicationStopTime); - _timeIntervals.setOplogApplication(oplogApplication); - - _transitionState(std::move(newRecipientCtx), boost::none, boost::none, factory); - _metrics()->endApplyingOplogEntries(oplogApplicationStopTime); + auto currentTime = getCurrentTime(); + _metrics()->endApplyingOplogEntries(currentTime); } void ReshardingRecipientService::RecipientStateMachine::_transitionToError( @@ -1015,9 +980,6 @@ void ReshardingRecipientService::RecipientStateMachine::_updateRecipientDocument setBuilder.append(ReshardingRecipientDocument::kDonorShardsFieldName, donorShardsArrayBuilder.arr()); - - setBuilder.append(ReshardingRecipientDocument::kApproxBytesToCopyFieldName, - cloneDetails->approxBytesToCopy); } if (configStartTime) { @@ -1025,8 +987,6 @@ void ReshardingRecipientService::RecipientStateMachine::_updateRecipientDocument *configStartTime); } - setBuilder.append(ReshardingRecipientDocument::kMetricsFieldName, _timeIntervals.toBSON()); - setBuilder.doneFast(); } @@ -1044,7 +1004,6 @@ void ReshardingRecipientService::RecipientStateMachine::_updateRecipientDocument if (cloneDetails) { _cloneTimestamp = cloneDetails->cloneTimestamp; _donorShards = std::move(cloneDetails->donorShards); - _approxBytesToCopy = cloneDetails->approxBytesToCopy; } if (configStartTime) { @@ -1104,6 +1063,7 @@ ExecutorFuture<void> ReshardingRecipientService::RecipientStateMachine::_startMe const std::shared_ptr<executor::ScopedTaskExecutor>& executor, const CancellationToken& abortToken) { if (_recipientCtx.getState() > RecipientStateEnum::kAwaitingFetchTimestamp) { + _metrics()->onStepUp(ReshardingMetrics::Role::kRecipient); return _restoreMetricsWithRetry(executor, abortToken); } _metrics()->onStart(ReshardingMetrics::Role::kRecipient, getCurrentTime()); @@ -1113,6 +1073,7 @@ ExecutorFuture<void> ReshardingRecipientService::RecipientStateMachine::_startMe ExecutorFuture<void> ReshardingRecipientService::RecipientStateMachine::_restoreMetricsWithRetry( const std::shared_ptr<executor::ScopedTaskExecutor>& executor, const CancellationToken& abortToken) { + _metrics()->setRecipientState(_recipientCtx.getState()); return _retryingCancelableOpCtxFactory ->withAutomaticRetry( [this, executor, abortToken](const auto& factory) { _restoreMetrics(factory); }) @@ -1178,13 +1139,8 @@ void ReshardingRecipientService::RecipientStateMachine::_restoreMetrics( } } - _metrics()->onStepUp(_recipientCtx.getState(), - ReshardingMetrics::ReshardingRecipientCountsAndMetrics{documentCountCopied, - documentBytesCopied, - oplogEntriesFetched, - oplogEntriesApplied, - _approxBytesToCopy, - _timeIntervals}); + _metrics()->restoreForCurrentOp( + documentCountCopied, documentBytesCopied, oplogEntriesFetched, oplogEntriesApplied); } CancellationToken ReshardingRecipientService::RecipientStateMachine::_initAbortSource( diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.h b/src/mongo/db/s/resharding/resharding_recipient_service.h index 86d3236b9c9..e1fe504a970 100644 --- a/src/mongo/db/s/resharding/resharding_recipient_service.h +++ b/src/mongo/db/s/resharding/resharding_recipient_service.h @@ -288,8 +288,6 @@ private: RecipientShardContext _recipientCtx; std::vector<DonorShardFetchTimestamp> _donorShards; boost::optional<Timestamp> _cloneTimestamp; - ReshardingRecipientMetrics _timeIntervals; - boost::optional<int64_t> _approxBytesToCopy; const std::unique_ptr<RecipientStateMachineExternalState> _externalState; diff --git a/src/mongo/db/s/resharding/resharding_recipient_service_external_state.cpp b/src/mongo/db/s/resharding/resharding_recipient_service_external_state.cpp index b99f1f60437..2dc584953d6 100644 --- a/src/mongo/db/s/resharding/resharding_recipient_service_external_state.cpp +++ b/src/mongo/db/s/resharding/resharding_recipient_service_external_state.cpp @@ -124,8 +124,13 @@ RecipientStateMachineExternalStateImpl::getCollectionOptions(OperationContext* o StringData reason) { // Load the collection options from the primary shard for the database. return _withShardVersionRetry(opCtx, nss, reason, [&] { + auto cm = getShardedCollectionRoutingInfo(opCtx, nss); return MigrationDestinationManager::getCollectionOptions( - opCtx, NamespaceStringOrUUID{nss.db().toString(), uuid}, afterClusterTime); + opCtx, + NamespaceStringOrUUID{nss.db().toString(), uuid}, + cm.dbPrimary(), + cm, + afterClusterTime); }); } diff --git a/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp b/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp index b598f32baf1..43ccab32bec 100644 --- a/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp +++ b/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp @@ -48,13 +48,10 @@ #include "mongo/db/s/resharding/resharding_recipient_service.h" #include "mongo/db/s/resharding/resharding_recipient_service_external_state.h" #include "mongo/db/s/resharding/resharding_service_test_helpers.h" -#include "mongo/db/s/sharding_ddl_util.h" #include "mongo/logv2/log.h" #include "mongo/unittest/death_test.h" -#include "mongo/util/clock_source_mock.h" #include "mongo/util/fail_point.h" - namespace mongo { namespace { @@ -211,8 +208,6 @@ public: */ class ReshardingRecipientServiceTest : public repl::PrimaryOnlyServiceMongoDTest { public: - ReshardingRecipientServiceTest() : PrimaryOnlyServiceMongoDTest(Options{}.useMockClock(true)) {} - using RecipientStateMachine = ReshardingRecipientService::RecipientStateMachine; std::unique_ptr<repl::PrimaryOnlyService> makeService(ServiceContext* serviceContext) override { @@ -227,6 +222,7 @@ public: repl::DropPendingCollectionReaper::set( serviceContext, std::make_unique<repl::DropPendingCollectionReaper>(storageMock.get())); repl::StorageInterface::set(serviceContext, std::move(storageMock)); + _controller = std::make_shared<RecipientStateTransitionController>(); _opObserverRegistry->addObserver(std::make_unique<RecipientOpObserverForTest>(_controller)); } @@ -267,8 +263,7 @@ public: const ReshardingRecipientDocument& recipientDoc) { CollectionOptions options; options.uuid = recipientDoc.getSourceUUID(); - mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent(opCtx, - recipientDoc.getSourceNss()); + resharding::data_copy::ensureCollectionDropped(opCtx, recipientDoc.getSourceNss()); resharding::data_copy::ensureCollectionExists(opCtx, recipientDoc.getSourceNss(), options); } @@ -379,10 +374,9 @@ TEST_F(ReshardingRecipientServiceTest, StepDownStepUpEachTransition) { RecipientStateMachine::insertStateDocument(opCtx.get(), doc); return RecipientStateMachine::getOrCreate(opCtx.get(), _service, doc.toBSON()); } else { - auto [maybeRecipient, isPausedOrShutdown] = + auto maybeRecipient = RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeRecipient); - ASSERT_FALSE(isPausedOrShutdown); + ASSERT_TRUE(bool(maybeRecipient)); // Allow the transition to prevState to succeed on this primary-only service // instance. @@ -429,10 +423,8 @@ TEST_F(ReshardingRecipientServiceTest, StepDownStepUpEachTransition) { } // Finally complete the operation and ensure its success. - auto [maybeRecipient, isPausedOrShutdown] = - RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeRecipient); - ASSERT_FALSE(isPausedOrShutdown); + auto maybeRecipient = RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeRecipient)); auto recipient = *maybeRecipient; @@ -477,10 +469,8 @@ TEST_F(ReshardingRecipientServiceTest, OpCtxKilledWhileRestoringMetrics) { stepUp(opCtx.get()); // After the failpoint is disabled, the operation should succeed. - auto [maybeRecipient, isPausedOrShutdown] = - RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeRecipient); - ASSERT_FALSE(isPausedOrShutdown); + auto maybeRecipient = RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeRecipient)); recipient = *maybeRecipient; notifyReshardingCommitting(opCtx.get(), *recipient, doc); ASSERT_OK(recipient->getCompletionFuture().getNoThrow()); @@ -506,89 +496,69 @@ DEATH_TEST_REGEX_F(ReshardingRecipientServiceTest, CommitFn, "4457001.*tripwire" TEST_F(ReshardingRecipientServiceTest, DropsTemporaryReshardingCollectionOnAbort) { auto metrics = ReshardingRecipientServiceTest::metrics(); for (bool isAlsoDonor : {false, true}) { - for (bool waitForMetricsInitialized : {false, true}) { - LOGV2(5551107, - "Running case", - "test"_attr = _agent.getTestName(), - "isAlsoDonor"_attr = isAlsoDonor, - "waitForMetricsInitialized"_attr = waitForMetricsInitialized); - - boost::optional<PauseDuringStateTransitions> stateTransitionGuard; - if (waitForMetricsInitialized) { - std::vector<RecipientStateEnum> recipientStates{ - RecipientStateEnum::kDone, RecipientStateEnum::kCreatingCollection}; - stateTransitionGuard.emplace(controller(), recipientStates); - } else { - stateTransitionGuard.emplace(controller(), RecipientStateEnum::kDone); - } - - auto doc = makeStateDocument(isAlsoDonor); - auto instanceId = BSON(ReshardingRecipientDocument::kReshardingUUIDFieldName - << doc.getReshardingUUID()); + LOGV2(5551107, + "Running case", + "test"_attr = _agent.getTestName(), + "isAlsoDonor"_attr = isAlsoDonor); - auto opCtx = makeOperationContext(); + boost::optional<PauseDuringStateTransitions> doneTransitionGuard; + doneTransitionGuard.emplace(controller(), RecipientStateEnum::kDone); - if (isAlsoDonor) { - // If the recipient is also a donor, the original collection should already exist on - // this shard. - createSourceCollection(opCtx.get(), doc); - } + auto doc = makeStateDocument(isAlsoDonor); + auto instanceId = + BSON(ReshardingRecipientDocument::kReshardingUUIDFieldName << doc.getReshardingUUID()); - RecipientStateMachine::insertStateDocument(opCtx.get(), doc); - auto recipient = - RecipientStateMachine::getOrCreate(opCtx.get(), _service, doc.toBSON()); + auto opCtx = makeOperationContext(); - notifyToStartCloning(opCtx.get(), *recipient, doc); - if (waitForMetricsInitialized) { - // Waiting for the metrics to be initialized here causes the second abort to occur - // before the metrics are initialized after the step up, thereby testing a different - // code path. - stateTransitionGuard->wait(RecipientStateEnum::kCreatingCollection); - } + if (isAlsoDonor) { + // If the recipient is also a donor, the original collection should already exist on + // this shard. + createSourceCollection(opCtx.get(), doc); + } - recipient->abort(false); + RecipientStateMachine::insertStateDocument(opCtx.get(), doc); + auto recipient = RecipientStateMachine::getOrCreate(opCtx.get(), _service, doc.toBSON()); - stateTransitionGuard->wait(RecipientStateEnum::kDone); + notifyToStartCloning(opCtx.get(), *recipient, doc); + recipient->abort(false); - stepDown(); + doneTransitionGuard->wait(RecipientStateEnum::kDone); + stepDown(); - ASSERT_EQ(recipient->getCompletionFuture().getNoThrow(), - ErrorCodes::InterruptedDueToReplStateChange); + ASSERT_EQ(recipient->getCompletionFuture().getNoThrow(), + ErrorCodes::InterruptedDueToReplStateChange); - recipient.reset(); - stepUp(opCtx.get()); + recipient.reset(); + stepUp(opCtx.get()); - auto [maybeRecipient, isPausedOrShutdown] = - RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeRecipient); - ASSERT_FALSE(isPausedOrShutdown); - recipient = *maybeRecipient; + auto maybeRecipient = RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); + ASSERT_TRUE(bool(maybeRecipient)); + recipient = *maybeRecipient; - stateTransitionGuard.reset(); - recipient->abort(false); + doneTransitionGuard.reset(); + recipient->abort(false); - ASSERT_OK(recipient->getCompletionFuture().getNoThrow()); - checkStateDocumentRemoved(opCtx.get()); + ASSERT_OK(recipient->getCompletionFuture().getNoThrow()); + checkStateDocumentRemoved(opCtx.get()); - if (isAlsoDonor) { - // Verify original collection still exists after aborting. - AutoGetCollection coll(opCtx.get(), doc.getSourceNss(), MODE_IS); - ASSERT_TRUE(bool(coll)); - ASSERT_EQ(coll->uuid(), doc.getSourceUUID()); - } + if (isAlsoDonor) { + // Verify original collection still exists after aborting. + AutoGetCollection coll(opCtx.get(), doc.getSourceNss(), MODE_IS); + ASSERT_TRUE(bool(coll)); + ASSERT_EQ(coll->uuid(), doc.getSourceUUID()); + } - // Verify the temporary collection no longer exists. - { - AutoGetCollection coll(opCtx.get(), doc.getTempReshardingNss(), MODE_IS); - ASSERT_FALSE(bool(coll)); - } + // Verify the temporary collection no longer exists. + { + AutoGetCollection coll(opCtx.get(), doc.getTempReshardingNss(), MODE_IS); + ASSERT_FALSE(bool(coll)); } } BSONObjBuilder result; metrics->serializeCumulativeOpMetrics(&result); - ASSERT_LESS_THAN_OR_EQUALS(result.obj().getField("countReshardingFailures").numberLong(), 4); + ASSERT_EQ(result.obj().getField("countReshardingFailures").numberLong(), 2); } TEST_F(ReshardingRecipientServiceTest, RenamesTemporaryReshardingCollectionWhenDone) { @@ -778,10 +748,9 @@ TEST_F(ReshardingRecipientServiceTest, RestoreMetricsAfterStepUp) { RecipientStateMachine::insertStateDocument(opCtx.get(), doc); return RecipientStateMachine::getOrCreate(opCtx.get(), _service, doc.toBSON()); } else { - auto [maybeRecipient, isPausedOrShutdown] = + auto maybeRecipient = RecipientStateMachine::lookup(opCtx.get(), _service, instanceId); - ASSERT_TRUE(maybeRecipient); - ASSERT_FALSE(isPausedOrShutdown); + ASSERT_TRUE(bool(maybeRecipient)); // Allow the transition to prevState to succeed on this primary-only service // instance. @@ -847,33 +816,22 @@ TEST_F(ReshardingRecipientServiceTest, RestoreMetricsAfterStepUp) { } // Step down before the transition to state can complete. stateTransitionsGuard.wait(state); - - dynamic_cast<ClockSourceMock*>(getServiceContext()->getFastClockSource()) - ->advance(Seconds(1)); - auto currOp = - recipient - ->reportForCurrentOp(MongoProcessInterface::CurrentOpConnectionsMode::kExcludeIdle, - MongoProcessInterface::CurrentOpSessionsMode::kExcludeIdle) - .get(); - - - if (state == RecipientStateEnum::kApplying) { - ASSERT_EQ(currOp.getField("totalApplyTimeElapsedSecs").Long(), 0); - ASSERT_EQ(currOp.getStringField("recipientState"), - RecipientState_serializer(RecipientStateEnum::kCloning)); - ASSERT_GT(currOp.getField("totalCopyTimeElapsedSecs").Long(), 0); - ASSERT_GT(currOp.getField("approxBytesToCopy").Long(), 0); - - } else if (state == RecipientStateEnum::kStrictConsistency) { + if (state == RecipientStateEnum::kStrictConsistency) { + auto currOp = recipient + ->reportForCurrentOp( + MongoProcessInterface::CurrentOpConnectionsMode::kExcludeIdle, + MongoProcessInterface::CurrentOpSessionsMode::kExcludeIdle) + .get(); ASSERT_EQ(currOp.getField("documentsCopied").Long(), 1L); ASSERT_EQ(currOp.getField("bytesCopied").Long(), (long)reshardedDoc.objsize()); ASSERT_EQ(currOp.getStringField("recipientState"), RecipientState_serializer(RecipientStateEnum::kApplying)); - ASSERT_GT(currOp.getField("totalCopyTimeElapsedSecs").Long(), 0); - ASSERT_GT(currOp.getField("totalApplyTimeElapsedSecs").Long(), 0); - ASSERT_GT(currOp.getField("approxBytesToCopy").Long(), 0); - } else if (state == RecipientStateEnum::kDone) { + auto currOp = recipient + ->reportForCurrentOp( + MongoProcessInterface::CurrentOpConnectionsMode::kExcludeIdle, + MongoProcessInterface::CurrentOpSessionsMode::kExcludeIdle) + .get(); ASSERT_EQ(currOp.getField("documentsCopied").Long(), 1L); ASSERT_EQ(currOp.getField("bytesCopied").Long(), (long)reshardedDoc.objsize()); ASSERT_EQ(currOp.getField("oplogEntriesFetched").Long(), @@ -882,11 +840,7 @@ TEST_F(ReshardingRecipientServiceTest, RestoreMetricsAfterStepUp) { oplogEntriesAppliedOnEachDonor * doc.getDonorShards().size()); ASSERT_EQ(currOp.getStringField("recipientState"), RecipientState_serializer(RecipientStateEnum::kStrictConsistency)); - ASSERT_GT(currOp.getField("totalCopyTimeElapsedSecs").Long(), 0); - ASSERT_GT(currOp.getField("totalApplyTimeElapsedSecs").Long(), 0); - ASSERT_GT(currOp.getField("approxBytesToCopy").Long(), 0); } - stepDown(); ASSERT_EQ(recipient->getCompletionFuture().getNoThrow(), diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp index 96a08c210fa..7ab491286f3 100644 --- a/src/mongo/db/s/resharding/resharding_txn_cloner.cpp +++ b/src/mongo/db/s/resharding/resharding_txn_cloner.cpp @@ -40,6 +40,7 @@ #include "mongo/client/read_preference.h" #include "mongo/db/client.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/curop.h" #include "mongo/db/logical_session_id.h" #include "mongo/db/namespace_string.h" @@ -190,8 +191,7 @@ boost::optional<SharedSemiFuture<void>> ReshardingTxnCloner::doOneRecord( TransactionParticipant::kDeadEndSentinel, {kIncompleteHistoryStmtId}, boost::none /* preImageOpTime */, - boost::none /* postImageOpTime */, - {}); + boost::none /* postImageOpTime */); }); } diff --git a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp index b77db34ef11..e4f4f1f2b61 100644 --- a/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp +++ b/src/mongo/db/s/resharding/resharding_txn_cloner_test.cpp @@ -40,7 +40,6 @@ #include "mongo/db/logical_session_cache_noop.h" #include "mongo/db/persistent_task_store.h" #include "mongo/db/pipeline/process_interface/shardsvr_process_interface.h" -#include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/repl/wait_for_majority_service.h" #include "mongo/db/s/resharding/resharding_server_parameters_gen.h" @@ -130,9 +129,7 @@ class ReshardingTxnClonerTest : public ShardServerTestFixture { StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { std::vector<ShardType> shardTypes; for (const auto& shardId : _shardIds) { const ConnectionString cs = ConnectionString::forReplicaSet( diff --git a/src/mongo/db/s/resharding/resharding_util.cpp b/src/mongo/db/s/resharding/resharding_util.cpp index 3e0db3b383e..27d319264dd 100644 --- a/src/mongo/db/s/resharding/resharding_util.cpp +++ b/src/mongo/db/s/resharding/resharding_util.cpp @@ -38,7 +38,7 @@ #include "mongo/bson/bsonobj.h" #include "mongo/bson/json.h" #include "mongo/bson/util/bson_extract.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/exec/document_value/document.h" #include "mongo/db/namespace_string.h" #include "mongo/db/op_observer.h" @@ -207,14 +207,11 @@ void checkForOverlappingZones(std::vector<ReshardingZoneType>& zones) { } std::vector<BSONObj> buildTagsDocsFromZones(const NamespaceString& tempNss, - const std::vector<ReshardingZoneType>& zones, - const ShardKeyPattern& shardKey) { + const std::vector<ReshardingZoneType>& zones) { std::vector<BSONObj> tags; tags.reserve(zones.size()); - for (auto& zone : zones) { - auto min = shardKey.getKeyPattern().extendRangeBound(zone.getMin(), false); - auto max = shardKey.getKeyPattern().extendRangeBound(zone.getMax(), false); - ChunkRange range(min, max); + for (const auto& zone : zones) { + ChunkRange range(zone.getMin(), zone.getMax()); TagsType tag(tempNss, zone.getZone().toString(), range); tags.push_back(tag.toBSON()); } diff --git a/src/mongo/db/s/resharding/resharding_util.h b/src/mongo/db/s/resharding/resharding_util.h index f0e60e99555..856d7cbb081 100644 --- a/src/mongo/db/s/resharding/resharding_util.h +++ b/src/mongo/db/s/resharding/resharding_util.h @@ -260,8 +260,7 @@ void checkForOverlappingZones(std::vector<ReshardingZoneType>& zones); * Builds documents to insert into config.tags from zones provided to reshardCollection cmd. */ std::vector<BSONObj> buildTagsDocsFromZones(const NamespaceString& tempNss, - const std::vector<ReshardingZoneType>& zones, - const ShardKeyPattern& shardKey); + const std::vector<ReshardingZoneType>& zones); /** * Creates a pipeline that can be serialized into a query for fetching oplog entries. `startAfter` diff --git a/src/mongo/db/s/scoped_operation_completion_sharding_actions.cpp b/src/mongo/db/s/scoped_operation_completion_sharding_actions.cpp index 791bfb50f8b..7e3d89d58f0 100644 --- a/src/mongo/db/s/scoped_operation_completion_sharding_actions.cpp +++ b/src/mongo/db/s/scoped_operation_completion_sharding_actions.cpp @@ -37,8 +37,6 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/s/sharding_statistics.h" #include "mongo/logv2/log.h" -#include "mongo/s/grid.h" -#include "mongo/s/shard_cannot_refresh_due_to_locks_held_exception.h" #include "mongo/s/stale_exception.h" namespace mongo { @@ -123,18 +121,8 @@ ScopedOperationCompletionShardingActions::~ScopedOperationCompletionShardingActi LOGV2(22054, "Failed to handle database version exception as part of the current operation: " "{error}", - "Failed to handle database version exception as part of the current operation", + "Failed to database version exception as part of the current operation", "error"_attr = redact(handleMismatchStatus)); - } else if (auto failedRefreshInfo = status->extraInfo<ShardCannotRefreshDueToLocksHeldInfo>()) { - // It is OK to synchronously refresh the catalog cache here. - const auto swCri = Grid::get(_opCtx)->catalogCache()->getCollectionRoutingInfo( - _opCtx, failedRefreshInfo->getNss()); - if (!swCri.getStatus().isOK()) { - LOGV2(8150800, - "Failed to handle ShardCannotRefreshDueToLocksHeld as part of the current " - "operation", - "error"_attr = redact(swCri.getStatus())); - } } } diff --git a/src/mongo/db/s/session_catalog_migration_destination.cpp b/src/mongo/db/s/session_catalog_migration_destination.cpp index 34864708db1..f00a2215460 100644 --- a/src/mongo/db/s/session_catalog_migration_destination.cpp +++ b/src/mongo/db/s/session_catalog_migration_destination.cpp @@ -39,7 +39,7 @@ #include "mongo/client/connection_string.h" #include "mongo/db/cancelable_operation_context.h" #include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/logical_session_id.h" #include "mongo/db/ops/write_ops_retryability.h" #include "mongo/db/repl/oplog.h" @@ -58,8 +58,6 @@ namespace mongo { namespace { -MONGO_FAIL_POINT_DEFINE(interruptBeforeProcessingPrePostImageOriginatingOp); - const auto kOplogField = "oplog"; const WriteConcernOptions kMajorityWC(WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, @@ -254,7 +252,6 @@ ProcessOplogResult processSessionOplog(const BSONObj& oplogBSON, auto uniqueOpCtx = CancelableOperationContext(cc().makeOperationContext(), cancellationToken, executor); auto opCtx = uniqueOpCtx.get(); - opCtx->setAlwaysInterruptAtStepDownOrUp(); opCtx->setLogicalSessionId(result.sessionId); opCtx->setTxnNumber(result.txnNum); MongoDOperationContextSession ocs(opCtx); @@ -443,7 +440,6 @@ void SessionCatalogMigrationDestination::_retrieveSessionStateFromSource(Service auto uniqueCtx = CancelableOperationContext( cc().makeOperationContext(), _cancellationToken, executor); auto opCtx = uniqueCtx.get(); - opCtx->setAlwaysInterruptAtStepDownOrUp(); nextBatch = getNextSessionOplogBatch(opCtx, _fromShard, _migrationSessionId); oplogArray = BSONArray{nextBatch[kOplogField].Obj()}; @@ -495,24 +491,10 @@ void SessionCatalogMigrationDestination::_retrieveSessionStateFromSource(Service lastOpTimeWaited = lastResult.oplogTime; } } - for (BSONArrayIteratorSorted oplogIter(oplogArray); oplogIter.more();) { - auto oplogEntry = oplogIter.next().Obj(); - interruptBeforeProcessingPrePostImageOriginatingOp.executeIf( - [&](const auto&) { - uasserted(6749200, - "Intentionally failing session migration before processing post/pre " - "image originating update oplog entry"); - }, - // SERVER-68728 The latter two conditions are needed if the donor shard is v5.0 in - // multi-version clusters - [&](const auto&) { - return !oplogEntry["needsRetryImage"].eoo() || - !oplogEntry["preImageOpTime"].eoo() || !oplogEntry["postImageOpTime"].eoo(); - }); try { - lastResult = - processSessionOplog(oplogEntry, lastResult, service, _cancellationToken); + lastResult = processSessionOplog( + oplogIter.next().Obj(), lastResult, service, _cancellationToken); } catch (const ExceptionFor<ErrorCodes::TransactionTooOld>&) { // This means that the server has a newer txnNumber than the oplog being // migrated, so just skip it @@ -526,7 +508,6 @@ void SessionCatalogMigrationDestination::_retrieveSessionStateFromSource(Service auto executor = Grid::get(service)->getExecutorPool()->getFixedExecutor(); auto uniqueOpCtx = CancelableOperationContext(cc().makeOperationContext(), _cancellationToken, executor); - uniqueOpCtx->setAlwaysInterruptAtStepDownOrUp(); uassertStatusOK( waitForWriteConcern(uniqueOpCtx.get(), lastResult.oplogTime, kMajorityWC, &unusedWCResult)); diff --git a/src/mongo/db/s/session_catalog_migration_destination_test.cpp b/src/mongo/db/s/session_catalog_migration_destination_test.cpp index cfff58b6540..d1e5a11848c 100644 --- a/src/mongo/db/s/session_catalog_migration_destination_test.cpp +++ b/src/mongo/db/s/session_catalog_migration_destination_test.cpp @@ -98,7 +98,6 @@ repl::OplogEntry makeOplogEntry( kNs, // namespace boost::none, // uuid boost::none, // fromMigrate - boost::none, // checkExistenceForDiffInsert 0, // version object, // o object2, // o2 @@ -291,9 +290,7 @@ private: StaticCatalogClient() = default; StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { ShardType donorShard; donorShard.setName(kDonorConnStr.getSetName()); diff --git a/src/mongo/db/s/session_catalog_migration_source.cpp b/src/mongo/db/s/session_catalog_migration_source.cpp index c83439080e0..5706e76ba44 100644 --- a/src/mongo/db/s/session_catalog_migration_source.cpp +++ b/src/mongo/db/s/session_catalog_migration_source.cpp @@ -34,7 +34,7 @@ #include <memory> #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/namespace_string.h" #include "mongo/db/op_observer.h" @@ -50,7 +50,6 @@ #include "mongo/db/transaction_history_iterator.h" #include "mongo/db/transaction_participant.h" #include "mongo/db/write_concern.h" -#include "mongo/logv2/redaction.h" #include "mongo/platform/random.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/shard_key_pattern.h" @@ -160,7 +159,6 @@ repl::OplogEntry makeOplogEntry(repl::OpTime opTime, {}, // namespace boost::none, // uuid boost::none, // fromMigrate - boost::none, // checkExistenceForDiffInsert repl::OplogEntry::kOplogVersion, // version oField, // o o2Field, // o2 @@ -485,23 +483,11 @@ bool SessionCatalogMigrationSource::_handleWriteHistory(WithLock lk, OperationCo // oplog entries derived from it to the oplog buffer. if (isInternalSessionForRetryableWrite(*nextOplog->getSessionId())) { - if (nextOplog->getCommandType() == repl::OplogEntry::CommandType::kApplyOps) { - // Derive retryable write oplog entries from this retryable internal transaction - // applyOps oplog entry, and add them to the oplog buffer. - _extractOplogEntriesForInternalTransactionForRetryableWrite( - lk, *nextOplog, &_unprocessedOplogBuffer); - } else { - tassert(7393800, - str::stream() << "Found an oplog entry for a retrayble internal " - "transaction with an unexpected type" - << redact(nextOplog->toBSONForLogging()), - nextOplog->getOpType() == repl::OpTypeEnum::kNoop); - if (!nextOplog->getStatementIds().empty() && - !shouldSkipOplogEntry(nextOplog.value(), _keyPattern, _chunkRange)) { - _unprocessedOplogBuffer.emplace_back(*nextOplog); - } - } - + invariant(nextOplog->getCommandType() == repl::OplogEntry::CommandType::kApplyOps); + // Derive retryable write oplog entries from this retryable internal transaction + // applyOps oplog entry, and add them to the oplog buffer. + _extractOplogEntriesForInternalTransactionForRetryableWrite( + lk, *nextOplog, &_unprocessedOplogBuffer); continue; } diff --git a/src/mongo/db/s/session_catalog_migration_source_test.cpp b/src/mongo/db/s/session_catalog_migration_source_test.cpp index 726cf6c239d..edf1049a847 100644 --- a/src/mongo/db/s/session_catalog_migration_source_test.cpp +++ b/src/mongo/db/s/session_catalog_migration_source_test.cpp @@ -96,7 +96,6 @@ repl::OplogEntry makeOplogEntry(repl::OpTime opTime, nss, // namespace boost::none, // uuid boost::none, // fromMigrate - boost::none, // checkExistenceForDiffInsert repl::OplogEntry::kOplogVersion, // version object, // o object2, // o2 diff --git a/src/mongo/db/s/sessions_collection_config_server.cpp b/src/mongo/db/s/sessions_collection_config_server.cpp index 4cc60077aec..48374167b41 100644 --- a/src/mongo/db/s/sessions_collection_config_server.cpp +++ b/src/mongo/db/s/sessions_collection_config_server.cpp @@ -30,8 +30,6 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kControl #include "mongo/db/s/sessions_collection_config_server.h" -#include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/repl/replication_coordinator.h" #include "mongo/logv2/log.h" #include "mongo/s/client/shard_registry.h" @@ -98,10 +96,8 @@ void SessionsCollectionConfigServer::_generateIndexesIfNeeded(OperationContext* SessionsCollection::generateCreateIndexesCmd(), ReadPreferenceSetting(ReadPreference::PrimaryOnly), Shard::RetryPolicy::kNoRetry, - BSONObj() /*query*/, - BSONObj() /*collation*/, - boost::none /*letParameters*/, - boost::none /*runtimeConstants*/); + BSONObj() /* query */, + BSONObj() /* collation */); }); } @@ -115,29 +111,6 @@ void SessionsCollectionConfigServer::setupSessionsCollection(OperationContext* o _shardCollectionIfNeeded(opCtx); _generateIndexesIfNeeded(opCtx); - static constexpr int64_t kAverageSessionDocSizeBytes = 200; - static constexpr int64_t kDesiredDocsInChunks = 1000; - static constexpr int64_t kMaxChunkSizeBytes = - kAverageSessionDocSizeBytes * kDesiredDocsInChunks; - - Lock::GlobalLock lock(opCtx, MODE_IX); - if (const auto replCoord = repl::ReplicationCoordinator::get(opCtx); - replCoord->canAcceptWritesFor(opCtx, CollectionType::ConfigNS)) { - auto filterQuery = - BSON("_id" << NamespaceString::kLogicalSessionsNamespace.ns() - << CollectionType::kMaxChunkSizeBytesFieldName << BSON("$exists" << false)); - auto updateQuery = BSON("$set" << BSON(CollectionType::kMaxChunkSizeBytesFieldName - << kMaxChunkSizeBytes - << CollectionType::kNoAutoSplitFieldName << true)); - - uassertStatusOK(Grid::get(opCtx)->catalogClient()->updateConfigDocument( - opCtx, - CollectionType::ConfigNS, - filterQuery, - updateQuery, - false, - ShardingCatalogClient::kLocalWriteConcern)); - } } } // namespace mongo diff --git a/src/mongo/db/s/set_allow_migrations_coordinator.cpp b/src/mongo/db/s/set_allow_migrations_coordinator.cpp index aee3052a8ee..1cf2edac166 100644 --- a/src/mongo/db/s/set_allow_migrations_coordinator.cpp +++ b/src/mongo/db/s/set_allow_migrations_coordinator.cpp @@ -122,12 +122,12 @@ ExecutorFuture<void> SetAllowMigrationsCoordinator::_runImpl( return updateOp; }()); - auto response = - configShard->runBatchWriteCommand(opCtx, - Shard::kDefaultConfigCommandTimeout, - updateRequest, - ShardingCatalogClient::kMajorityWriteConcern, - Shard::RetryPolicy::kIdempotent); + updateRequest.setWriteConcern(ShardingCatalogClient::kMajorityWriteConcern.toBSON()); + + auto response = configShard->runBatchWriteCommand(opCtx, + Shard::kDefaultConfigCommandTimeout, + updateRequest, + Shard::RetryPolicy::kIdempotent); uassertStatusOK(response.toStatus()); }) diff --git a/src/mongo/db/s/shard_filtering_metadata_refresh.cpp b/src/mongo/db/s/shard_filtering_metadata_refresh.cpp index 3c5d5f70a04..0a7cecfe632 100644 --- a/src/mongo/db/s/shard_filtering_metadata_refresh.cpp +++ b/src/mongo/db/s/shard_filtering_metadata_refresh.cpp @@ -37,7 +37,6 @@ #include "mongo/db/operation_context.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/database_sharding_state.h" -#include "mongo/db/s/forwardable_operation_metadata.h" #include "mongo/db/s/migration_source_manager.h" #include "mongo/db/s/migration_util.h" #include "mongo/db/s/operation_sharding_state.h" @@ -56,294 +55,69 @@ MONGO_FAIL_POINT_DEFINE(skipDatabaseVersionMetadataRefresh); MONGO_FAIL_POINT_DEFINE(skipShardFilteringMetadataRefresh); MONGO_FAIL_POINT_DEFINE(hangInRecoverRefreshThread); -/** - * Blocking method, which will wait for any concurrent operations that could change the database - * version to complete (namely critical section and concurrent onDbVersionMismatch invocations). - * - * Returns 'true' if there were concurrent operations that had to be joined (in which case all locks - * will be dropped). If there were none, returns false and the locks continue to be held. - */ -bool joinDbVersionOperation(OperationContext* opCtx, - DatabaseShardingState* dss, - boost::optional<Lock::DBLock>* dbLock, - boost::optional<DatabaseShardingState::DSSLock>* dssLock) { - invariant(dbLock->has_value()); - invariant(dssLock->has_value()); - - if (auto critSect = - dss->getCriticalSectionSignal(ShardingMigrationCriticalSection::kWrite, **dssLock)) { - LOGV2_DEBUG(6697201, - 2, - "Waiting for exit from the critical section", - "db"_attr = dss->getDbName(), - "reason"_attr = dss->getCriticalSectionReason(**dssLock)); - - dbLock->reset(); - dssLock->reset(); - - uassertStatusOK(OperationShardingState::waitForCriticalSectionToComplete(opCtx, *critSect)); - return true; - } - - if (auto refreshVersionFuture = dss->getDbMetadataRefreshFuture(**dssLock)) { - LOGV2_DEBUG(6697202, - 2, - "Waiting for completion of another database metadata refresh", - "db"_attr = dss->getDbName()); - - dbLock->reset(); - dssLock->reset(); - - try { - refreshVersionFuture->get(opCtx); - } catch (const ExceptionFor<ErrorCodes::DatabaseMetadataRefreshCanceled>&) { - // The refresh was canceled by another thread that entered the critical section. - } - - return true; - } - - return false; -} - -/** - * Unconditionally refreshes the database metadata from the config server. - * - * NOTE: Does network I/O and acquires the database lock in X mode. - */ -Status refreshDbMetadata(OperationContext* opCtx, - const StringData& dbName, - const CancellationToken& cancellationToken) { - invariant(!opCtx->lockState()->isLocked()); - invariant(!opCtx->getClient()->isInDirectClient()); - invariant(ShardingState::get(opCtx)->canAcceptShardedCommands()); - - ScopeGuard resetRefreshFutureOnError([&] { - UninterruptibleLockGuard noInterrupt(opCtx->lockState()); - - Lock::DBLock dbLock(opCtx, dbName, MODE_IX); - auto* dss = DatabaseShardingState::get(opCtx, dbName); - const auto dssLock = DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss); - - dss->resetDbMetadataRefreshFuture(dssLock); - }); - - // Force a refresh of the cached database metadata from the config server. - const auto swDbMetadata = - Grid::get(opCtx)->catalogCache()->getDatabaseWithRefresh(opCtx, dbName); - - // Before setting the database metadata, exit early if the database version received by the - // config server is not newer than the cached one. This is a best-effort optimization to reduce - // the number of possible threads convoying on the exclusive lock below. - { - Lock::DBLock dbLock(opCtx, dbName, MODE_IS); - auto* dss = DatabaseShardingState::get(opCtx, dbName); - auto dssLock = DatabaseShardingState::DSSLock::lockShared(opCtx, dss); - - const auto cachedDbVersion = dss->getDbVersion(opCtx, dssLock); - if (swDbMetadata.isOK() && swDbMetadata.getValue()->getVersion() <= cachedDbVersion) { - LOGV2_DEBUG(7079300, - 2, - "Skip setting cached database metadata as there are no updates", - "db"_attr = dbName, - "cachedDbVersion"_attr = *cachedDbVersion, - "refreshedDbVersion"_attr = swDbMetadata.getValue()->getVersion()); - - return Status::OK(); - } - } - - Lock::DBLock dbLock(opCtx, dbName, MODE_X); - auto* dss = DatabaseShardingState::get(opCtx, dbName); - auto dssLock = DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss); - - if (!cancellationToken.isCanceled()) { - if (swDbMetadata.isOK()) { - // Set the refreshed database metadata. - dss->setDatabaseInfo(opCtx, DatabaseType(*swDbMetadata.getValue()), dssLock); - } else if (swDbMetadata == ErrorCodes::NamespaceNotFound) { - // The database has been dropped, so clear its metadata. - dss->clearDatabaseInfo(opCtx); - } - } - - // Reset the future reference to allow any other thread to refresh the database metadata. - dss->resetDbMetadataRefreshFuture(dssLock); - resetRefreshFutureOnError.dismiss(); - - return swDbMetadata.getStatus(); -} - -SharedSemiFuture<void> recoverRefreshDbVersion(OperationContext* opCtx, - const StringData& dbName, - const CancellationToken& cancellationToken) { - const auto executor = Grid::get(opCtx)->getExecutorPool()->getFixedExecutor(); - return ExecutorFuture<void>(executor) - .then([=, - serviceCtx = opCtx->getServiceContext(), - forwardableOpMetadata = ForwardableOperationMetadata(opCtx), - dbNameStr = dbName.toString()] { - ThreadClient tc("DbMetadataRefreshThread", serviceCtx); - { - stdx::lock_guard<Client> lk(*tc.get()); - tc->setSystemOperationKillableByStepdown(lk); - } - - const auto opCtxHolder = - CancelableOperationContext(tc->makeOperationContext(), cancellationToken, executor); - auto opCtx = opCtxHolder.get(); - - // Forward `users` and `roles` attributes from the original request. - forwardableOpMetadata.setOn(opCtx); - - LOGV2_DEBUG(6697203, 2, "Started database metadata refresh", "db"_attr = dbNameStr); - - return refreshDbMetadata(opCtx, dbNameStr, cancellationToken); - }) - .onCompletion([=, dbNameStr = dbName.toString()](Status status) { - uassert(ErrorCodes::DatabaseMetadataRefreshCanceled, - str::stream() << "Canceled metadata refresh for database " << dbNameStr, - !cancellationToken.isCanceled()); - - if (status.isOK() || status == ErrorCodes::NamespaceNotFound) { - LOGV2(6697204, "Refreshed database metadata", "db"_attr = dbNameStr); - } else { - LOGV2_ERROR(6697205, - "Failed database metadata refresh", - "db"_attr = dbNameStr, - "error"_attr = redact(status)); - } - }) - .semi() - .share(); -} - void onDbVersionMismatch(OperationContext* opCtx, const StringData dbName, - const boost::optional<DatabaseVersion> receivedDbVersion) { + boost::optional<DatabaseVersion> clientDbVersion) { invariant(!opCtx->lockState()->isLocked()); invariant(!opCtx->getClient()->isInDirectClient()); invariant(ShardingState::get(opCtx)->canAcceptShardedCommands()); - tassert(ErrorCodes::IllegalOperation, - "Can't check version of {} database"_format(dbName), - dbName != NamespaceString::kAdminDb && dbName != NamespaceString::kConfigDb); - - LOGV2_DEBUG(6697200, - 2, - "Handle database version mismatch", - "db"_attr = dbName, - "receivedDbVersion"_attr = receivedDbVersion); - - while (true) { - boost::optional<SharedSemiFuture<void>> dbMetadataRefreshFuture; - - { - auto dbLock = boost::make_optional(Lock::DBLock(opCtx, dbName, MODE_IS)); - auto* dss = DatabaseShardingState::get(opCtx, dbName); - - if (receivedDbVersion) { - auto dssLock = - boost::make_optional(DatabaseShardingState::DSSLock::lockShared(opCtx, dss)); - - if (joinDbVersionOperation(opCtx, dss, &dbLock, &dssLock)) { - // Waited for another thread to exit from the critical section or to complete an - // ongoing refresh, so reacquire the locks. - continue; - } - - // From now until the end of this block [1] no thread is in the critical section or - // can enter it (would require to X-lock the database) and [2] no metadata refresh - // is in progress or can start (would require to exclusive lock the DSS). - // Therefore, the database version can be accessed safely. - - const auto wantedDbVersion = dss->getDbVersion(opCtx, *dssLock); - if (receivedDbVersion <= wantedDbVersion) { - // No need to refresh the database metadata as the wanted version is newer - // than the one received. - return; - } - } - - if (MONGO_unlikely(skipDatabaseVersionMetadataRefresh.shouldFail())) { + { + // Take the DBLock directly rather than using AutoGetDb, to prevent a recursive call into + // checkDbVersion(). + // + // TODO: It is not safe here to read the DB version without checking for critical section + // + if (clientDbVersion) { + Lock::DBLock dbLock(opCtx, dbName, MODE_IS); + auto dss = DatabaseShardingState::get(opCtx, dbName); + auto dssLock = DatabaseShardingState::DSSLock::lockShared(opCtx, dss); + const auto serverDbVersion = dss->getDbVersion(opCtx, dssLock); + if (clientDbVersion <= serverDbVersion) { + // The client was stale return; } - - auto dssLock = - boost::make_optional(DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss)); - - if (joinDbVersionOperation(opCtx, dss, &dbLock, &dssLock)) { - // Waited for another thread to exit from the critical section or to complete an - // ongoing refresh, so reacquire the locks. - continue; - } - - // From now until the end of this block [1] no thread is in the critical section or can - // enter it (would require to X-lock the database) and [2] this is the only metadata - // refresh in progress (holding the exclusive lock on the DSS). - // Therefore, the future to refresh the database metadata can be set. - - CancellationSource cancellationSource; - CancellationToken cancellationToken = cancellationSource.token(); - dss->setDbMetadataRefreshFuture( - recoverRefreshDbVersion(opCtx, dbName, cancellationToken), - std::move(cancellationSource), - *dssLock); - dbMetadataRefreshFuture = dss->getDbMetadataRefreshFuture(*dssLock); - } - - // No other metadata refresh for this database can run in parallel. If another thread enters - // the critical section, the ongoing refresh would be interrupted and subsequently - // re-queued. - - try { - dbMetadataRefreshFuture->get(opCtx); - } catch (const ExceptionFor<ErrorCodes::DatabaseMetadataRefreshCanceled>&) { - // The refresh was canceled by another thread that entered the critical section. - continue; } + } - break; + if (MONGO_unlikely(skipDatabaseVersionMetadataRefresh.shouldFail())) { + return; } + + forceDatabaseRefresh(opCtx, dbName); } -/** - * Blocking method, which will wait for any concurrent operations that could change the shard - * version to complete (namely critical section and concurrent onShardVersionMismatch invocations). - * - * Returns 'true' if there were concurrent operations that had to be joined (in which case all locks - * will be dropped). If there were none, returns false and the locks continue to be held. - */ +// Return true if joins a shard version update/recover/refresh (in that case, all locks are dropped) bool joinShardVersionOperation(OperationContext* opCtx, CollectionShardingRuntime* csr, boost::optional<Lock::DBLock>* dbLock, boost::optional<Lock::CollectionLock>* collLock, boost::optional<CollectionShardingRuntime::CSRLock>* csrLock) { - invariant(dbLock->has_value()); invariant(collLock->has_value()); invariant(csrLock->has_value()); - if (auto critSecSignal = - csr->getCriticalSectionSignal(opCtx, ShardingMigrationCriticalSection::kWrite)) { - csrLock->reset(); - collLock->reset(); - dbLock->reset(); - - uassertStatusOK( - OperationShardingState::waitForCriticalSectionToComplete(opCtx, *critSecSignal)); - - return true; - } + // If another thread is currently holding the critical section or the shard version future, it + // will be necessary to wait on one of the two variables to finish the update/recover/refresh. + auto inRecoverOrRefresh = csr->getShardVersionRecoverRefreshFuture(opCtx); + auto critSecSignal = + csr->getCriticalSectionSignal(opCtx, ShardingMigrationCriticalSection::kWrite); - if (auto inRecoverOrRefresh = csr->getShardVersionRecoverRefreshFuture(opCtx)) { + if (inRecoverOrRefresh || critSecSignal) { + // Drop the locks and wait for an ongoing shard version's recovery/refresh/update csrLock->reset(); collLock->reset(); dbLock->reset(); - try { - inRecoverOrRefresh->get(opCtx); - } catch (const ExceptionFor<ErrorCodes::ShardVersionRefreshCanceled>&) { - // The ongoing refresh has finished, although it was interrupted. + if (critSecSignal) { + uassertStatusOK( + OperationShardingState::waitForCriticalSectionToComplete(opCtx, *critSecSignal)); + } else { + try { + inRecoverOrRefresh->get(opCtx); + } catch (const ExceptionFor<ErrorCodes::ShardVersionRefreshCanceled>&) { + // The ongoing refresh has finished, although it was canceled by a + // 'clearFilteringMetadata'. + } } return true; @@ -353,7 +127,7 @@ bool joinShardVersionOperation(OperationContext* opCtx, } SharedSemiFuture<void> recoverRefreshShardVersion(ServiceContext* serviceContext, - const NamespaceString& nss, + const NamespaceString nss, bool runRecover, CancellationToken cancellationToken) { auto executor = Grid::get(serviceContext)->getExecutorPool()->getFixedExecutor(); @@ -376,57 +150,25 @@ SharedSemiFuture<void> recoverRefreshShardVersion(ServiceContext* serviceContext boost::optional<CollectionMetadata> currentMetadataToInstall; ON_BLOCK_EXIT([&] { - boost::optional<SharedSemiFuture<void>> waitForMigrationAbort; - { - UninterruptibleLockGuard noInterrupt(opCtx->lockState()); - // A view can potentially be created after spawning a thread to recover nss's - // shard version. It is then ok to lock views in order to clear filtering - // metadata. - // - // DBLock and CollectionLock must be used in order to avoid shard version - // checks. - Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); - Lock::CollectionLock collLock(opCtx, nss, MODE_IX); - - auto* const csr = CollectionShardingRuntime::get(opCtx, nss); - - auto csrLock = CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr); - // cancellationToken needs to be checked under the CSR lock before overwriting - // the filtering metadata to serialize with other threads calling - // 'clearFilteringMetadata'. - if (currentMetadataToInstall && !cancellationToken.isCanceled()) { - csr->setFilteringMetadata_withLock( - opCtx, *currentMetadataToInstall, csrLock); - - if (currentMetadataToInstall->isSharded() && - !currentMetadataToInstall->allowMigrations()) { - if (auto msm = MigrationSourceManager::get(csr, csrLock)) { - waitForMigrationAbort.emplace(msm->abort()); - } - } - } + UninterruptibleLockGuard noInterrupt(opCtx->lockState()); + // A view can potentially be created after spawning a thread to recover nss's shard + // version. It is then ok to lock views in order to clear filtering metadata. + // + // DBLock and CollectionLock must be used in order to avoid shard version checks + Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); + Lock::CollectionLock collLock(opCtx, nss, MODE_IX); + + auto* const csr = CollectionShardingRuntime::get(opCtx, nss); + + auto csrLock = CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr); + // cancellationToken needs to be checked under the CSR lock before overwriting the + // filtering metadata to serialize with other threads calling + // 'clearFilteringMetadata' + if (currentMetadataToInstall && !cancellationToken.isCanceled()) { + csr->setFilteringMetadata_withLock(opCtx, *currentMetadataToInstall, csrLock); } - // Join any ongoing migration outside of the CSR lock. Considering we're technically - // inside a destructor, we can't allow this wait to throw and neither can we return - // without having waited. It is acceptable to wait here uninterruptibly because - // we are not holding any resources and nothing that holds resources should be - // waiting on the refresh thread. - if (waitForMigrationAbort) { - waitForMigrationAbort->waitNoThrow().ignore(); - } - - { - UninterruptibleLockGuard noInterrupt(opCtx->lockState()); - // Remember to wake all waiting threads for this refresh to finish. - Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); - Lock::CollectionLock collLock(opCtx, nss, MODE_IX); - - auto* const csr = CollectionShardingRuntime::get(opCtx, nss); - auto csrLock = CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr); - - csr->resetShardVersionRecoverRefreshFuture(csrLock); - } + csr->resetShardVersionRecoverRefreshFuture(csrLock); }); if (runRecover) { @@ -439,22 +181,20 @@ SharedSemiFuture<void> recoverRefreshShardVersion(ServiceContext* serviceContext auto currentMetadata = forceGetCurrentMetadata(opCtx, nss); if (currentMetadata.isSharded()) { - // Abort and join any ongoing migration if migrations are disallowed for the - // namespace. + // If migrations are disallowed for the namespace, join any migrations which may be + // executing currently if (!currentMetadata.allowMigrations()) { boost::optional<SharedSemiFuture<void>> waitForMigrationAbort; { + // DBLock and CollectionLock must be used in order to avoid shard version + // checks Lock::DBLock dbLock(opCtx, nss.db(), MODE_IX); Lock::CollectionLock collLock(opCtx, nss, MODE_IX); auto const& csr = CollectionShardingRuntime::get(opCtx, nss); auto csrLock = CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); - // There is no need to abort an ongoing migration if the refresh is - // cancelled. - if (!cancellationToken.isCanceled()) { - if (auto msm = MigrationSourceManager::get(csr, csrLock)) { - waitForMigrationAbort.emplace(msm->abort()); - } + if (auto msm = MigrationSourceManager::get(csr, csrLock)) { + waitForMigrationAbort.emplace(msm->abort()); } } @@ -483,8 +223,7 @@ SharedSemiFuture<void> recoverRefreshShardVersion(ServiceContext* serviceContext if (cancellationToken.isCanceled() && (status.isOK() || status == ErrorCodes::Interrupted)) { uasserted(ErrorCodes::ShardVersionRefreshCanceled, - "Shard version refresh canceled by an interruption, probably due to a " - "'clearFilteringMetadata'"); + "Shard version refresh canceled by a 'clearFilteringMetadata'"); } return status; }) @@ -515,7 +254,6 @@ void onShardVersionMismatch(OperationContext* opCtx, while (true) { boost::optional<SharedSemiFuture<void>> inRecoverOrRefresh; - { boost::optional<Lock::DBLock> dbLock; boost::optional<Lock::CollectionLock> collLock; @@ -523,16 +261,17 @@ void onShardVersionMismatch(OperationContext* opCtx, collLock.emplace(opCtx, nss, MODE_IS); auto* const csr = CollectionShardingRuntime::get(opCtx, nss); + boost::optional<CollectionShardingRuntime::CSRLock> csrLock = + CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); - if (shardVersionReceived) { - boost::optional<CollectionShardingRuntime::CSRLock> csrLock = - CollectionShardingRuntime::CSRLock::lockShared(opCtx, csr); - - if (joinShardVersionOperation(opCtx, csr, &dbLock, &collLock, &csrLock)) { - continue; - } + if (joinShardVersionOperation(opCtx, csr, &dbLock, &collLock, &csrLock)) { + continue; + } - if (auto metadata = csr->getCurrentMetadataIfKnown()) { + auto metadata = csr->getCurrentMetadataIfKnown(); + if (metadata) { + // Check if the current shard version is fresh enough + if (shardVersionReceived) { const auto currentShardVersion = metadata->getShardVersion(); // Don't need to remotely reload if the requested version is smaller than the // known one. This means that the remote side is behind. @@ -542,26 +281,25 @@ void onShardVersionMismatch(OperationContext* opCtx, } } - boost::optional<CollectionShardingRuntime::CSRLock> csrLock = - CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr); - - if (joinShardVersionOperation(opCtx, csr, &dbLock, &collLock, &csrLock)) { + csrLock.reset(); + csrLock.emplace(CollectionShardingRuntime::CSRLock::lockExclusive(opCtx, csr)); + + // If there is no ongoing shard version operation, initialize the RecoverRefreshThread + // thread and associate it to the CSR. + if (!joinShardVersionOperation(opCtx, csr, &dbLock, &collLock, &csrLock)) { + // If the shard doesn't yet know its filtering metadata, recovery needs to be run + const bool runRecover = metadata ? false : true; + CancellationSource cancellationSource; + CancellationToken cancellationToken = cancellationSource.token(); + csr->setShardVersionRecoverRefreshFuture( + recoverRefreshShardVersion( + opCtx->getServiceContext(), nss, runRecover, std::move(cancellationToken)), + std::move(cancellationSource), + *csrLock); + inRecoverOrRefresh = csr->getShardVersionRecoverRefreshFuture(opCtx); + } else { continue; } - - // If we reached here, there were no ongoing critical sections or recoverRefresh running - // and we are holding the exclusive CSR lock. - - // If the shard doesn't yet know its filtering metadata, recovery needs to be run - const bool runRecover = csr->getCurrentMetadataIfKnown() ? false : true; - CancellationSource cancellationSource; - CancellationToken cancellationToken = cancellationSource.token(); - csr->setShardVersionRecoverRefreshFuture( - recoverRefreshShardVersion( - opCtx->getServiceContext(), nss, runRecover, std::move(cancellationToken)), - std::move(cancellationSource), - *csrLock); - inRecoverOrRefresh = csr->getShardVersionRecoverRefreshFuture(opCtx); } try { @@ -733,4 +471,56 @@ Status onDbVersionMismatchNoExcept(OperationContext* opCtx, } } +void forceDatabaseRefresh(OperationContext* opCtx, const StringData dbName) { + invariant(!opCtx->lockState()->isLocked()); + invariant(!opCtx->getClient()->isInDirectClient()); + + auto const shardingState = ShardingState::get(opCtx); + invariant(shardingState->canAcceptShardedCommands()); + + const auto swRefreshedDbInfo = + Grid::get(opCtx)->catalogCache()->getDatabaseWithRefresh(opCtx, dbName); + + if (swRefreshedDbInfo == ErrorCodes::NamespaceNotFound) { + // db has been dropped, set the db version to boost::none + Lock::DBLock dbLock(opCtx, dbName, MODE_X); + auto dss = DatabaseShardingState::get(opCtx, dbName); + dss->clearDatabaseInfo(opCtx); + return; + } + + const auto refreshedDbInfo = uassertStatusOK(std::move(swRefreshedDbInfo)); + const auto& refreshedDBVersion = refreshedDbInfo->getVersion(); + + // First, check under a shared lock if another thread already updated the cached version. + // This is a best-effort optimization to make as few threads as possible to convoy on the + // exclusive lock below. + { + // Take the DBLock directly rather than using AutoGetDb, to prevent a recursive call + // into checkDbVersion(). + Lock::DBLock dbLock(opCtx, dbName, MODE_IS); + auto dss = DatabaseShardingState::get(opCtx, dbName); + auto dssLock = DatabaseShardingState::DSSLock::lockShared(opCtx, dss); + + const auto cachedDbVersion = dss->getDbVersion(opCtx, dssLock); + if (cachedDbVersion && *cachedDbVersion >= refreshedDBVersion) { + LOGV2_DEBUG(5369130, + 2, + "Skipping updating cached database info from refreshed version " + "because the one currently cached is more recent", + "db"_attr = dbName, + "refreshedDbVersion"_attr = refreshedDBVersion, + "cachedDbVersion"_attr = *cachedDbVersion); + return; + } + } + + // The cached version is older than the refreshed version; update the cached version. + Lock::DBLock dbLock(opCtx, dbName, MODE_X); + auto dss = DatabaseShardingState::get(opCtx, dbName); + auto dssLock = DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss); + + dss->setDatabaseInfo(opCtx, DatabaseType(*refreshedDbInfo), dssLock); +} + } // namespace mongo diff --git a/src/mongo/db/s/shard_filtering_metadata_refresh.h b/src/mongo/db/s/shard_filtering_metadata_refresh.h index fd3a8b8ca6d..724409b4621 100644 --- a/src/mongo/db/s/shard_filtering_metadata_refresh.h +++ b/src/mongo/db/s/shard_filtering_metadata_refresh.h @@ -39,12 +39,11 @@ namespace mongo { class OperationContext; /** - * Must be invoked whenever code, which is executing on a shard encounters a StaleConfig error and - * should be passed the placement version from the 'version received' in the exception. If the - * shard's current placement version is behind 'chunkVersionReceived', causes the shard's filtering - * metadata to be refreshed from the config server, otherwise does nothing and immediately returns. - * If there are other threads currently performing refresh, blocks so that only one of them hits the - * config server. + * Must be invoked whenever code, which is executing on a shard encounters a StaleConfig exception + * and should be passed the 'version received' from the exception. If the shard's current version is + * behind 'shardVersionReceived', causes the shard's filtering metadata to be refreshed from the + * config server, otherwise does nothing and immediately returns. If there are other threads + * currently performing refresh, blocks so that only one of them hits the config server. * * If refresh fails for any reason (most commonly ExceededTimeLimit), returns a failed status. * @@ -92,4 +91,6 @@ Status onDbVersionMismatchNoExcept(OperationContext* opCtx, StringData dbName, boost::optional<DatabaseVersion> clientDbVersion) noexcept; +void forceDatabaseRefresh(OperationContext* opCtx, StringData dbName); + } // namespace mongo diff --git a/src/mongo/db/s/shard_key_index_util.cpp b/src/mongo/db/s/shard_key_index_util.cpp index 826c73c1430..56d6ca08656 100644 --- a/src/mongo/db/s/shard_key_index_util.cpp +++ b/src/mongo/db/s/shard_key_index_util.cpp @@ -36,7 +36,6 @@ #include "mongo/db/catalog/collection.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/s/shard_key_index_util.h" -#include "mongo/db/server_feature_flags_gen.h" namespace mongo { @@ -47,8 +46,7 @@ const boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex( const IndexCatalog* indexCatalog, const boost::optional<std::string>& excludeName, const BSONObj& shardKey, - bool requireSingleKey, - std::string* errMsg = nullptr) { + bool requireSingleKey) { if (collection->isClustered() && clustered_util::matchesClusterKey(shardKey, collection->getClusteredInfo())) { auto clusteredIndexSpec = collection->getClusteredInfo()->getIndexSpec(); @@ -57,8 +55,7 @@ const boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex( const IndexDescriptor* best = nullptr; - auto indexIterator = - indexCatalog->getIndexIterator(opCtx, IndexCatalog::InclusionPolicy::kReady); + auto indexIterator = indexCatalog->getIndexIterator(opCtx, false); while (indexIterator->more()) { auto indexEntry = indexIterator->next(); auto indexDescriptor = indexEntry->descriptor(); @@ -67,12 +64,7 @@ const boost::optional<ShardKeyIndex> _findShardKeyPrefixedIndex( continue; } - if (indexDescriptor->hidden()) { - continue; - } - - if (isCompatibleWithShardKey( - opCtx, collection, indexEntry, shardKey, requireSingleKey, errMsg)) { + if (isCompatibleWithShardKey(opCtx, collection, indexEntry, shardKey, requireSingleKey)) { if (!indexEntry->isMultikey(opCtx, collection)) { return ShardKeyIndex(indexDescriptor); } @@ -113,94 +105,34 @@ bool isCompatibleWithShardKey(OperationContext* opCtx, const CollectionPtr& collection, const IndexCatalogEntry* indexEntry, const BSONObj& shardKey, - bool requireSingleKey, - std::string* errMsg) { - // Return a descriptive error for each index that shares a prefix with shardKey but - // cannot be used for sharding. - const int kErrorPartial = 0x01; - const int kErrorSparse = 0x02; - const int kErrorMultikey = 0x04; - const int kErrorCollation = 0x08; - const int kErrorNotPrefix = 0x10; - int reasons = 0; - + bool requireSingleKey) { auto desc = indexEntry->descriptor(); bool hasSimpleCollation = desc->collation().isEmpty(); - if (desc->isPartial()) { - reasons |= kErrorPartial; - } - - if (desc->isSparse()) { - reasons |= kErrorSparse; + if (desc->isPartial() || desc->isSparse()) { + return false; } if (!shardKey.isPrefixOf(desc->keyPattern(), SimpleBSONElementComparator::kInstance)) { - reasons |= kErrorNotPrefix; + return false; } - if (reasons == 0) { // that is, not partial index, not sparse, and not prefix, then: - if (!indexEntry->isMultikey(opCtx, collection)) { - if (hasSimpleCollation) { - return true; - } - } else { - reasons |= kErrorMultikey; - } - if (!requireSingleKey && hasSimpleCollation) { - return true; - } + if (!indexEntry->isMultikey(opCtx, collection) && hasSimpleCollation) { + return true; } - if (!hasSimpleCollation) { - reasons |= kErrorCollation; + if (!requireSingleKey && hasSimpleCollation) { + return true; } - if (errMsg && reasons != 0) { - std::string errors = "Index " + indexEntry->descriptor()->indexName() + - " cannot be used for sharding because:"; - if (reasons & kErrorPartial) { - errors += " Index key is partial."; - } - if (reasons & kErrorSparse) { - errors += " Index key is sparse."; - } - if (reasons & kErrorMultikey) { - errors += " Index key is multikey."; - } - if (reasons & kErrorCollation) { - errors += " Index has a non-simple collation."; - } - if (reasons & kErrorNotPrefix) { - errors += " Shard key is not a prefix of index key."; - } - if (!errMsg->empty()) { - *errMsg += "\n"; - } - *errMsg += errors; - } return false; } -bool isLastNonHiddenRangedShardKeyIndex(OperationContext* opCtx, - const CollectionPtr& collection, - const IndexCatalog* indexCatalog, - const std::string& indexName, - const BSONObj& shardKey) { - const auto index = indexCatalog->findIndexByName(opCtx, indexName); - if (!index || - !isCompatibleWithShardKey( - opCtx, collection, index->getEntry(), shardKey, false /* requireSingleKey */)) { - return false; - } - - // Users are allowed to drop hashed shard key indexes. - if (gFeatureFlagShardKeyIndexOptionalHashedSharding.isEnabled( - serverGlobalParams.featureCompatibility) && - ShardKeyPattern(shardKey).isHashedPattern()) { - return false; - } - +bool isLastShardKeyIndex(OperationContext* opCtx, + const CollectionPtr& collection, + const IndexCatalog* indexCatalog, + const std::string& indexName, + const BSONObj& shardKey) { return !_findShardKeyPrefixedIndex( opCtx, collection, indexCatalog, indexName, shardKey, false /* requireSingleKey */) .is_initialized(); @@ -210,10 +142,9 @@ const boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* const CollectionPtr& collection, const IndexCatalog* indexCatalog, const BSONObj& shardKey, - bool requireSingleKey, - std::string* errMsg) { + bool requireSingleKey) { return _findShardKeyPrefixedIndex( - opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey, errMsg); + opCtx, collection, indexCatalog, boost::none, shardKey, requireSingleKey); } } // namespace mongo diff --git a/src/mongo/db/s/shard_key_index_util.h b/src/mongo/db/s/shard_key_index_util.h index 0d67ac8d7f0..db59c57ceca 100644 --- a/src/mongo/db/s/shard_key_index_util.h +++ b/src/mongo/db/s/shard_key_index_util.h @@ -32,7 +32,6 @@ #include "mongo/bson/bsonobj.h" #include "mongo/db/catalog/clustered_collection_options_gen.h" #include "mongo/db/catalog/index_catalog.h" -#include "mongo/s/shard_key_pattern.h" namespace mongo { @@ -68,16 +67,12 @@ private: /** * Returns true if the given index is compatible with the shard key pattern. - * - * If return value is false and errMsg is non-null, the reasons that the existing index is - * incompatible will be appended to errMsg. */ bool isCompatibleWithShardKey(OperationContext* opCtx, const CollectionPtr& collection, const IndexCatalogEntry* indexEntry, const BSONObj& shardKey, - bool requireSingleKey, - std::string* errMsg = nullptr); + bool requireSingleKey); /** * Returns an index suitable for shard key range scans if it exists. @@ -86,7 +81,6 @@ bool isCompatibleWithShardKey(OperationContext* opCtx, * - must be prefixed by 'shardKey', and * - must not be a partial index. * - must have the simple collation. - * - must not be hidden. * * If the parameter 'requireSingleKey' is true, then this index additionally must not be * multi-key. @@ -95,18 +89,16 @@ const boost::optional<ShardKeyIndex> findShardKeyPrefixedIndex(OperationContext* const CollectionPtr& collection, const IndexCatalog* indexCatalog, const BSONObj& shardKey, - bool requireSingleKey, - std::string* errMsg = nullptr); + bool requireSingleKey); /** - * Returns true if the given index exists and it is the last non-hidden index compatible with the - * ranged shard key. False otherwise. Hashed indexes are excluded here because users are allowed - * to drop shard key compatible hashed indexes. + * Returns true if the given index name is the last remaining index that is compatible with the + * shard key index. */ -bool isLastNonHiddenRangedShardKeyIndex(OperationContext* opCtx, - const CollectionPtr& collection, - const IndexCatalog* indexCatalog, - const std::string& indexName, - const BSONObj& shardKey); +bool isLastShardKeyIndex(OperationContext* opCtx, + const CollectionPtr& collection, + const IndexCatalog* indexCatalog, + const std::string& indexName, + const BSONObj& shardKey); } // namespace mongo diff --git a/src/mongo/db/s/shard_key_index_util_test.cpp b/src/mongo/db/s/shard_key_index_util_test.cpp index cba87669945..c9c1b4fdc23 100644 --- a/src/mongo/db/s/shard_key_index_util_test.cpp +++ b/src/mongo/db/s/shard_key_index_util_test.cpp @@ -203,12 +203,9 @@ TEST_F(ShardKeyIndexUtilTest, LastShardIndexWithSingleCandidate) { createIndex(BSON("key" << BSON("x" << 1) << "name" << "x" << "v" << kIndexVersion)); - createIndex(BSON("key" << BSON("x" << 1 << "y" << 1) << "name" - << "xy" - << "v" << kIndexVersion << "hidden" << true)); - ASSERT_TRUE(isLastNonHiddenRangedShardKeyIndex( - opCtx(), coll(), coll()->getIndexCatalog(), "x", BSON("x" << 1))); + ASSERT_TRUE( + isLastShardKeyIndex(opCtx(), coll(), coll()->getIndexCatalog(), "x", BSON("x" << 1))); } TEST_F(ShardKeyIndexUtilTest, LastShardIndexWithMultipleCandidates) { @@ -222,60 +219,8 @@ TEST_F(ShardKeyIndexUtilTest, LastShardIndexWithMultipleCandidates) { << "xy" << "v" << kIndexVersion)); - ASSERT_FALSE(isLastNonHiddenRangedShardKeyIndex( - opCtx(), coll(), coll()->getIndexCatalog(), "x", BSON("x" << 1))); -} - -TEST_F(ShardKeyIndexUtilTest, LastShardIndexWithIncompatibleIndex) { - createIndex(BSON("key" << BSON("y" << 1) << "name" - << "y" - << "v" << kIndexVersion)); - createIndex(BSON("key" << BSON("x" << 1) << "name" - << "x" - << "v" << kIndexVersion)); - - ASSERT_FALSE(isLastNonHiddenRangedShardKeyIndex( - opCtx(), coll(), coll()->getIndexCatalog(), "y", BSON("x" << 1))); -} - -TEST_F(ShardKeyIndexUtilTest, LastShardIndexWithNonExistingIndex) { - createIndex(BSON("key" << BSON("x" << 1) << "name" - << "x" - << "v" << kIndexVersion)); - - ASSERT_FALSE(isLastNonHiddenRangedShardKeyIndex( - opCtx(), coll(), coll()->getIndexCatalog(), "y", BSON("x" << 1))); -} - -TEST_F(ShardKeyIndexUtilTest, LastShardIndexExcludesHashedIndex) { - createIndex(BSON("key" << BSON("x" - << "hashed") - << "name" - << "xhashed" - << "v" << kIndexVersion)); - - ASSERT_FALSE(isLastNonHiddenRangedShardKeyIndex(opCtx(), - coll(), - coll()->getIndexCatalog(), - "y", - BSON("x" - << "hashed"))); -} - -TEST_F(ShardKeyIndexUtilTest, LastShardIndexExcludesCompoundHashedIndex) { - createIndex(BSON("key" << BSON("x" - << "hashed" - << "y" << 1) - << "name" - << "xhashed" - << "v" << kIndexVersion)); - - ASSERT_FALSE(isLastNonHiddenRangedShardKeyIndex(opCtx(), - coll(), - coll()->getIndexCatalog(), - "y", - BSON("x" - << "hashed"))); + ASSERT_FALSE( + isLastShardKeyIndex(opCtx(), coll(), coll()->getIndexCatalog(), "x", BSON("x" << 1))); } } // namespace diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp index ebb4497260a..5a0acaeb2a4 100644 --- a/src/mongo/db/s/shard_key_util.cpp +++ b/src/mongo/db/s/shard_key_util.cpp @@ -107,29 +107,24 @@ bool validShardKeyIndexExists(OperationContext* opCtx, const ShardKeyPattern& shardKeyPattern, const boost::optional<BSONObj>& defaultCollation, bool requiresUnique, - const ShardKeyValidationBehaviors& behaviors, - std::string* errMsg) { + const ShardKeyValidationBehaviors& behaviors) { auto indexes = behaviors.loadIndexes(nss); // 1. Verify consistency with existing unique indexes for (const auto& idx : indexes) { BSONObj currentKey = idx["key"].embeddedObject(); bool isUnique = idx["unique"].trueValue(); - bool isPrepareUnique = idx["prepareUnique"].trueValue(); uassert(ErrorCodes::InvalidOptions, str::stream() << "can't shard collection '" << nss.ns() << "' with unique index on " << currentKey << " and proposed shard key " << shardKeyPattern.toBSON() << ". Uniqueness can't be maintained unless shard key is a prefix", - (!isUnique && !isPrepareUnique) || - shardKeyPattern.isIndexUniquenessCompatible(currentKey)); + !isUnique || shardKeyPattern.isUniqueIndexCompatible(currentKey)); } // 2. Check for a useful index bool hasUsefulIndexForKey = false; - std::string allReasons; for (const auto& idx : indexes) { - std::string reasons; BSONObj currentKey = idx["key"].embeddedObject(); // Check 2.i. and 2.ii. if (!idx["sparse"].trueValue() && idx["filter"].eoo() && idx["collation"].eoo() && @@ -148,19 +143,6 @@ bool validShardKeyIndexExists(OperationContext* opCtx, idx["seed"].numberInt() == BSONElementHasher::DEFAULT_HASH_SEED); hasUsefulIndexForKey = true; } - if (idx["sparse"].trueValue()) { - reasons += " Index key is sparse."; - } - if (idx["filter"].ok()) { - reasons += " Index key is partial."; - } - if (idx["collation"].ok()) { - reasons += " Index has a non-simple collation."; - } - if (!reasons.empty()) { - allReasons = - " Index " + idx["name"] + " cannot be used for sharding because [" + reasons + " ]"; - } } // 3. If proposed key is required to be unique, additionally check for exact match. @@ -191,10 +173,6 @@ bool validShardKeyIndexExists(OperationContext* opCtx, } } - if (errMsg && !allReasons.empty()) { - *errMsg += allReasons; - } - if (hasUsefulIndexForKey) { // Check 2.iii Make sure that there is a useful, non-multikey index available. behaviors.verifyUsefulNonMultiKeyIndex(nss, shardKeyPattern.toBSON()); @@ -210,19 +188,17 @@ bool validateShardKeyIndexExistsOrCreateIfPossible(OperationContext* opCtx, bool unique, bool enforceUniquenessCheck, const ShardKeyValidationBehaviors& behaviors) { - std::string errMsg; if (validShardKeyIndexExists(opCtx, nss, shardKeyPattern, defaultCollation, unique && enforceUniquenessCheck, - behaviors, - &errMsg)) { + behaviors)) { return false; } // 4. If no useful index, verify we can create one. - behaviors.verifyCanCreateShardKeyIndex(nss, &errMsg); + behaviors.verifyCanCreateShardKeyIndex(nss); // 5. If no useful index exists and we can create one, create one on proposedKey. Only need // to call ensureIndex on primary shard, since indexes get copied to receiving shard @@ -292,23 +268,14 @@ void ValidationBehaviorsShardCollection::verifyUsefulNonMultiKeyIndex( "admin", BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey), res); - - // checkShardingIndex may return UnknownError if a compatible shard key index cannot be - // found when the command is executed on a node with an old binary. In this case, we should - // return InvalidOptions to correspond with the shardCollection behavior. - const auto status = getStatusFromCommandResult(res); - if (status == ErrorCodes::UnknownError) { - uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success); - } - uassertStatusOK(status); + uassert(ErrorCodes::InvalidOptions, res["errmsg"].str(), success); } -void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex(const NamespaceString& nss, - std::string* errMsg) const { +void ValidationBehaviorsShardCollection::verifyCanCreateShardKeyIndex( + const NamespaceString& nss) const { uassert(ErrorCodes::InvalidOptions, - str::stream() << "Please create an index that starts with the proposed shard key before" - " sharding the collection. " - << *errMsg, + "Please create an index that starts with the proposed shard key before " + "sharding the collection", _localClient->findOne(nss, BSONObj{}).isEmpty()); } @@ -350,7 +317,7 @@ std::vector<BSONObj> ValidationBehaviorsRefineShardKey::loadIndexes( void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex( const NamespaceString& nss, const BSONObj& proposedKey) const { - auto res = uassertStatusOK(_indexShard->runCommand( + auto checkShardingIndexRes = uassertStatusOK(_indexShard->runCommand( _opCtx, ReadPreferenceSetting(ReadPreference::PrimaryOnly), "admin", @@ -358,23 +325,20 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex( BSON(kCheckShardingIndexCmdName << nss.ns() << kKeyPatternField << proposedKey), _cm.getVersion(_indexShard->getId())), Shard::RetryPolicy::kIdempotent)); - - // checkShardingIndex may return UnknownError if a compatible shard key index cannot be - // found when the command is executed on a node with an old binary. In this case, we should - // return InvalidOptions to correspond with the shardCollection behavior. - if (res.commandStatus == ErrorCodes::UnknownError) { - uasserted(ErrorCodes::InvalidOptions, res.response["errmsg"].str()); + if (checkShardingIndexRes.commandStatus == ErrorCodes::UnknownError) { + // CheckShardingIndex returns UnknownError if a compatible shard key index cannot be found, + // but we return InvalidOptions to correspond with the shardCollection behavior. + uasserted(ErrorCodes::InvalidOptions, checkShardingIndexRes.response["errmsg"].str()); } - uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(res)); + // Rethrow any other error to allow retries on retryable errors. + uassertStatusOK(checkShardingIndexRes.commandStatus); } -void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex(const NamespaceString& nss, - std::string* errMsg) const { - uasserted( - ErrorCodes::InvalidOptions, - str::stream() << "Please create an index that starts with the proposed shard key before" - " sharding the collection. " - << *errMsg); +void ValidationBehaviorsRefineShardKey::verifyCanCreateShardKeyIndex( + const NamespaceString& nss) const { + uasserted(ErrorCodes::InvalidOptions, + "Please create an index that starts with the proposed shard key before " + "refining the shard key of the collection"); } void ValidationBehaviorsRefineShardKey::createShardKeyIndex( diff --git a/src/mongo/db/s/shard_key_util.h b/src/mongo/db/s/shard_key_util.h index 55905e7beb7..5d20a013bef 100644 --- a/src/mongo/db/s/shard_key_util.h +++ b/src/mongo/db/s/shard_key_util.h @@ -51,8 +51,7 @@ public: virtual void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey) const = 0; - virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss, - std::string* errMsg) const = 0; + virtual void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const = 0; virtual void createShardKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey, @@ -73,8 +72,7 @@ public: void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey) const override; - void verifyCanCreateShardKeyIndex(const NamespaceString& nss, - std::string* errMsg) const override; + void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override; void createShardKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey, @@ -97,8 +95,7 @@ public: void verifyUsefulNonMultiKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey) const override; - void verifyCanCreateShardKeyIndex(const NamespaceString& nss, - std::string* errMsg) const override; + void verifyCanCreateShardKeyIndex(const NamespaceString& nss) const override; void createShardKeyIndex(const NamespaceString& nss, const BSONObj& proposedKey, @@ -168,8 +165,7 @@ bool validShardKeyIndexExists(OperationContext* opCtx, const ShardKeyPattern& shardKeyPattern, const boost::optional<BSONObj>& defaultCollation, bool requiresUnique, - const ShardKeyValidationBehaviors& behaviors, - std::string* errMsg = nullptr); + const ShardKeyValidationBehaviors& behaviors); void validateShardKeyIsNotEncrypted(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/s/shard_local.cpp b/src/mongo/db/s/shard_local.cpp index 229a61e8480..69ca9ba6332 100644 --- a/src/mongo/db/s/shard_local.cpp +++ b/src/mongo/db/s/shard_local.cpp @@ -36,7 +36,7 @@ #include "mongo/client/remote_command_targeter.h" #include "mongo/db/catalog/index_catalog.h" #include "mongo/db/catalog_raii.h" -#include "mongo/db/concurrency/exception_util.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/index_builds_coordinator.h" #include "mongo/db/repl/repl_client_info.h" @@ -227,24 +227,4 @@ Status ShardLocal::runAggregation( return _rsLocalClient.runAggregation(opCtx, aggRequest, callback); } -BatchedCommandResponse ShardLocal::runBatchWriteCommand(OperationContext* opCtx, - const Milliseconds maxTimeMS, - const BatchedCommandRequest& batchRequest, - const WriteConcernOptions& writeConcern, - RetryPolicy retryPolicy) { - // A request dispatched through a local client is served within the same thread that submits it - // (so that the opCtx needs to be used as the vehicle to pass the WC to the ServiceEntryPoint). - const auto originalWC = opCtx->getWriteConcern(); - ScopeGuard resetWCGuard([&] { opCtx->setWriteConcern(originalWC); }); - opCtx->setWriteConcern(writeConcern); - - const auto dbName = batchRequest.getNS().db(); - const BSONObj cmdObj = [&] { - BSONObjBuilder cmdObjBuilder; - batchRequest.serialize(&cmdObjBuilder); - return cmdObjBuilder.obj(); - }(); - - return _submitBatchWriteCommand(opCtx, cmdObj, dbName, maxTimeMS, retryPolicy); -} } // namespace mongo diff --git a/src/mongo/db/s/shard_local.h b/src/mongo/db/s/shard_local.h index f82e3f7bb96..e7c6e74b000 100644 --- a/src/mongo/db/s/shard_local.h +++ b/src/mongo/db/s/shard_local.h @@ -81,12 +81,6 @@ public: std::function<bool(const std::vector<BSONObj>& batch, const boost::optional<BSONObj>& postBatchResumeToken)> callback); - BatchedCommandResponse runBatchWriteCommand(OperationContext* opCtx, - Milliseconds maxTimeMS, - const BatchedCommandRequest& batchRequest, - const WriteConcernOptions& writeConcern, - RetryPolicy retryPolicy) final; - private: StatusWith<Shard::CommandResponse> _runCommand(OperationContext* opCtx, const ReadPreferenceSetting& unused, diff --git a/src/mongo/db/s/shard_metadata_util.cpp b/src/mongo/db/s/shard_metadata_util.cpp index 934abab7812..6472807afd0 100644 --- a/src/mongo/db/s/shard_metadata_util.cpp +++ b/src/mongo/db/s/shard_metadata_util.cpp @@ -33,13 +33,10 @@ #include "mongo/db/s/shard_metadata_util.h" -#include <algorithm> #include <memory> #include "mongo/db/dbdirectclient.h" #include "mongo/db/ops/write_ops.h" -#include "mongo/db/ops/write_ops_exec.h" -#include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/type_shard_collection.h" #include "mongo/db/s/type_shard_database.h" #include "mongo/db/write_concern_options.h" @@ -360,52 +357,35 @@ Status updateShardChunks(OperationContext* opCtx, * {_id: 19, max: 22, version 2.0} * */ - - auto chunkIt = chunks.cbegin(); - while (chunkIt != chunks.cend()) { - const auto currBatchSize = - std::min(static_cast<ptrdiff_t>(persistedChunkCacheUpdateMaxBatchSize.load()), - std::distance(chunkIt, chunks.end())); - - { - // Delete any overlapping chunk ranges. Overlapping chunks will have a min value - // ("_id") between (chunk.min, chunk.max]. - // - // query: { "_id" : {"$gte": chunk.min, "$lt": chunk.max}} - std::vector<write_ops::DeleteOpEntry> deletes; - deletes.reserve(currBatchSize); - for (auto it = chunkIt; it < chunkIt + currBatchSize; it++) { - const auto& chunk = *it; - invariant(chunk.getVersion().epoch() == currEpoch); - auto query = BSON(ChunkType::minShardID - << BSON("$gte" << chunk.getMin() << "$lt" << chunk.getMax())); - deletes.emplace_back(std::move(query), true /* multi */); - } - write_ops::DeleteCommandRequest deleteReq{chunksNss, std::move(deletes)}; - deleteReq.getWriteCommandRequestBase().setOrdered(true); - deleteReq.getWriteCommandRequestBase().setBypassDocumentValidation(true); - auto deleteResp = write_ops_exec::performDeletes(opCtx, deleteReq); - // Since the writes are ordered, it's ok to check just the last writeOp result. - uassertStatusOK(deleteResp.results.back()); - } - - // Now the documents can be expected to cleanly insert without overlap - { - std::vector<BSONObj> docs; - docs.reserve(currBatchSize); - for (auto it = chunkIt; it < chunkIt + currBatchSize; it++) { - const auto& chunk = *it; - docs.emplace_back(chunk.toShardBSON()); - } - write_ops::InsertCommandRequest insertReq{chunksNss, std::move(docs)}; - insertReq.getWriteCommandRequestBase().setOrdered(true); - insertReq.getWriteCommandRequestBase().setBypassDocumentValidation(true); - auto insertResp = write_ops_exec::performInserts(opCtx, insertReq); - // Since the writes are ordered, it's ok to check just the last writeOp result. - uassertStatusOK(insertResp.results.back()); - } - - std::advance(chunkIt, currBatchSize); + for (auto& chunk : chunks) { + invariant(chunk.getVersion().epoch() == currEpoch); + + // Delete any overlapping chunk ranges. Overlapping chunks will have a min value + // ("_id") between (chunk.min, chunk.max]. + // + // query: { "_id" : {"$gte": chunk.min, "$lt": chunk.max}} + auto deleteCommandResponse = client.runCommand([&] { + write_ops::DeleteCommandRequest deleteOp(chunksNss); + deleteOp.setDeletes({[&] { + write_ops::DeleteOpEntry entry; + entry.setQ(BSON(ChunkType::minShardID + << BSON("$gte" << chunk.getMin() << "$lt" << chunk.getMax()))); + entry.setMulti(true); + return entry; + }()}); + return deleteOp.serialize({}); + }()); + uassertStatusOK( + getStatusFromWriteCommandResponse(deleteCommandResponse->getCommandReply())); + + // Now the document can be expected to cleanly insert without overlap + auto insertCommandResponse = client.runCommand([&] { + write_ops::InsertCommandRequest insertOp(chunksNss); + insertOp.setDocuments({chunk.toShardBSON()}); + return insertOp.serialize({}); + }()); + uassertStatusOK( + getStatusFromWriteCommandResponse(insertCommandResponse->getCommandReply())); } return Status::OK(); diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp index de7780165a3..6fb9435819f 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp +++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp @@ -37,14 +37,11 @@ #include "mongo/db/s/shard_server_catalog_cache_loader.h" -#include <fmt/format.h> #include <memory> #include "mongo/db/catalog/rename_collection.h" #include "mongo/db/client.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/db_raii.h" -#include "mongo/db/op_observer.h" #include "mongo/db/operation_context.h" #include "mongo/db/operation_context_group.h" #include "mongo/db/read_concern.h" @@ -54,7 +51,6 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/s/type_shard_collection.h" #include "mongo/db/s/type_shard_database.h" -#include "mongo/db/write_concern_options.h" #include "mongo/logv2/log.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" @@ -317,31 +313,6 @@ StatusWith<CollectionAndChangedChunks> getIncompletePersistedMetadataSinceVersio } /** - * Runs the given function and throws an exception if the topology term has changed between the - * start and end of it. If it changes this will uassert with an InterruptedDueToReplStateChange - * error. - */ -template <typename F> -auto runAndThrowIfTermChanged(OperationContext* opCtx, F&& fn) { - auto termBeforeOperation = repl::ReplicationCoordinator::get(opCtx)->getTerm(); - - if constexpr (!std::is_same_v<void, std::invoke_result_t<F, decltype(termBeforeOperation)>>) { - auto result = fn(termBeforeOperation); - auto termAtEndOfOperation = repl::ReplicationCoordinator::get(opCtx)->getTerm(); - uassert(ErrorCodes::InterruptedDueToReplStateChange, - "Change of ReplicaSet term detected between start and end of operation", - termBeforeOperation == termAtEndOfOperation); - return result; - } else { - fn(termBeforeOperation); - auto termAtEndOfOperation = repl::ReplicationCoordinator::get(opCtx)->getTerm(); - uassert(ErrorCodes::InterruptedDueToReplStateChange, - "Change of ReplicaSet term detected between start and end of operation", - termBeforeOperation == termAtEndOfOperation); - } -} - -/** * Sends _flushRoutingTableCacheUpdates to the primary to force it to refresh its routing table for * collection 'nss' and then waits for the refresh to replicate to this node. */ @@ -353,28 +324,18 @@ void forcePrimaryCollectionRefreshAndWaitForReplication(OperationContext* opCtx, auto selfShard = uassertStatusOK( Grid::get(opCtx)->shardRegistry()->getShard(opCtx, shardingState->shardId())); - // Run the operation on the primary and await the result to be replicated to this node. To avoid - // issues with rollback/term changes this is wrapped in a runAndThrowIfTermChanged since we - // currently have no way to detect a topology change/rollback after the return from primary is - // received. - runAndThrowIfTermChanged(opCtx, [&](auto term) { - auto cmdResponse = uassertStatusOK(selfShard->runCommandWithFixedRetryAttempts( - opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - "admin", - BSON("_flushRoutingTableCacheUpdates" << nss.ns()), - Seconds{30}, - Shard::RetryPolicy::kIdempotent)); + auto cmdResponse = uassertStatusOK(selfShard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + "admin", + BSON("_flushRoutingTableCacheUpdates" << nss.ns()), + Seconds{30}, + Shard::RetryPolicy::kIdempotent)); - uassertStatusOK(cmdResponse.commandStatus); + uassertStatusOK(cmdResponse.commandStatus); - uassertStatusOK(repl::ReplicationCoordinator::get(opCtx)->waitUntilOpTimeForRead( - opCtx, - {repl::OpTime{ - cmdResponse.response.getField(LogicalTime::kOperationTimeFieldName).timestamp(), - term}, - boost::none})); - }); + uassertStatusOK(repl::ReplicationCoordinator::get(opCtx)->waitUntilOpTimeForRead( + opCtx, {LogicalTime::fromOperationTime(cmdResponse.response), boost::none})); } /** @@ -388,54 +349,19 @@ void forcePrimaryDatabaseRefreshAndWaitForReplication(OperationContext* opCtx, S auto selfShard = uassertStatusOK( Grid::get(opCtx)->shardRegistry()->getShard(opCtx, shardingState->shardId())); - runAndThrowIfTermChanged(opCtx, [&](const auto term) { - auto cmdResponse = uassertStatusOK(selfShard->runCommandWithFixedRetryAttempts( - opCtx, - ReadPreferenceSetting{ReadPreference::PrimaryOnly}, - "admin", - BSON("_flushDatabaseCacheUpdates" << dbName.toString()), - Seconds{30}, - Shard::RetryPolicy::kIdempotent)); - - uassertStatusOK(cmdResponse.commandStatus); - - uassertStatusOK(repl::ReplicationCoordinator::get(opCtx)->waitUntilOpTimeForRead( - opCtx, - {repl::OpTime{ - cmdResponse.response.getField(LogicalTime::kOperationTimeFieldName).timestamp(), - term}, - boost::none})); - }); -} + auto cmdResponse = uassertStatusOK(selfShard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + "admin", + BSON("_flushDatabaseCacheUpdates" << dbName.toString()), + Seconds{30}, + Shard::RetryPolicy::kIdempotent)); -void performNoopMajorityWriteLocally(OperationContext* opCtx, StringData msg) { - const auto replCoord = repl::ReplicationCoordinator::get(opCtx); + uassertStatusOK(cmdResponse.commandStatus); - { - AutoGetOplog oplogWrite(opCtx, OplogAccessMode::kWrite); - uassert(ErrorCodes::NotWritablePrimary, - "Not primary when performing noop write for {}"_format(msg), - replCoord->canAcceptWritesForDatabase(opCtx, NamespaceString::kAdminDb)); - - writeConflictRetry( - opCtx, "performNoopWrite", NamespaceString::kRsOplogNamespace.ns(), [&opCtx, &msg] { - WriteUnitOfWork wuow(opCtx); - opCtx->getClient()->getServiceContext()->getOpObserver()->onOpMessage( - opCtx, BSON("msg" << msg)); - wuow.commit(); - }); - } - auto& replClient = repl::ReplClientInfo::forClient(opCtx->getClient()); - WriteConcernResult writeConcernResult; - uassertStatusOK( - waitForWriteConcern(opCtx, - replClient.getLastOp(), - WriteConcernOptions(WriteConcernOptions::kMajority, - WriteConcernOptions::SyncMode::UNSET, - WriteConcernOptions::kWriteConcernTimeoutSharding), - &writeConcernResult)); + uassertStatusOK(repl::ReplicationCoordinator::get(opCtx)->waitUntilOpTimeForRead( + opCtx, {LogicalTime::fromOperationTime(cmdResponse.response), boost::none})); } - } // namespace ShardServerCatalogCacheLoader::ShardServerCatalogCacheLoader( @@ -486,13 +412,6 @@ void ShardServerCatalogCacheLoader::onStepUp() { _role = ReplicaSetRole::Primary; } -void ShardServerCatalogCacheLoader::onReplicationRollback() { - // No need to increment the term since this interruption is only to prevent the secondary - // refresh thread from getting stuck or waiting on an incorrect opTime. - stdx::lock_guard<Latch> lg(_mutex); - _contexts.interrupt(ErrorCodes::Interrupted); -} - void ShardServerCatalogCacheLoader::shutDown() { { stdx::lock_guard<Latch> lg(_mutex); @@ -553,14 +472,6 @@ SemiFuture<CollectionAndChangedChunks> ShardServerCatalogCacheLoader::getChunksS } SemiFuture<DatabaseType> ShardServerCatalogCacheLoader::getDatabase(StringData dbName) { - // The admin and config database have fixed metadata that does not need to be refreshed. - if (dbName == NamespaceString::kAdminDb || dbName == NamespaceString::kConfigDb) { - auto dbMetadata = - DatabaseType(dbName.toString(), ShardId::kConfigServerId, DatabaseVersion::makeFixed()); - dbMetadata.setSharded(true); - return dbMetadata; - } - const auto [isPrimary, term] = [&] { stdx::lock_guard<Latch> lock(_mutex); return std::make_tuple(_role == ReplicaSetRole::Primary, _term); @@ -859,10 +770,9 @@ ShardServerCatalogCacheLoader::_schedulePrimaryGetChunksSince( } // After finding metadata remotely, we must have found metadata locally. - tassert(7032350, - str::stream() << "No chunks metadata found for collection '" << nss - << "' despite the config server returned actual information", - !collAndChunks.changedChunks.empty()); + invariant(!collAndChunks.changedChunks.empty(), + str::stream() << "No chunks metadata found for collection '" << nss + << "' despite the config server returned actual information"); return swCollectionAndChangedChunks; }; @@ -1046,10 +956,6 @@ std::pair<bool, CollectionAndChangedChunks> ShardServerCatalogCacheLoader::_getE void ShardServerCatalogCacheLoader::_ensureMajorityPrimaryAndScheduleCollAndChunksTask( OperationContext* opCtx, const NamespaceString& nss, CollAndChunkTask task) { - // Ensure that this node is primary before using or persisting the information fetched from the - // config server. This prevents using incorrect filtering information in split brain scenarios. - performNoopMajorityWriteLocally(opCtx, "ensureMajorityPrimaryAndScheduleCollAndChunksTask"); - { stdx::lock_guard<Latch> lock(_mutex); @@ -1078,11 +984,6 @@ void ShardServerCatalogCacheLoader::_ensureMajorityPrimaryAndScheduleDbTask(Oper StringData dbName, DBTask task) { - - // Ensure that this node is primary before using or persisting the information fetched from the - // config server. This prevents using incorrect filtering information in split brain scenarios. - performNoopMajorityWriteLocally(opCtx, "ensureMajorityPrimaryAndScheduleDbTask"); - { stdx::lock_guard<Latch> lock(_mutex); @@ -1266,9 +1167,7 @@ void ShardServerCatalogCacheLoader::_updatePersistedCollAndChunksMetadata( stdx::unique_lock<Latch> lock(_mutex); const CollAndChunkTask& task = _collAndChunkTaskLists[nss].front(); - tassert(7032351, - "Invalid CollAndChunkTask state", - task.dropped || !task.collectionAndChangedChunks->changedChunks.empty()); + invariant(task.dropped || !task.collectionAndChangedChunks->changedChunks.empty()); // If this task is from an old term and no longer valid, do not execute and return true so that // the task gets removed from the task list @@ -1394,9 +1293,7 @@ ShardServerCatalogCacheLoader::CollAndChunkTask::CollAndChunkTask( termCreated(currentTerm) { if (statusWithCollectionAndChangedChunks.isOK()) { collectionAndChangedChunks = std::move(statusWithCollectionAndChangedChunks.getValue()); - tassert(7032354, - "Found no chunks in retrieved collection metadata", - !collectionAndChangedChunks->changedChunks.empty()); + invariant(!collectionAndChangedChunks->changedChunks.empty()); const auto highestVersion = collectionAndChangedChunks->changedChunks.back().getVersion(); // Note that due to the way Phase 1 of the FCV upgrade writes timestamps to chunks // (non-atomically), it is possible that chunks exist with timestamps, but the @@ -1408,10 +1305,7 @@ ShardServerCatalogCacheLoader::CollAndChunkTask::CollAndChunkTask( highestVersion.epoch(), collectionAndChangedChunks->timestamp); } else { - tassert(7032358, - fmt::format("Encountered unexpected error while fetching collection metadata: {}", - statusWithCollectionAndChangedChunks.getStatus().toString()), - statusWithCollectionAndChangedChunks == ErrorCodes::NamespaceNotFound); + invariant(statusWithCollectionAndChangedChunks == ErrorCodes::NamespaceNotFound); dropped = true; maxQueryVersion = ChunkVersion::UNSHARDED(); } @@ -1423,10 +1317,7 @@ ShardServerCatalogCacheLoader::DBTask::DBTask(StatusWith<DatabaseType> swDatabas if (swDatabaseType.isOK()) { dbType = std::move(swDatabaseType.getValue()); } else { - tassert(7032355, - fmt::format("Encountered unexpected error while fetching database metadata: {}", - swDatabaseType.getStatus().toString()), - swDatabaseType == ErrorCodes::NamespaceNotFound); + invariant(swDatabaseType == ErrorCodes::NamespaceNotFound); } } @@ -1449,11 +1340,10 @@ void ShardServerCatalogCacheLoader::CollAndChunkTaskList::addTask(CollAndChunkTa } if (task.dropped) { - tassert(7032356, - str::stream() << "The version of the added task is not contiguous with that of " - << "the previous one: LastTask {" << lastTask.toString() << "}, " - << "AddedTask {" << task.toString() << "}", - lastTask.maxQueryVersion == task.minQueryVersion); + invariant(lastTask.maxQueryVersion == task.minQueryVersion, + str::stream() << "The version of the added task is not contiguous with that of " + << "the previous one: LastTask {" << lastTask.toString() << "}, " + << "AddedTask {" << task.toString() << "}"); // As an optimization, on collection drop, clear any pending tasks in order to prevent any // throw-away work from executing. Because we have no way to differentiate whether the @@ -1467,11 +1357,11 @@ void ShardServerCatalogCacheLoader::CollAndChunkTaskList::addTask(CollAndChunkTa } } else { // Tasks must have contiguous versions, unless a complete reload occurs. - tassert(7032357, - str::stream() << "The added task is not the first and its version is not " - << "contiguous with that of the previous one: LastTask {" - << lastTask.toString() << "}, AddedTask {" << task.toString() << "}", - lastTask.maxQueryVersion == task.minQueryVersion || !task.minQueryVersion.isSet()); + invariant(lastTask.maxQueryVersion == task.minQueryVersion || !task.minQueryVersion.isSet(), + str::stream() << "The added task is not the first and its version is not " + << "contiguous with that of the previous one: LastTask {" + << lastTask.toString() << "}, AddedTask {" << task.toString() + << "}"); _tasks.emplace_back(std::move(task)); } diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.h b/src/mongo/db/s/shard_server_catalog_cache_loader.h index d5a0bde6046..88bf4611b4d 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader.h +++ b/src/mongo/db/s/shard_server_catalog_cache_loader.h @@ -72,13 +72,6 @@ public: void shutDown() override; /** - * Interrupts ongoing refreshes to prevent secondaries from waiting for opTimes from wrong terms - * in case of rollback. Primaries must step down before going through rollback, so this should - * only be run on secondaries. - */ - void onReplicationRollback(); - - /** * Sets any notifications waiting for this version to arrive and invalidates the catalog cache's * chunk metadata for collection 'nss' so that the next caller provokes a refresh. */ diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp index a374159bc87..228ef5ffbb9 100644 --- a/src/mongo/db/s/shard_server_op_observer.cpp +++ b/src/mongo/db/s/shard_server_op_observer.cpp @@ -80,10 +80,8 @@ bool isStandaloneOrPrimary(OperationContext* opCtx) { */ class CollectionVersionLogOpHandler final : public RecoveryUnit::Change { public: - CollectionVersionLogOpHandler(OperationContext* opCtx, - const NamespaceString& nss, - bool droppingCollection) - : _opCtx(opCtx), _nss(nss), _droppingCollection(droppingCollection) {} + CollectionVersionLogOpHandler(OperationContext* opCtx, const NamespaceString& nss) + : _opCtx(opCtx), _nss(nss) {} void commit(boost::optional<Timestamp>) override { invariant(_opCtx->lockState()->isCollectionLockedForMode(_nss, MODE_IX)); @@ -93,11 +91,7 @@ public: // Force subsequent uses of the namespace to refresh the filtering metadata so they can // synchronize with any work happening on the primary (e.g., migration critical section). UninterruptibleLockGuard noInterrupt(_opCtx->lockState()); - if (_droppingCollection) - CollectionShardingRuntime::get(_opCtx, _nss) - ->clearFilteringMetadataForDroppedCollection(_opCtx); - else - CollectionShardingRuntime::get(_opCtx, _nss)->clearFilteringMetadata(_opCtx); + CollectionShardingRuntime::get(_opCtx, _nss)->clearFilteringMetadata(_opCtx); } void rollback() override {} @@ -105,7 +99,6 @@ public: private: OperationContext* _opCtx; const NamespaceString _nss; - const bool _droppingCollection; }; /** @@ -157,8 +150,8 @@ void onConfigDeleteInvalidateCachedCollectionMetadataAndNotify(OperationContext* AllowLockAcquisitionOnTimestampedUnitOfWork allowLockAcquisition(opCtx->lockState()); AutoGetCollection autoColl(opCtx, deletedNss, MODE_IX); - opCtx->recoveryUnit()->registerChange(std::make_unique<CollectionVersionLogOpHandler>( - opCtx, deletedNss, /* droppingCollection */ true)); + opCtx->recoveryUnit()->registerChange( + std::make_unique<CollectionVersionLogOpHandler>(opCtx, deletedNss)); } /** @@ -196,9 +189,7 @@ void incrementChunkOnInsertOrUpdate(OperationContext* opCtx, return balancerConfig->getMaxChunkSizeBytes(); }(); - if (!feature_flags::gNoMoreAutoSplitter.isEnabled( - serverGlobalParams.featureCompatibility) && - balancerConfig->getShouldAutoSplit() && chunkManager.allowAutoSplit() && + if (balancerConfig->getShouldAutoSplit() && chunkManager.allowAutoSplit() && chunkWritesTracker->shouldSplit(maxChunkSizeBytes)) { auto chunkSplitStateDriver = ChunkSplitStateDriver::tryInitiateSplit(chunkWritesTracker); @@ -361,8 +352,8 @@ void ShardServerOpObserver::onUpdate(OperationContext* opCtx, const OplogUpdateE AllowLockAcquisitionOnTimestampedUnitOfWork allowLockAcquisition(opCtx->lockState()); AutoGetCollection autoColl(opCtx, updatedNss, MODE_IX); if (refreshingFieldNewVal.isBoolean() && !refreshingFieldNewVal.boolean()) { - opCtx->recoveryUnit()->registerChange(std::make_unique<CollectionVersionLogOpHandler>( - opCtx, updatedNss, /* droppingCollection */ false)); + opCtx->recoveryUnit()->registerChange( + std::make_unique<CollectionVersionLogOpHandler>(opCtx, updatedNss)); } if (enterCriticalSectionFieldNewVal.ok()) { @@ -399,13 +390,9 @@ void ShardServerOpObserver::onUpdate(OperationContext* opCtx, const OplogUpdateE // TODO SERVER-58223: evaluate whether this is safe or whether acquiring the lock can // block. AllowLockAcquisitionOnTimestampedUnitOfWork allowLockAcquisition(opCtx->lockState()); - AutoGetDb autoDb(opCtx, db, MODE_X); auto dss = DatabaseShardingState::get(opCtx, db); dss->clearDatabaseInfo(opCtx); - - const auto dssLock = DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss); - dss->cancelDbMetadataRefresh(dssLock); } } @@ -500,13 +487,9 @@ void ShardServerOpObserver::onDelete(OperationContext* opCtx, // TODO SERVER-58223: evaluate whether this is safe or whether acquiring the lock can block. AllowLockAcquisitionOnTimestampedUnitOfWork allowLockAcquisition(opCtx->lockState()); - AutoGetDb autoDb(opCtx, deletedDatabase, MODE_X); auto dss = DatabaseShardingState::get(opCtx, deletedDatabase); dss->clearDatabaseInfo(opCtx); - - const auto dssLock = DatabaseShardingState::DSSLock::lockExclusive(opCtx, dss); - dss->cancelDbMetadataRefresh(dssLock); } if (nss == NamespaceString::kServerConfigurationNamespace) { @@ -588,7 +571,6 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx, const OplogSlot& createOpTime, bool fromMigrate) { // Only the shard primay nodes control the collection creation and secondaries just follow - // Secondaries CSR will be the defaulted one (UNKNOWN in most of the cases) if (!opCtx->writesAreReplicated()) { return; } @@ -615,9 +597,7 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx, // If the check above passes, this means the collection doesn't exist and is being created and // that the caller will be responsible to eventially set the proper shard version auto* const csr = CollectionShardingRuntime::get(opCtx, collectionName); - if (oss._forceCSRAsUnknownAfterCollectionCreation) { - csr->clearFilteringMetadata(opCtx); - } else if (!csr->getCurrentMetadataIfKnown()) { + if (!csr->getCurrentMetadataIfKnown()) { csr->setFilteringMetadata(opCtx, CollectionMetadata()); } } @@ -644,14 +624,6 @@ repl::OpTime ShardServerOpObserver::onDropCollection(OperationContext* opCtx, return {}; } -void ShardServerOpObserver::onCreateIndex(OperationContext* opCtx, - const NamespaceString& nss, - const UUID& uuid, - BSONObj indexDoc, - bool fromMigrate) { - abortOngoingMigrationIfNeeded(opCtx, nss); -} - void ShardServerOpObserver::onStartIndexBuild(OperationContext* opCtx, const NamespaceString& nss, const UUID& collUUID, @@ -694,17 +666,6 @@ void ShardServerOpObserver::_onReplicationRollback(OperationContext* opCtx, rbInfo.rollbackNamespaces.end()) { RecoverableCriticalSectionService::get(opCtx)->recoverRecoverableCriticalSections(opCtx); } - - // If writes to config.cache.collections or config.cache.* have been rolled back, interrupt - // the SSCCL to ensure secondary waits for replication do not use incorrect opTimes. - if (std::any_of(rbInfo.rollbackNamespaces.begin(), - rbInfo.rollbackNamespaces.end(), - [](const NamespaceString& nss) { - return nss == NamespaceString::kShardConfigCollectionsNamespace || - nss.isConfigDotCacheDotChunks(); - })) { - CatalogCacheLoader::get(opCtx).onReplicationRollback(); - } } diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 702949295be..5a0671254d6 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -49,7 +49,7 @@ public: const NamespaceString& nss, const UUID& uuid, BSONObj indexDoc, - bool fromMigrate) override; + bool fromMigrate) override {} void onStartIndexBuild(OperationContext* opCtx, const NamespaceString& nss, @@ -207,10 +207,6 @@ public: size_t numberOfPrePostImagesToWrite, Date_t wallClockTime) override {} - void onTransactionPrepareNonPrimary(OperationContext* opCtx, - const std::vector<repl::OplogEntry>& statements, - const repl::OpTime& prepareOpTime) override {} - void onTransactionAbort(OperationContext* opCtx, boost::optional<OplogSlot> abortOplogEntryOpTime) override {} diff --git a/src/mongo/db/s/shard_server_test_fixture.cpp b/src/mongo/db/s/shard_server_test_fixture.cpp index 05bd61a4022..fe4de75a820 100644 --- a/src/mongo/db/s/shard_server_test_fixture.cpp +++ b/src/mongo/db/s/shard_server_test_fixture.cpp @@ -85,8 +85,9 @@ void ShardServerTestFixture::setCatalogCacheLoader(std::unique_ptr<CatalogCacheL } void ShardServerTestFixture::tearDown() { - ShardingMongodTestFixture::tearDown(); CatalogCacheLoader::clearForTests(getServiceContext()); + + ShardingMongodTestFixture::tearDown(); } std::unique_ptr<ShardingCatalogClient> ShardServerTestFixture::makeShardingCatalogClient() { diff --git a/src/mongo/db/s/sharded_rename_collection.idl b/src/mongo/db/s/sharded_rename_collection.idl index 0d972dc6f64..1d43633d5e8 100644 --- a/src/mongo/db/s/sharded_rename_collection.idl +++ b/src/mongo/db/s/sharded_rename_collection.idl @@ -123,11 +123,6 @@ structs: type: uuid description: "ID of the existing collection getting dropped." optional: true - allowEncryptedCollectionRename: - description: "Encrypted Collection renames are usually disallowed to minimize user error. - C2C needs to do the renames to replicate create collection." - type: bool - optional: true RenameCollectionParticipantDocument: description: "Represents a rename collection operation on a participant shard." diff --git a/src/mongo/db/s/sharding_config_server_parameters.idl b/src/mongo/db/s/sharding_config_server_parameters.idl index 327ea6fb429..00a5c8154e3 100644 --- a/src/mongo/db/s/sharding_config_server_parameters.idl +++ b/src/mongo/db/s/sharding_config_server_parameters.idl @@ -46,36 +46,3 @@ server_parameters: validator: gte: 0 default: 1000 - - balancerMigrationsThrottlingMs: - # Sleep between balancer rounds in the case where the last round found some chunks which needed to - # be balanced. This value should be set sufficiently low so that imbalanced clusters will quickly - # reach balanced state, but setting it too low may cause CRUD operations to start failing due to - # not being able to establish a stable shard version. - description: The minimum amount of time between two consecutive balancing rounds - set_at: [startup, runtime] - cpp_vartype: AtomicWord<int32_t> - cpp_varname: balancerMigrationsThrottlingMs - default: 1000 # 1 sec - - balancerChunksSelectionTimeoutMs: - description: >- - Maximum time in milliseconds the balancer will spend deciding which ranges to move in - the current balancing round. - set_at: [startup, runtime] - cpp_vartype: AtomicWord<int32_t> - cpp_varname: balancerChunksSelectionTimeoutMs - default: 5000 # 5 seconds - - newShardExistingClusterTimeKeysExpirationSecs: - description: >- - The amount of time in seconds that the config server should wait before removing the - key documents fetched from a replica set when it gets added to the cluster - as a shard. - set_at: [ startup, runtime ] - cpp_vartype: AtomicWord<int> - cpp_varname: gNewShardExistingClusterTimeKeysExpirationSecs - default: - expr: 7 * 24 * 3600 # 1 week - validator: - gte: 1 diff --git a/src/mongo/db/s/sharding_ddl_coordinator.cpp b/src/mongo/db/s/sharding_ddl_coordinator.cpp index 5502f0232ff..12925bad589 100644 --- a/src/mongo/db/s/sharding_ddl_coordinator.cpp +++ b/src/mongo/db/s/sharding_ddl_coordinator.cpp @@ -52,13 +52,12 @@ namespace mongo { MONGO_FAIL_POINT_DEFINE(hangBeforeRunningCoordinatorInstance); MONGO_FAIL_POINT_DEFINE(overrideDDLLockTimeout); -MONGO_FAIL_POINT_DEFINE(hangBeforeRemovingCoordinatorDocument); namespace { const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); -} // namespace +} ShardingDDLCoordinatorMetadata extractShardingDDLCoordinatorMetadata(const BSONObj& coorDoc) { return ShardingDDLCoordinatorMetadata::parse( @@ -159,23 +158,11 @@ ExecutorFuture<void> ShardingDDLCoordinator::_acquireLockAsync( }(); auto distLock = distLockManager->lockDirectLocally(opCtx, resource, lockTimeOut); + _scopedLocks.emplace(std::move(distLock)); uassertStatusOK(distLockManager->lockDirect(opCtx, resource, coorName, lockTimeOut)); - _scopedLocks.emplace(std::move(distLock)); }) - .until([this, resource = resource.toString()](Status status) { - if (!status.isOK()) { - LOGV2_WARNING(6819300, - "DDL lock acquisition attempt failed", - "coordinatorId"_attr = _coordId, - "resource"_attr = resource, - "error"_attr = redact(status)); - } - // Sharding DDL operations are not rollbackable so in case we recovered a coordinator - // from disk we need to ensure eventual completion of the DDL operation, so we must - // retry until we manage to acquire the lock. - return (!_recoveredFromDisk) || status.isOK(); - }) + .until([this](Status status) { return (!_recoveredFromDisk) || status.isOK(); }) .withBackoffBetweenIterations(kExponentialBackoff) .on(**executor, token); } @@ -292,7 +279,16 @@ SemiFuture<void> ShardingDDLCoordinator::run(std::shared_ptr<executor::ScopedTas // If the token is not cancelled we retry because it could have been generated // by a remote node. if (!status.isOK() && !_completeOnError && - (_mustAlwaysMakeProgress() || _isRetriableErrorForDDLCoordinator(status)) && + (_mustAlwaysMakeProgress() || + status.isA<ErrorCategory::CursorInvalidatedError>() || + status.isA<ErrorCategory::ShutdownError>() || + status.isA<ErrorCategory::RetriableError>() || + status.isA<ErrorCategory::CancellationError>() || + status.isA<ErrorCategory::ExceededTimeLimitError>() || + status.isA<ErrorCategory::WriteConcernError>() || + status == ErrorCodes::FailedToSatisfyReadPreference || + status == ErrorCodes::Interrupted || status == ErrorCodes::LockBusy || + status == ErrorCodes::CommandNotFound) && !token.isCanceled()) { LOGV2_INFO(5656000, "Re-executing sharding DDL coordinator", @@ -330,8 +326,6 @@ SemiFuture<void> ShardingDDLCoordinator::run(std::shared_ptr<executor::ScopedTas if (cleanup()) { try { - hangBeforeRemovingCoordinatorDocument.pauseWhileSet(); - LOGV2(5565601, "Releasing sharding DDL coordinator", "coordinatorId"_attr = _coordId); @@ -408,15 +402,4 @@ void ShardingDDLCoordinator::_performNoopRetryableWriteOnAllShardsAndConfigsvr( sharding_ddl_util::performNoopRetryableWriteOnShards(opCtx, shardsAndConfigsvr, osi, executor); } -bool ShardingDDLCoordinator::_isRetriableErrorForDDLCoordinator(const Status& status) { - return status.isA<ErrorCategory::CursorInvalidatedError>() || - status.isA<ErrorCategory::ShutdownError>() || status.isA<ErrorCategory::RetriableError>() || - status.isA<ErrorCategory::Interruption>() || - status.isA<ErrorCategory::CancellationError>() || - status.isA<ErrorCategory::ExceededTimeLimitError>() || - status.isA<ErrorCategory::WriteConcernError>() || - status == ErrorCodes::FailedToSatisfyReadPreference || status == ErrorCodes::Interrupted || - status == ErrorCodes::LockBusy || status == ErrorCodes::CommandNotFound; -} - } // namespace mongo diff --git a/src/mongo/db/s/sharding_ddl_coordinator.h b/src/mongo/db/s/sharding_ddl_coordinator.h index f18b3b0ff5d..5972c7ce9e6 100644 --- a/src/mongo/db/s/sharding_ddl_coordinator.h +++ b/src/mongo/db/s/sharding_ddl_coordinator.h @@ -203,11 +203,6 @@ protected: return false; }; - /* - * Specify if the given error will be retried by the ddl coordinator infrastructure. - */ - bool _isRetriableErrorForDDLCoordinator(const Status& status); - ShardingDDLCoordinatorService* _service; const ShardingDDLCoordinatorId _coordId; diff --git a/src/mongo/db/s/sharding_ddl_util.cpp b/src/mongo/db/s/sharding_ddl_util.cpp index ba13de4abbd..d516fd5e668 100644 --- a/src/mongo/db/s/sharding_ddl_util.cpp +++ b/src/mongo/db/s/sharding_ddl_util.cpp @@ -33,7 +33,6 @@ #include "mongo/db/catalog/collection_catalog.h" #include "mongo/db/commands/feature_compatibility_version.h" -#include "mongo/db/concurrency/exception_util.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/repl/repl_client_info.h" @@ -77,14 +76,11 @@ void updateTags(OperationContext* opCtx, }()}); return updateOp; }()); + request.setWriteConcern(writeConcern.toBSON()); auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = - configShard->runBatchWriteCommand(opCtx, - Milliseconds::max(), - request, - writeConcern, - Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = configShard->runBatchWriteCommand( + opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } @@ -108,13 +104,11 @@ void deleteChunks(OperationContext* opCtx, return deleteOp; }()); + request.setWriteConcern(writeConcern.toBSON()); + auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = - configShard->runBatchWriteCommand(opCtx, - Milliseconds::max(), - request, - writeConcern, - Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = configShard->runBatchWriteCommand( + opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } @@ -179,93 +173,6 @@ void setAllowMigrations(OperationContext* opCtx, } } - -// Check that the collection UUID is the same in every shard knowing the collection -void checkCollectionUUIDConsistencyAcrossShards( - OperationContext* opCtx, - const NamespaceString& nss, - const UUID& collectionUuid, - const std::vector<mongo::ShardId>& shardIds, - std::shared_ptr<executor::ScopedTaskExecutor> executor) { - const BSONObj filterObj = BSON("name" << nss.coll()); - BSONObj cmdObj = BSON("listCollections" << 1 << "filter" << filterObj); - - auto responses = sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, nss.db().toString(), cmdObj, shardIds, **executor); - - struct MismatchedShard { - std::string shardId; - std::string uuid; - }; - - std::vector<MismatchedShard> mismatches; - - for (auto cmdResponse : responses) { - auto responseData = uassertStatusOK(cmdResponse.swResponse); - auto collectionVector = responseData.data.firstElement()["firstBatch"].Array(); - auto shardId = cmdResponse.shardId; - - if (collectionVector.empty()) { - // Collection does not exist on the shard - continue; - } - - auto bsonCollectionUuid = collectionVector.front()["info"]["uuid"]; - if (collectionUuid.data() != bsonCollectionUuid.uuid()) { - mismatches.push_back({shardId.toString(), bsonCollectionUuid.toString()}); - } - } - - if (!mismatches.empty()) { - std::stringstream errorMessage; - errorMessage << "The collection " << nss.toString() - << " with expected UUID: " << collectionUuid.toString() - << " has different UUIDs on the following shards: ["; - - for (auto mismatch : mismatches) { - errorMessage << "{ " << mismatch.shardId << ":" << mismatch.uuid << " },"; - } - errorMessage << "]"; - uasserted(ErrorCodes::InvalidUUID, errorMessage.str()); - } -} - - -// Check the collection does not exist in any shard when `dropTarget` is set to false -void checkTargetCollectionDoesNotExistInCluster( - OperationContext* opCtx, - const NamespaceString& toNss, - const std::vector<mongo::ShardId>& shardIds, - std::shared_ptr<executor::ScopedTaskExecutor> executor) { - const BSONObj filterObj = BSON("name" << toNss.coll()); - BSONObj cmdObj = BSON("listCollections" << 1 << "filter" << filterObj); - - auto responses = sharding_ddl_util::sendAuthenticatedCommandToShards( - opCtx, toNss.db(), cmdObj, shardIds, **executor); - - std::vector<std::string> shardsContainingTargetCollection; - for (auto cmdResponse : responses) { - uassertStatusOK(cmdResponse.swResponse); - auto responseData = uassertStatusOK(cmdResponse.swResponse); - auto collectionVector = responseData.data.firstElement()["firstBatch"].Array(); - - if (!collectionVector.empty()) { - shardsContainingTargetCollection.push_back(cmdResponse.shardId.toString()); - } - } - - if (!shardsContainingTargetCollection.empty()) { - std::stringstream errorMessage; - errorMessage << "The collection " << toNss.toString() - << " already exists in the following shards: ["; - std::move(shardsContainingTargetCollection.begin(), - shardsContainingTargetCollection.end(), - std::ostream_iterator<std::string>(errorMessage, ", ")); - errorMessage << "]"; - uasserted(ErrorCodes::NamespaceExists, errorMessage.str()); - } -} - } // namespace void linearizeCSRSReads(OperationContext* opCtx) { @@ -285,20 +192,27 @@ std::vector<AsyncRequestsSender::Response> sendAuthenticatedCommandToShards( StringData dbName, const BSONObj& command, const std::vector<ShardId>& shardIds, - const std::shared_ptr<executor::TaskExecutor>& executor, - const bool throwOnError) { + const std::shared_ptr<executor::TaskExecutor>& executor) { + // TODO SERVER-57519: remove the following scope + { + // Ensure ShardRegistry is initialized before using the AsyncRequestsSender that relies on + // unsafe functions (SERVER-57280) + auto shardRegistry = Grid::get(opCtx)->shardRegistry(); + if (!shardRegistry->isUp()) { + shardRegistry->reload(opCtx); + } + } // The AsyncRequestsSender ignore impersonation metadata so we need to manually attach them to // the command BSONObjBuilder bob(command); rpc::writeAuthDataToImpersonatedUserMetadata(opCtx, &bob); - if (serverGlobalParams.featureCompatibility.isVersionInitialized() && - gFeatureFlagUserWriteBlocking.isEnabled(serverGlobalParams.featureCompatibility)) { + if (gFeatureFlagUserWriteBlocking.isEnabled(serverGlobalParams.featureCompatibility)) { WriteBlockBypass::get(opCtx).writeAsMetadata(&bob); } auto authenticatedCommand = bob.obj(); return sharding_util::sendCommandToShards( - opCtx, dbName, authenticatedCommand, shardIds, executor, throwOnError); + opCtx, dbName, authenticatedCommand, shardIds, executor); } void removeTagsMetadataFromConfig(OperationContext* opCtx, @@ -341,13 +255,11 @@ void removeTagsMetadataFromConfig_notIdempotent(OperationContext* opCtx, return deleteOp; }()); + request.setWriteConcern(writeConcern.toBSON()); + auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - auto response = - configShard->runBatchWriteCommand(opCtx, - Milliseconds::max(), - request, - writeConcern, - Shard::RetryPolicy::kIdempotentOrCursorInvalidated); + auto response = configShard->runBatchWriteCommand( + opCtx, Milliseconds::max(), request, Shard::RetryPolicy::kIdempotentOrCursorInvalidated); uassertStatusOK(response.toStatus()); } @@ -440,24 +352,6 @@ void shardedRenameMetadata(OperationContext* opCtx, opCtx, CollectionType::ConfigNS, fromCollType.toBSON(), writeConcern)); } -void checkCatalogConsistencyAcrossShardsForRename( - OperationContext* opCtx, - const NamespaceString& fromNss, - const NamespaceString& toNss, - const bool dropTarget, - std::shared_ptr<executor::ScopedTaskExecutor> executor) { - - auto participants = Grid::get(opCtx)->shardRegistry()->getAllShardIds(opCtx); - - auto sourceCollUuid = *getCollectionUUID(opCtx, fromNss); - checkCollectionUUIDConsistencyAcrossShards( - opCtx, fromNss, sourceCollUuid, participants, executor); - - if (!dropTarget) { - checkTargetCollectionDoesNotExistInCluster(opCtx, toNss, participants, executor); - } -} - void checkRenamePreconditions(OperationContext* opCtx, bool sourceIsSharded, const NamespaceString& toNss, @@ -557,26 +451,6 @@ void resumeMigrations(OperationContext* opCtx, setAllowMigrations(opCtx, nss, expectedCollectionUUID, true); } -bool checkAllowMigrations(OperationContext* opCtx, const NamespaceString& nss) { - auto collDoc = - uassertStatusOK(Grid::get(opCtx)->shardRegistry()->getConfigShard()->exhaustiveFindOnConfig( - opCtx, - ReadPreferenceSetting(ReadPreference::PrimaryOnly, TagSet{}), - repl::ReadConcernLevel::kMajorityReadConcern, - CollectionType::ConfigNS, - BSON(CollectionType::kNssFieldName << nss.ns()), - BSONObj(), - 1)) - .docs; - - uassert(ErrorCodes::NamespaceNotFound, - str::stream() << "collection " << nss.ns() << " not found", - !collDoc.empty()); - - auto coll = CollectionType(collDoc[0]); - return coll.getAllowMigrations(); -} - boost::optional<UUID> getCollectionUUID(OperationContext* opCtx, const NamespaceString& nss, bool allowViews) { @@ -625,11 +499,8 @@ void sendDropCollectionParticipantCommandToShards(OperationContext* opCtx, const NamespaceString& nss, const std::vector<ShardId>& shardIds, std::shared_ptr<executor::TaskExecutor> executor, - const OperationSessionInfo& osi, - bool fromMigrate) { - ShardsvrDropCollectionParticipant dropCollectionParticipant(nss); - dropCollectionParticipant.setFromMigrate(fromMigrate); - + const OperationSessionInfo& osi) { + const ShardsvrDropCollectionParticipant dropCollectionParticipant(nss); const auto cmdObj = CommandHelpers::appendMajorityWriteConcern(dropCollectionParticipant.toBSON({})); @@ -645,34 +516,5 @@ void sendDropCollectionParticipantCommandToShards(OperationContext* opCtx, } } -BSONObj getCriticalSectionReasonForRename(const NamespaceString& from, const NamespaceString& to) { - return BSON("command" - << "rename" - << "from" << from.toString() << "to" << to.toString()); -} - -void ensureCollectionDroppedNoChangeEvent(OperationContext* opCtx, - const NamespaceString& nss, - const boost::optional<UUID>& uuid) { - invariant(!opCtx->lockState()->isLocked()); - invariant(!opCtx->lockState()->inAWriteUnitOfWork()); - - writeConflictRetry(opCtx, - "mongo::sharding_ddl_util::ensureCollectionDroppedNoChangeEvent", - nss.toString(), - [&] { - AutoGetCollection coll(opCtx, nss, MODE_X); - if (!coll || (uuid && coll->uuid() != uuid)) { - // If the collection doesn't exist or exists with a different UUID, - // then the requested collection has been dropped already. - return; - } - - WriteUnitOfWork wuow(opCtx); - uassertStatusOK(coll.getDb()->dropCollectionEvenIfSystem( - opCtx, nss, {} /* dropOpTime */, true /* markFromMigrate */)); - wuow.commit(); - }); -} } // namespace sharding_ddl_util } // namespace mongo diff --git a/src/mongo/db/s/sharding_ddl_util.h b/src/mongo/db/s/sharding_ddl_util.h index b16f749d1fe..afe53cd5ee6 100644 --- a/src/mongo/db/s/sharding_ddl_util.h +++ b/src/mongo/db/s/sharding_ddl_util.h @@ -54,8 +54,7 @@ std::vector<AsyncRequestsSender::Response> sendAuthenticatedCommandToShards( StringData dbName, const BSONObj& command, const std::vector<ShardId>& shardIds, - const std::shared_ptr<executor::TaskExecutor>& executor, - bool throwOnError = true); + const std::shared_ptr<executor::TaskExecutor>& executor); /** * Erase tags metadata from config server for the given namespace, using the _configsvrRemoveTags @@ -105,17 +104,6 @@ void shardedRenameMetadata(OperationContext* opCtx, const WriteConcernOptions& writeConcern); /** - * Ensure source collection uuid is consistent on every shard - * Ensure target collection is not present on any shard when `dropTarget` is false - */ -void checkCatalogConsistencyAcrossShardsForRename( - OperationContext* opCtx, - const NamespaceString& fromNss, - const NamespaceString& toNss, - bool dropTarget, - std::shared_ptr<executor::ScopedTaskExecutor> executor); - -/** * Ensures rename preconditions for collections are met: * - Check that the namespace of the destination collection is not too long * - Check that `dropTarget` is true if the destination collection exists @@ -167,12 +155,6 @@ void resumeMigrations(OperationContext* opCtx, const NamespaceString& nss, const boost::optional<UUID>& expectedCollectionUUID); -/** - * Calls to the config server primary to get the collection document for the given nss. - * Returns the value of the allowMigrations flag on the collection document. - */ -bool checkAllowMigrations(OperationContext* opCtx, const NamespaceString& nss); - /* * Returns the UUID of the collection (if exists) using the catalog. It does not provide any locking * guarantees after the call. @@ -203,21 +185,7 @@ void sendDropCollectionParticipantCommandToShards(OperationContext* opCtx, const NamespaceString& nss, const std::vector<ShardId>& shardIds, std::shared_ptr<executor::TaskExecutor> executor, - const OperationSessionInfo& osi, - bool fromMigrate); - -BSONObj getCriticalSectionReasonForRename(const NamespaceString& from, const NamespaceString& to); - -/** - * Drops the specified collection or returns without error if the collection has already been - * dropped. A particular incarnation of the collection can be dropped by specifying its UUID. - * - * This functions assumes the collection being dropped doesn't have any two-phase index builds - * active on it. - */ -void ensureCollectionDroppedNoChangeEvent(OperationContext* opCtx, - const NamespaceString& nss, - const boost::optional<UUID>& uuid = boost::none); + const OperationSessionInfo& osi); } // namespace sharding_ddl_util } // namespace mongo diff --git a/src/mongo/db/s/sharding_initialization_mongod.cpp b/src/mongo/db/s/sharding_initialization_mongod.cpp index 33a55c10c09..410ef901ebf 100644 --- a/src/mongo/db/s/sharding_initialization_mongod.cpp +++ b/src/mongo/db/s/sharding_initialization_mongod.cpp @@ -283,6 +283,9 @@ void ShardingInitializationMongoD::shutDown(OperationContext* opCtx) { DistLockManager::get(opCtx)->shutDown(opCtx); + auto const grid = Grid::get(opCtx); + grid->shardRegistry()->shutdown(); + _replicaSetChangeListener.reset(); } diff --git a/src/mongo/db/s/sharding_logging.cpp b/src/mongo/db/s/sharding_logging.cpp index ccbc2d2a6c4..dfa7626e29f 100644 --- a/src/mongo/db/s/sharding_logging.cpp +++ b/src/mongo/db/s/sharding_logging.cpp @@ -164,10 +164,10 @@ Status ShardingLogging::_log(OperationContext* opCtx, opCtx, nss, changeLogBSON, writeConcern); if (!result.isOK()) { - LOGV2_WARNING(5538900, - "Error encountered while logging config change", - "changeDocument"_attr = changeLog, - "error"_attr = redact(result)); + LOGV2_ERROR(5538900, + "Error encountered while logging config change", + "changeDocument"_attr = changeLog, + "error"_attr = redact(result)); } return result; diff --git a/src/mongo/db/s/sharding_runtime_d_params.h b/src/mongo/db/s/sharding_runtime_d_params.h deleted file mode 100644 index 88f76450d4f..00000000000 --- a/src/mongo/db/s/sharding_runtime_d_params.h +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Copyright (C) 2022-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "fmt/core.h" -#include "mongo/base/status.h" -#include "mongo/db/commands/test_commands_enabled.h" -#include "mongo/s/sharding_feature_flags_gen.h" -#include "mongo/util/processinfo.h" - -namespace mongo { - -inline Status validateChunkMigrationConcurrency(const int& chunkMigrationConcurrency) { - const int maxConcurrency = 500; - if (!mongo::feature_flags::gConcurrencyInChunkMigration.isEnabledAndIgnoreFCV()) { - return Status{ErrorCodes::InvalidOptions, - "Cannot set migration concurrency number without enabling migration " - "concurrency feature flag"}; - } - - if (chunkMigrationConcurrency <= 0 || - (chunkMigrationConcurrency > maxConcurrency && !getTestCommandsEnabled())) { - return Status{ - ErrorCodes::InvalidOptions, - fmt::format("Chunk migration concurrency level must be positive and less than {}.", - maxConcurrency)}; - } - return Status::OK(); -} - -inline Status validateChunkMigrationFetcherMaxBufferedSizeBytesPerThread(const int& maxSize) { - if (maxSize == 0) { - return Status::OK(); - } - - if (maxSize < BSONObjMaxInternalSize) { - return Status{ErrorCodes::InvalidOptions, - fmt::format("Chunk migration concurrency level must be 0 (no limit) or " - "greater than or equal to {}.", - BSONObjMaxInternalSize)}; - } - return Status::OK(); -} - -} // namespace mongo diff --git a/src/mongo/db/s/sharding_runtime_d_params.idl b/src/mongo/db/s/sharding_runtime_d_params.idl index 6a4a9cd57ee..aeb0c8b676a 100644 --- a/src/mongo/db/s/sharding_runtime_d_params.idl +++ b/src/mongo/db/s/sharding_runtime_d_params.idl @@ -27,34 +27,8 @@ global: cpp_namespace: mongo - cpp_includes: - - "mongo/db/s/sharding_runtime_d_params.h" server_parameters: - chunkMigrationConcurrency: - description: >- - The number of threads doing insertions on the recipient during a chunk migration and - also the number of _migrateClone requests that the recipient sends to the source in parallel. - set_at: [startup, runtime] - cpp_vartype: AtomicWord<int> - cpp_varname: chunkMigrationConcurrency - validator: - callback: validateChunkMigrationConcurrency - default: 1 - - chunkMigrationFetcherMaxBufferedSizeBytesPerThread: - description: >- - Maximum size of buffered documents that the chunk migration cloner will keep per - thread. A value of 0 means there will be no limit. The minimum size is the maximum - BSON size limit. - set_at: [startup, runtime] - cpp_vartype: AtomicWord<int> - cpp_varname: chunkMigrationFetcherMaxBufferedSizeBytesPerThread - validator: - callback: validateChunkMigrationFetcherMaxBufferedSizeBytesPerThread - default: - expr: 4 * BSONObjMaxInternalSize - rangeDeleterBatchSize: description: >- The maximum number of documents in each batch to delete during the cleanup stage of chunk @@ -83,7 +57,7 @@ server_parameters: receiveChunkWaitForRangeDeleterTimeoutMS: description: >- - Amount of time in milliseconds an incoming migration will wait for an intersecting range + Amount of time in milliseconds an incoming migration will wait for an intersecting range with data in it to be cleared up before failing. set_at: [startup, runtime] cpp_vartype: AtomicWord<int> @@ -170,13 +144,3 @@ server_parameters: cpp_vartype: int cpp_varname: shardedIndexConsistencyCheckIntervalMS default: 600000 - - persistedChunkCacheUpdateMaxBatchSize: - description: >- - Maximum batch size used for updating shard persisted chunk cache. - set_at: [startup, runtime] - cpp_vartype: AtomicWord<int> - cpp_varname: persistedChunkCacheUpdateMaxBatchSize - validator: - gte: 1 - default: 1000 diff --git a/src/mongo/db/s/sharding_server_status.cpp b/src/mongo/db/s/sharding_server_status.cpp index bf292614506..5a331364032 100644 --- a/src/mongo/db/s/sharding_server_status.cpp +++ b/src/mongo/db/s/sharding_server_status.cpp @@ -31,7 +31,6 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/commands/server_status.h" -#include "mongo/db/db_raii.h" #include "mongo/db/s/active_migrations_registry.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/resharding/resharding_metrics.h" @@ -124,16 +123,6 @@ public: CollectionShardingState::appendInfoForServerStatus(opCtx, &result); } - // To calculate the number of sharded collection we simply get the number of records from - // `config.collections` collection. This count must only be appended when serverStatus is - // invoked on the config server. - if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { - AutoGetCollectionForReadLockFree autoColl(opCtx, CollectionType::ConfigNS); - const auto& collection = autoColl.getCollection(); - const auto numShardedCollections = collection ? collection->numRecords(opCtx) : 0; - result.append("numShardedCollections", numShardedCollections); - } - // The serverStatus command is run before the FCV is initialized so we ignore it when // checking whether the resharding feature is enabled here. if (resharding::gFeatureFlagResharding.isEnabledAndIgnoreFCV()) { diff --git a/src/mongo/db/s/sharding_statistics.cpp b/src/mongo/db/s/sharding_statistics.cpp index 00b082aeb32..a32642776dd 100644 --- a/src/mongo/db/s/sharding_statistics.cpp +++ b/src/mongo/db/s/sharding_statistics.cpp @@ -34,7 +34,6 @@ #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/operation_context.h" #include "mongo/db/service_context.h" -#include "mongo/s/sharding_feature_flags_gen.h" namespace mongo { namespace { @@ -55,9 +54,6 @@ void ShardingStatistics::report(BSONObjBuilder* builder) const { builder->append("countStaleConfigErrors", countStaleConfigErrors.load()); builder->append("countDonorMoveChunkStarted", countDonorMoveChunkStarted.load()); - builder->append("countDonorMoveChunkCommitted", countDonorMoveChunkCommitted.load()); - builder->append("countDonorMoveChunkAborted", countDonorMoveChunkAborted.load()); - builder->append("totalDonorMoveChunkTimeMillis", totalDonorMoveChunkTimeMillis.load()); builder->append("totalDonorChunkCloneTimeMillis", totalDonorChunkCloneTimeMillis.load()); builder->append("totalCriticalSectionCommitTimeMillis", totalCriticalSectionCommitTimeMillis.load()); @@ -65,23 +61,14 @@ void ShardingStatistics::report(BSONObjBuilder* builder) const { builder->append("totalRecipientCriticalSectionTimeMillis", totalRecipientCriticalSectionTimeMillis.load()); builder->append("countDocsClonedOnRecipient", countDocsClonedOnRecipient.load()); - builder->append("countBytesClonedOnRecipient", countBytesClonedOnRecipient.load()); - builder->append("countDocsClonedOnCatchUpOnRecipient", - countDocsClonedOnCatchUpOnRecipient.load()); - builder->append("countBytesClonedOnCatchUpOnRecipient", - countBytesClonedOnCatchUpOnRecipient.load()); builder->append("countDocsClonedOnDonor", countDocsClonedOnDonor.load()); - builder->append("countBytesClonedOnDonor", countBytesClonedOnDonor.load()); builder->append("countRecipientMoveChunkStarted", countRecipientMoveChunkStarted.load()); - builder->append("countDocsDeletedByRangeDeleter", countDocsDeletedByRangeDeleter.load()); - builder->append("countBytesDeletedByRangeDeleter", countBytesDeletedByRangeDeleter.load()); + builder->append("countDocsDeletedOnDonor", countDocsDeletedOnDonor.load()); builder->append("countDonorMoveChunkLockTimeout", countDonorMoveChunkLockTimeout.load()); builder->append("countDonorMoveChunkAbortConflictingIndexOperation", countDonorMoveChunkAbortConflictingIndexOperation.load()); builder->append("unfinishedMigrationFromPreviousPrimary", unfinishedMigrationFromPreviousPrimary.load()); - if (mongo::feature_flags::gConcurrencyInChunkMigration.isEnabledAndIgnoreFCV()) - builder->append("chunkMigrationConcurrency", chunkMigrationConcurrencyCnt.load()); } } // namespace mongo diff --git a/src/mongo/db/s/sharding_statistics.h b/src/mongo/db/s/sharding_statistics.h index e5b86283906..b4332452a8d 100644 --- a/src/mongo/db/s/sharding_statistics.h +++ b/src/mongo/db/s/sharding_statistics.h @@ -49,12 +49,6 @@ struct ShardingStatistics { // (whether they succeeded or not). AtomicWord<long long> countDonorMoveChunkStarted{0}; - // Cumulative, always-increasing counter of how many chunks this node successfully committed. - AtomicWord<long long> countDonorMoveChunkCommitted{0}; - - // Cumulative, always-increasing counter of how many move chunks this node aborted. - AtomicWord<long long> countDonorMoveChunkAborted{0}; - // Cumulative, always-increasing counter of how much time the entire move chunk operation took // (excluding range deletion). AtomicWord<long long> totalDonorMoveChunkTimeMillis{0}; @@ -67,33 +61,13 @@ struct ShardingStatistics { // recipient node. AtomicWord<long long> countDocsClonedOnRecipient{0}; - // Cumulative, always-increasing counter of how many documents have been cloned on the catch up - // phase on the recipient node. - AtomicWord<long long> countDocsClonedOnCatchUpOnRecipient{0}; - - // Cumulative, always-increasing counter of how many bytes have been cloned on the catch up - // phase on the recipient node. - AtomicWord<long long> countBytesClonedOnCatchUpOnRecipient{0}; - - // Cumulative, always-increasing counter of how many bytes have been cloned on the - // recipient node. - AtomicWord<long long> countBytesClonedOnRecipient{0}; - // Cumulative, always-increasing counter of how many documents have been cloned on the donor // node. AtomicWord<long long> countDocsClonedOnDonor{0}; - // Cumulative, always-increasing counter of how many bytes have been cloned on the donor - // node. - AtomicWord<long long> countBytesClonedOnDonor{0}; - - // Cumulative, always-increasing counter of how many documents have been deleted by the - // rangeDeleter. - AtomicWord<long long> countDocsDeletedByRangeDeleter{0}; - - // Cumulative, always-increasing counter of how many bytes have been deleted by the - // rangeDeleter. - AtomicWord<long long> countBytesDeletedByRangeDeleter{0}; + // Cumulative, always-increasing counter of how many documents have been deleted on the donor + // node by the rangeDeleter. + AtomicWord<long long> countDocsDeletedOnDonor{0}; // Cumulative, always-increasing counter of how many chunks this node started to receive // (whether the receiving succeeded or not) @@ -130,9 +104,6 @@ struct ShardingStatistics { // completion. Valid only when this process is the repl set primary. AtomicWord<long long> unfinishedMigrationFromPreviousPrimary{0}; - // Current number for chunkMigrationConcurrency that defines concurrent fetchers and inserters - // used for _migrateClone(step 4) of chunk migration - AtomicWord<int> chunkMigrationConcurrencyCnt{1}; /** * Obtains the per-process instance of the sharding statistics object. */ diff --git a/src/mongo/db/s/sharding_util.cpp b/src/mongo/db/s/sharding_util.cpp index 7e82e09ebf5..9bd6c0ae795 100644 --- a/src/mongo/db/s/sharding_util.cpp +++ b/src/mongo/db/s/sharding_util.cpp @@ -95,13 +95,13 @@ std::vector<AsyncRequestsSender::Response> sendCommandToShards( "Failed command {} for database '{}' on shard '{}'"_format( command.toString(), dbName, StringData{response.shardId}); - uassertStatusOKWithContext(response.swResponse.getStatus(), errorContext); - const auto& respBody = response.swResponse.getValue().data; + auto shardResponse = + uassertStatusOKWithContext(std::move(response.swResponse), errorContext); - const auto status = getStatusFromCommandResult(respBody); + auto status = getStatusFromCommandResult(shardResponse.data); uassertStatusOKWithContext(status, errorContext); - const auto wcStatus = getWriteConcernStatusFromCommandResult(respBody); + auto wcStatus = getWriteConcernStatusFromCommandResult(shardResponse.data); uassertStatusOKWithContext(wcStatus, errorContext); } diff --git a/src/mongo/db/s/sharding_util.h b/src/mongo/db/s/sharding_util.h index c5021b4d46f..051b90b595d 100644 --- a/src/mongo/db/s/sharding_util.h +++ b/src/mongo/db/s/sharding_util.h @@ -50,8 +50,7 @@ void tellShardsToRefreshCollection(OperationContext* opCtx, const std::shared_ptr<executor::TaskExecutor>& executor); /** - * Generic utility to send a command to a list of shards. If `throwOnError=true`, throws in case one - * of the commands fails. + * Generic utility to send a command to a list of shards. Throws if one of the commands fails. */ std::vector<AsyncRequestsSender::Response> sendCommandToShards( OperationContext* opCtx, diff --git a/src/mongo/db/s/sharding_write_router.cpp b/src/mongo/db/s/sharding_write_router.cpp index b1c5c701ae0..87542f9c126 100644 --- a/src/mongo/db/s/sharding_write_router.cpp +++ b/src/mongo/db/s/sharding_write_router.cpp @@ -56,10 +56,6 @@ ShardingWriteRouter::ShardingWriteRouter(OperationContext* opCtx, _reshardingChunkMgr = uassertStatusOK(catalogCache->getCollectionRoutingInfo( opCtx, donorFields->getTempReshardingNss(), true /* allowLocks */)); - - tassert(6862800, - "Routing information for the temporary resharing collection is stale", - _reshardingChunkMgr->isSharded()); } } } diff --git a/src/mongo/db/s/shardsvr_abort_reshard_collection_command.cpp b/src/mongo/db/s/shardsvr_abort_reshard_collection_command.cpp index d22b5399040..40a884dd4ad 100644 --- a/src/mongo/db/s/shardsvr_abort_reshard_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_abort_reshard_collection_command.cpp @@ -66,7 +66,7 @@ public: std::vector<SharedSemiFuture<void>> futuresToWait; - if (auto machine = resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< + if (auto machine = resharding::tryGetReshardingStateMachine< ReshardingRecipientService, ReshardingRecipientService::RecipientStateMachine, ReshardingRecipientDocument>(opCtx, uuid())) { @@ -78,7 +78,7 @@ public: (*machine)->abort(isUserCanceled()); } - if (auto machine = resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< + if (auto machine = resharding::tryGetReshardingStateMachine< ReshardingDonorService, ReshardingDonorService::DonorStateMachine, ReshardingDonorDocument>(opCtx, uuid())) { diff --git a/src/mongo/db/s/shardsvr_collmod_participant_command.cpp b/src/mongo/db/s/shardsvr_collmod_participant_command.cpp index 693a8ff5ef3..7845a705185 100644 --- a/src/mongo/db/s/shardsvr_collmod_participant_command.cpp +++ b/src/mongo/db/s/shardsvr_collmod_participant_command.cpp @@ -89,10 +89,12 @@ public: timeseries::getTimeseriesOptions(opCtx, ns(), true)); auto bucketNs = ns().makeTimeseriesBucketsNamespace(); - { - // Clear the filtering metadata before releasing the critical section to prevent - // scenarios where a stepDown/stepUp will leave the node with wrong metadata. - // Cleanup on secondary nodes is performed by the release of the section. + try { + forceShardFilteringMetadataRefresh(opCtx, bucketNs); + } catch (const DBException&) { + // If the refresh fails, then set the shard version to UNKNOWN and let a future + // operation to refresh the metadata. + UninterruptibleLockGuard noInterrupt(opCtx->lockState()); AutoGetCollection autoColl(opCtx, bucketNs, MODE_IX); CollectionShardingRuntime::get(opCtx, bucketNs)->clearFilteringMetadata(opCtx); } diff --git a/src/mongo/db/s/shardsvr_commit_reshard_collection_command.cpp b/src/mongo/db/s/shardsvr_commit_reshard_collection_command.cpp index c72664d1f19..d059ab0dead 100644 --- a/src/mongo/db/s/shardsvr_commit_reshard_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_commit_reshard_collection_command.cpp @@ -67,13 +67,12 @@ public: std::vector<SharedSemiFuture<void>> futuresToWait; { - auto recipientMachine = - resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< - ReshardingRecipientService, - ReshardingRecipientService::RecipientStateMachine, - ReshardingRecipientDocument>(opCtx, uuid()); + auto recipientMachine = resharding::tryGetReshardingStateMachine< + ReshardingRecipientService, + ReshardingRecipientService::RecipientStateMachine, + ReshardingRecipientDocument>(opCtx, uuid()); - auto donorMachine = resharding::tryGetReshardingStateMachineAndThrowIfShuttingDown< + auto donorMachine = resharding::tryGetReshardingStateMachine< ReshardingDonorService, ReshardingDonorService::DonorStateMachine, ReshardingDonorDocument>(opCtx, uuid()); diff --git a/src/mongo/db/s/shardsvr_drop_collection_participant_command.cpp b/src/mongo/db/s/shardsvr_drop_collection_participant_command.cpp index dbfd5d1a3ef..a9b18cf3f61 100644 --- a/src/mongo/db/s/shardsvr_drop_collection_participant_command.cpp +++ b/src/mongo/db/s/shardsvr_drop_collection_participant_command.cpp @@ -75,8 +75,14 @@ public: opCtx->setAlwaysInterruptAtStepDownOrUp(); - bool fromMigrate = request().getFromMigrate().value_or(false); - DropCollectionCoordinator::dropCollectionLocally(opCtx, ns(), fromMigrate); + try { + DropCollectionCoordinator::dropCollectionLocally(opCtx, ns()); + } catch (const ExceptionFor<ErrorCodes::NamespaceNotFound>&) { + LOGV2_DEBUG(5280920, + 1, + "Namespace not found while trying to delete local collection", + "namespace"_attr = ns()); + } // The txnParticipant will only be missing when the command was sent from a coordinator // running an old 5.0.0 binary that didn't attach a sessionId & txnNumber. diff --git a/src/mongo/db/s/shardsvr_drop_indexes_command.cpp b/src/mongo/db/s/shardsvr_drop_indexes_command.cpp index b4527245b9d..cd1defb11f7 100644 --- a/src/mongo/db/s/shardsvr_drop_indexes_command.cpp +++ b/src/mongo/db/s/shardsvr_drop_indexes_command.cpp @@ -198,10 +198,8 @@ ShardsvrDropIndexesCommand::Invocation::Response ShardsvrDropIndexesCommand::Inv CommandHelpers::filterCommandRequestForPassthrough(cmdToBeSent)), ReadPreferenceSetting::get(opCtx), Shard::RetryPolicy::kNotIdempotent, - BSONObj() /*query*/, - BSONObj() /*collation*/, - boost::none /*letParameters*/, - boost::none /*runtimeConstants*/); + BSONObj() /* query */, + BSONObj() /* collation */); // Append responses we've received from previous retries of this operation due to a // stale config error. diff --git a/src/mongo/db/s/shardsvr_get_stats_for_balancing_command.cpp b/src/mongo/db/s/shardsvr_get_stats_for_balancing_command.cpp index 29810dfe420..512f0f4c721 100644 --- a/src/mongo/db/s/shardsvr_get_stats_for_balancing_command.cpp +++ b/src/mongo/db/s/shardsvr_get_stats_for_balancing_command.cpp @@ -33,10 +33,10 @@ #include "mongo/db/commands.h" #include "mongo/db/db_raii.h" #include "mongo/db/s/balancer_stats_registry.h" +#include "mongo/db/s/get_stats_for_balancing_gen.h" #include "mongo/db/s/sharding_state.h" #include "mongo/logv2/log.h" #include "mongo/s/grid.h" -#include "mongo/s/request_types/get_stats_for_balancing_gen.h" #include "mongo/s/sharding_feature_flags_gen.h" namespace mongo { diff --git a/src/mongo/db/s/shardsvr_move_range_command.cpp b/src/mongo/db/s/shardsvr_move_range_command.cpp index dab2d3ab073..df6d900aa3c 100644 --- a/src/mongo/db/s/shardsvr_move_range_command.cpp +++ b/src/mongo/db/s/shardsvr_move_range_command.cpp @@ -97,7 +97,7 @@ public: // Check if there is an existing migration running and if so, join it if (scopedMigration.mustExecute()) { auto moveChunkComplete = - ExecutorFuture<void>(Grid::get(opCtx)->getExecutorPool()->getFixedExecutor()) + ExecutorFuture<void>(_getExecutor()) .then([req = request(), writeConcern = opCtx->getWriteConcern(), scopedMigration = std::move(scopedMigration), @@ -215,13 +215,6 @@ public: opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}); }()); - long long totalDocsCloned = - ShardingStatistics::get(opCtx).countDocsClonedOnDonor.load(); - long long totalBytesCloned = - ShardingStatistics::get(opCtx).countBytesClonedOnDonor.load(); - long long totalCloneTime = - ShardingStatistics::get(opCtx).totalDonorChunkCloneTimeMillis.load(); - MigrationSourceManager migrationSourceManager( opCtx, std::move(request), std::move(writeConcern), donorConnStr, recipientHost); @@ -230,23 +223,28 @@ public: migrationSourceManager.enterCriticalSection(); migrationSourceManager.commitChunkOnRecipient(); migrationSourceManager.commitChunkMetadataOnConfig(); + } + + // Returns a single-threaded executor to be used to run moveChunk commands. The executor is + // initialized on the first call to this function. Uses a shared_ptr because a shared_ptr is + // required to work with ExecutorFutures. + static std::shared_ptr<ThreadPool> _getExecutor() { + static Mutex mutex = MONGO_MAKE_LATCH("MoveChunkExecutor::_mutex"); + static std::shared_ptr<ThreadPool> executor; + + stdx::lock_guard<Latch> lg(mutex); + if (!executor) { + ThreadPool::Options options; + options.poolName = "MoveChunk"; + options.minThreads = 0; + // We limit the size of the thread pool to a single thread because currently there + // can only be one moveRange operation on a shard at a time. + options.maxThreads = 1; + executor = std::make_shared<ThreadPool>(std::move(options)); + executor->startup(); + } - long long docsCloned = - ShardingStatistics::get(opCtx).countDocsClonedOnDonor.load() - totalDocsCloned; - long long bytesCloned = - ShardingStatistics::get(opCtx).countBytesClonedOnDonor.load() - totalBytesCloned; - long long cloneTime = - ShardingStatistics::get(opCtx).totalDonorChunkCloneTimeMillis.load() - - totalCloneTime; - auto migrationId = migrationSourceManager.getMigrationId(); - - LOGV2(7627801, - "Migration finished", - "migrationId"_attr = migrationId ? migrationId->toString() : "", - "totalTimeMillis"_attr = migrationSourceManager.getOpTimeMillis(), - "docsCloned"_attr = docsCloned, - "bytesCloned"_attr = bytesCloned, - "cloneTime"_attr = cloneTime); + return executor; } }; diff --git a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp index ed1e6b9c375..7a7fa8fceac 100644 --- a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp +++ b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp @@ -94,13 +94,7 @@ public: return true; } - void doCheckAuthorization(OperationContext* opCtx) const override { - uassert(ErrorCodes::Unauthorized, - "Unauthorized", - AuthorizationSession::get(opCtx->getClient()) - ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(), - ActionType::internal)); - } + void doCheckAuthorization(OperationContext*) const override {} /** * The ns() for when Request's IDL specifies "namespace: concatenate_with_db". diff --git a/src/mongo/db/s/shardsvr_rename_collection_command.cpp b/src/mongo/db/s/shardsvr_rename_collection_command.cpp index 97c129a029d..5a1918046f4 100644 --- a/src/mongo/db/s/shardsvr_rename_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_rename_collection_command.cpp @@ -86,15 +86,16 @@ public: CommandHelpers::uassertCommandRunWithMajority(Request::kCommandName, opCtx->getWriteConcern()); + if (fromNss.db() != toNss.db()) { + sharding_ddl_util::checkDbPrimariesOnTheSameShard(opCtx, fromNss, toNss); + } + validateNamespacesForRenameCollection(opCtx, fromNss, toNss); auto coordinatorDoc = RenameCollectionCoordinatorDocument(); coordinatorDoc.setRenameCollectionRequest(req.getRenameCollectionRequest()); coordinatorDoc.setShardingDDLCoordinatorMetadata( {{fromNss, DDLCoordinatorTypeEnum::kRenameCollection}}); - coordinatorDoc.setAllowEncryptedCollectionRename( - req.getAllowEncryptedCollectionRename().value_or(false)); - auto service = ShardingDDLCoordinatorService::getService(opCtx); auto renameCollectionCoordinator = checked_pointer_cast<RenameCollectionCoordinator>( service->getOrCreateInstance(opCtx, coordinatorDoc.toBSON())); diff --git a/src/mongo/db/s/shardsvr_rename_collection_participant_command.cpp b/src/mongo/db/s/shardsvr_rename_collection_participant_command.cpp index a1de64b2d69..23a96214a64 100644 --- a/src/mongo/db/s/shardsvr_rename_collection_participant_command.cpp +++ b/src/mongo/db/s/shardsvr_rename_collection_participant_command.cpp @@ -176,7 +176,7 @@ public: const auto service = RenameCollectionParticipantService::getService(opCtx); const auto id = BSON("_id" << fromNss.ns()); - const auto [optRenameCollectionParticipant, _] = + const auto optRenameCollectionParticipant = RenameParticipantInstance::lookup(opCtx, service, id); if (optRenameCollectionParticipant) { uassert(ErrorCodes::CommandFailed, diff --git a/src/mongo/db/s/shardsvr_reshard_collection_command.cpp b/src/mongo/db/s/shardsvr_reshard_collection_command.cpp index 78d9ecda673..5a10de6a559 100644 --- a/src/mongo/db/s/shardsvr_reshard_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_reshard_collection_command.cpp @@ -69,7 +69,6 @@ public: using InvocationBase::InvocationBase; void typedRun(OperationContext* opCtx) { - opCtx->setAlwaysInterruptAtStepDownOrUp(); uassertStatusOK(ShardingState::get(opCtx)->canAcceptShardedCommands()); CommandHelpers::uassertCommandRunWithMajority(Request::kCommandName, diff --git a/src/mongo/db/s/split_vector.cpp b/src/mongo/db/s/split_vector.cpp index 8b232db42a5..276b91951e7 100644 --- a/src/mongo/db/s/split_vector.cpp +++ b/src/mongo/db/s/split_vector.cpp @@ -57,16 +57,6 @@ BSONObj prettyKey(const BSONObj& keyPattern, const BSONObj& key) { return key.replaceFieldNames(keyPattern).clientReadable(); } -/* - * Reshuffle fields according to the shard key pattern. - */ -auto orderShardKeyFields(const BSONObj& keyPattern, const BSONObj& key) { - // Note: It is correct to hydrate the indexKey 'key' with 'keyPattern', because the index key - // pattern is a prefix of 'keyPattern'. - return dotted_path_support::extractElementsBasedOnTemplate(key.replaceFieldNames(keyPattern), - keyPattern); -} - } // namespace std::vector<BSONObj> splitVector(OperationContext* opCtx, @@ -232,14 +222,17 @@ std::vector<BSONObj> splitVector(OperationContext* opCtx, // to be removed at the end. If a key appears more times than entries allowed on a // chunk, we issue a warning and split on the following key. auto tooFrequentKeys = SimpleBSONObjComparator::kInstance.makeBSONObjSet(); - splitKeys.push_back(orderShardKeyFields(keyPattern, currKey.getOwned())); + splitKeys.push_back(dotted_path_support::extractElementsBasedOnTemplate( + prettyKey(shardKeyIdx->keyPattern(), currKey.getOwned()), keyPattern)); while (1) { while (PlanExecutor::ADVANCED == state) { currCount++; if (currCount > keyCount && !force) { - currKey = orderShardKeyFields(keyPattern, currKey.getOwned()); + currKey = dotted_path_support::extractElementsBasedOnTemplate( + prettyKey(shardKeyIdx->keyPattern(), currKey.getOwned()), keyPattern); + const auto compareWithPreviousSplitPoint = currKey.woCompare(splitKeys.back()); dassert(compareWithPreviousSplitPoint >= 0, diff --git a/src/mongo/db/s/split_vector_command.cpp b/src/mongo/db/s/split_vector_command.cpp index 2fc9963b133..b485c08361c 100644 --- a/src/mongo/db/s/split_vector_command.cpp +++ b/src/mongo/db/s/split_vector_command.cpp @@ -137,16 +137,17 @@ public: // Prevent maxChunkSizeBytes overflow. Check aimed to avoid fuzzer failures // since users are definitely not expected to specify maxChunkSize in exabytes. uassert(ErrorCodes::InvalidOptions, - str::stream() << "maxChunkSize must lie within the range [1MB, 1024MB]", - maxChunkSizeMB >= 1 && maxChunkSizeMB <= 1024); + str::stream() + << "The specified maxChunkSize in MB is too big: " << maxChunkSizeMB, + maxChunkSizeMB <= (LLONG_MAX >> 20)); ret = maxChunkSizeMB << 20; } else if (maxSizeBytesElem.isNumber()) { ret = maxSizeBytesElem.safeNumberLong(); - uassert(ErrorCodes::InvalidOptions, - "The specified max chunk size must lie within the range [1MB, 1024MB]", - *ret >= 1024 * 1024 && *ret <= 1024 * 1024 * 1024); } + uassert(ErrorCodes::InvalidOptions, + "The specified max chunk size must be at least 1MB", + ret == boost::none || *ret >= 1024 * 1024); return ret; }(); diff --git a/src/mongo/db/s/start_chunk_clone_request.cpp b/src/mongo/db/s/start_chunk_clone_request.cpp index 6838aedaffc..86567dce161 100644 --- a/src/mongo/db/s/start_chunk_clone_request.cpp +++ b/src/mongo/db/s/start_chunk_clone_request.cpp @@ -50,7 +50,6 @@ const char kToShardId[] = "toShardName"; const char kChunkMinKey[] = "min"; const char kChunkMaxKey[] = "max"; const char kShardKeyPattern[] = "shardKeyPattern"; -const char kParallelMigration[] = "parallelMigrateCloneSupported"; } // namespace @@ -153,14 +152,6 @@ StatusWith<StartChunkCloneRequest> StartChunkCloneRequest::createFromCommand(Nam } } - { - Status status = bsonExtractBooleanFieldWithDefault( - obj, kParallelMigration, false, &request._parallelFetchingSupported); - if (!status.isOK()) { - return status; - } - } - request._migrationId = UUID::parse(obj); request._lsid = LogicalSessionId::parse(IDLParserErrorContext("StartChunkCloneRequest"), obj[kLsid].Obj()); @@ -188,7 +179,6 @@ void StartChunkCloneRequest::appendAsCommand( invariant(fromShardConnectionString.isValid()); builder->append(kRecvChunkStart, nss.ns()); - builder->append(kParallelMigration, true); migrationId.appendToBuilder(builder, kMigrationId); builder->append(kLsid, lsid.toBSON()); diff --git a/src/mongo/db/s/start_chunk_clone_request.h b/src/mongo/db/s/start_chunk_clone_request.h index 8f433afd2f7..c6ecba1f839 100644 --- a/src/mongo/db/s/start_chunk_clone_request.h +++ b/src/mongo/db/s/start_chunk_clone_request.h @@ -93,10 +93,6 @@ public: return _migrationId.is_initialized(); } - bool parallelFetchingSupported() const { - return _parallelFetchingSupported; - } - const UUID& getMigrationId() const { invariant(_migrationId); return *_migrationId; @@ -165,8 +161,6 @@ private: // The parsed secondary throttle options MigrationSecondaryThrottleOptions _secondaryThrottle; - - bool _parallelFetchingSupported; }; } // namespace mongo diff --git a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp index 8f6686973f6..d3e7e2818e1 100644 --- a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp +++ b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp @@ -299,9 +299,7 @@ protected: StaticCatalogClient() = default; StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { std::vector<ShardType> shardTypes; for (const auto& shardId : makeThreeShardIdsList()) { const ConnectionString cs = ConnectionString::forReplicaSet( diff --git a/src/mongo/db/s/transaction_coordinator_service.cpp b/src/mongo/db/s/transaction_coordinator_service.cpp index 009f63c4bef..8dc5cf0a3cd 100644 --- a/src/mongo/db/s/transaction_coordinator_service.cpp +++ b/src/mongo/db/s/transaction_coordinator_service.cpp @@ -33,7 +33,6 @@ #include "mongo/db/s/transaction_coordinator_service.h" -#include "mongo/db/concurrency/lock_state.h" #include "mongo/db/internal_transactions_feature_flag_gen.h" #include "mongo/db/repl/repl_client_info.h" #include "mongo/db/s/transaction_coordinator_document_gen.h" @@ -47,8 +46,6 @@ namespace mongo { namespace { -MONGO_FAIL_POINT_DEFINE(hangBeforeTxnCoordinatorOnStepUpWork); - const auto transactionCoordinatorServiceDecoration = ServiceContext::declareDecoration<TransactionCoordinatorService>(); @@ -199,15 +196,6 @@ void TransactionCoordinatorService::onStepUp(OperationContext* opCtx, .scheduleWorkIn( recoveryDelayForTesting, [catalogAndScheduler = _catalogAndScheduler](OperationContext* opCtx) { - if (MONGO_unlikely(hangBeforeTxnCoordinatorOnStepUpWork.shouldFail())) { - LOGV2(8288301, "Hit hangBeforeTxnCoordinatorOnStepUpWork failpoint"); - hangBeforeTxnCoordinatorOnStepUpWork.pauseWhileSet(opCtx); - } - - // Skip ticket acquisition in order to prevent possible deadlock when - // participants are in the prepared state. See SERVER-82883 and SERVER-60682. - SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx); - auto& replClientInfo = repl::ReplClientInfo::forClient(opCtx->getClient()); replClientInfo.setLastOpToSystemLastOpTime(opCtx); @@ -389,10 +377,6 @@ TransactionCoordinatorService::getAllRemovalFuturesForCoordinatorsForInternalTra std::shared_ptr<CatalogAndScheduler> cas = _getCatalogAndScheduler(opCtx); auto& catalog = cas->catalog; - // On step up, we want to wait until the catalog has recovered all active transaction - // coordinators before getting the removal futures. - cas->recoveryTaskCompleted->get(opCtx); - auto predicate = [](const LogicalSessionId lsid, const TxnNumberAndRetryCounter txnNumberAndRetryCounter, const std::shared_ptr<TransactionCoordinator> transactionCoordinator) { diff --git a/src/mongo/db/s/transaction_coordinator_test_fixture.cpp b/src/mongo/db/s/transaction_coordinator_test_fixture.cpp index a914d05f33a..023e7deb4b4 100644 --- a/src/mongo/db/s/transaction_coordinator_test_fixture.cpp +++ b/src/mongo/db/s/transaction_coordinator_test_fixture.cpp @@ -81,9 +81,7 @@ TransactionCoordinatorTestFixture::makeShardingCatalogClient() { StaticCatalogClient(std::vector<ShardId> shardIds) : _shardIds(std::move(shardIds)) {} StatusWith<repl::OpTimeWith<std::vector<ShardType>>> getAllShards( - OperationContext* opCtx, - repl::ReadConcernLevel readConcern, - bool excludeDraining) override { + OperationContext* opCtx, repl::ReadConcernLevel readConcern) override { std::vector<ShardType> shardTypes; for (const auto& shardId : _shardIds) { const ConnectionString cs = diff --git a/src/mongo/db/s/transaction_coordinator_util.cpp b/src/mongo/db/s/transaction_coordinator_util.cpp index 7ddc8151f25..4328b7ee47f 100644 --- a/src/mongo/db/s/transaction_coordinator_util.cpp +++ b/src/mongo/db/s/transaction_coordinator_util.cpp @@ -204,10 +204,6 @@ Future<repl::OpTime> persistParticipantsList( return scheduler.scheduleWork( [lsid, txnNumberAndRetryCounter, participants](OperationContext* opCtx) { FlowControl::Bypass flowControlBypass(opCtx); - // This code is used by the TransactionCoordinator. As a result, we need to skip - // ticket acquisition in order to prevent possible deadlock when participants - // are in the prepared state. See SERVER-82883 and SERVER-60682. - SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx); getTransactionCoordinatorWorkerCurOpRepository()->set( opCtx, lsid, diff --git a/src/mongo/db/s/user_writes_recoverable_critical_section_service.cpp b/src/mongo/db/s/user_writes_recoverable_critical_section_service.cpp index 39b522dfde1..6e15c008f19 100644 --- a/src/mongo/db/s/user_writes_recoverable_critical_section_service.cpp +++ b/src/mongo/db/s/user_writes_recoverable_critical_section_service.cpp @@ -129,9 +129,6 @@ void acquireRecoverableCriticalSection(OperationContext* opCtx, 3, "The user writes recoverable critical section was already acquired", "namespace"_attr = nss); - - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return; } @@ -241,9 +238,6 @@ void UserWritesRecoverableCriticalSectionService:: "block user " "writes, do nothing", "namespace"_attr = nss); - - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return; } @@ -284,9 +278,6 @@ void UserWritesRecoverableCriticalSectionService:: 3, "The user writes recoverable critical section was not currently taken, do nothing", "namespace"_attr = nss); - - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return; } @@ -300,9 +291,6 @@ void UserWritesRecoverableCriticalSectionService:: "The user writes recoverable critical section was already not blocking " "user writes, do nothing", "namespace"_attr = nss); - - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return; } @@ -338,9 +326,6 @@ void UserWritesRecoverableCriticalSectionService::releaseRecoverableCriticalSect 3, "The user writes recoverable critical section was already released, do nothing", "namespace"_attr = nss); - - repl::ReplClientInfo::forClient(opCtx->getClient()).setLastOpToSystemLastOpTime(opCtx); - return; } |