diff options
4 files changed, 169 insertions, 0 deletions
diff --git a/buildscripts/resmokeconfig/suites/aggregation_unsplittable_collections_on_random_shard_passthrough_with_config_transitions.yml b/buildscripts/resmokeconfig/suites/aggregation_unsplittable_collections_on_random_shard_passthrough_with_config_transitions.yml new file mode 100644 index 00000000000..4208a0ff4e9 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/aggregation_unsplittable_collections_on_random_shard_passthrough_with_config_transitions.yml @@ -0,0 +1,147 @@ +test_kind: js_test + +selector: + roots: + - jstests/aggregation/**/*.js + - jstests/libs/random_data_shard_suite_selftest.js + exclude_files: + - jstests/aggregation/extras/*.js + - jstests/aggregation/data/*.js + # TODO SERVER-32311: These tests use getAggPlanStage(), which can't handle sharded explain output. + - jstests/aggregation/match_swapping_renamed_fields.js + - jstests/aggregation/use_query_project_and_sort.js + - jstests/aggregation/use_query_projection.js + - jstests/aggregation/use_query_sort.js + + # These tests are written such that, after we check the engine being used and the colocation of + # the involved collections, we drop and recreate our collections for each test case. Given that + # this suite creates unsplittable collections on random shards and we cannot reliably check the + # colocation of collections in these tests, this means that these tests are incompatible with + # this suite as written. + - jstests/aggregation/sources/lookup/lookup_equijoin_semantics_inlj.js + - jstests/aggregation/sources/lookup/lookup_equijoin_semantics_nlj.js + - jstests/aggregation/sources/lookup/lookup_equijoin_semantics_hj.js + + # Includes expectations on failpoints being hit on specific shards. + - jstests/aggregation/sources/indexStats/verify_index_stats_output.js + + # Runs getMores which can be interrupted by a config transition. + - jstests/aggregation/expressions/in.js + - jstests/aggregation/sources/densify/densify_sort_opt_comparison.js + - jstests/aggregation/bugs/server21632.js + - jstests/aggregation/bugs/server6240.js + - jstests/aggregation/sources/merge/mode_replace_fail.js + - jstests/aggregation/sources/collStats/query_exec_stats.js + - jstests/aggregation/sources/out/replace_collection.js + - jstests/aggregation/expressions/date_from_parts.js + - jstests/aggregation/sources/merge/batch_writes.js + - jstests/aggregation/expressions/expression_trigonometric.js + - jstests/aggregation/bugs/reverseArray.js + - jstests/aggregation/expressions/set.js + - jstests/aggregation/expressions/filter.js + - jstests/aggregation/expressions/trim.js + - jstests/aggregation/sources/merge/mode_merge_fail.js + - jstests/aggregation/expressions/regex_limits.js + - jstests/aggregation/bugs/server3253.js + - jstests/aggregation/bugs/server20163.js + - jstests/aggregation/sources/merge/batch_writes.js + - jstests/aggregation/expressions/arrayToObject.js + - jstests/aggregation/ifnull.js + - jstests/aggregation/sources/facet/facet_memory_consumption.js + - jstests/aggregation/bugs/server14670.js + - jstests/aggregation/bugs/server5932.js + - jstests/aggregation/expressions/binarySize.js + - jstests/aggregation/expressions/divide.js + - jstests/aggregation/sources/merge/mode_fail_insert.js + - jstests/aggregation/bugs/server7781.js + - jstests/aggregation/bugs/server10176.js + - jstests/aggregation/expressions/unary_numeric.js + - jstests/aggregation/bugs/server11118.js + - jstests/aggregation/expressions/map.js + - jstests/aggregation/expressions/date_to_parts.js + - jstests/aggregation/sources/geonear/requires_geo_index.js + - jstests/aggregation/expressions/expression_function.js + - jstests/aggregation/bugs/server6570.js + - jstests/aggregation/expressions/merge_objects.js + - jstests/aggregation/bugs/server6239.js + - jstests/aggregation/expressions/multiply.js + - jstests/aggregation/sources/merge/mode_replace_insert.js + - jstests/aggregation/expressions/regex.js + - jstests/aggregation/expressions/objectToArray.js + - jstests/aggregation/bugs/server14872.js + - jstests/aggregation/aggregation_with_uuids.js + - jstests/aggregation/sources/collStats/count.js + - jstests/aggregation/expressions/size.js + - jstests/aggregation/sources/graphLookup/error.js + - jstests/aggregation/sources/match/text_search_requires_index.js + - jstests/aggregation/sources/lookup/lookup_contains_text.js + - jstests/aggregation/expressions/switch_errors.js + - jstests/aggregation/spill_to_disk.js + - jstests/aggregation/expressions/internal_js_emit.js + - jstests/aggregation/sources/unionWith/unionWith.js + - jstests/aggregation/sources/merge/on_fields_validation.js + - jstests/aggregation/bugs/server10530.js + - jstests/aggregation/range.js + + # These tests run larger aggregations that can take a long time and can repeatedly get interrupted by config transitions. + - jstests/aggregation/query_limits_test.js + - jstests/aggregation/accumulators/accumulator_js_size_limits.js + + # TODO SERVER-90619 Investigate missing DISTINCT_SCAN in explain output in aggregation config transition suite. + - jstests/aggregation/group_conversion_to_distinct_scan.js + + exclude_with_any_tags: + - assumes_against_mongod_not_mongos + - requires_profiling + + # The following tags are excluded specifically for this suite. + - config_shard_incompatible + # Currently this passthrough enables the balancer to allow the config transition to successfully complete. + - assumes_balancer_off + +executor: + archive: + hooks: + - CheckReplDBHash + - CheckMetadataConsistencyInBackground + - ValidateCollections + config: + shell_options: + global_vars: + TestData: + runningWithBalancer: true + transitioningConfigShard: true + eval: + await import("jstests/libs/override_methods/implicitly_retry_on_migration_in_progress.js"); + await import("jstests/libs/override_methods/implicitly_retry_on_config_transition_errors.js"); + hooks: + - class: ContinuousConfigShardTransition + - class: CheckReplDBHash + - class: CheckMetadataConsistencyInBackground + shell_options: + global_vars: + TestData: + transitioningConfigShard: true + - class: ValidateCollections + - class: CheckOrphansDeleted + - class: CleanEveryN + n: 20 + fixture: + class: ShardedClusterFixture + config_shard: "any" + num_shards: 3 + mongos_options: + set_parameters: + enableTestCommands: 1 + mongod_options: + set_parameters: + enableTestCommands: 1 + skipDroppingHashedShardKeyIndex: true + # Manually set the random migration fail point so we can set a threshold for skipping + # moveCollection to prevent "starving" random chunk migrations. + failpoint.balancerShouldReturnRandomMigrations: + data: + skipMoveCollectionThreshold: 0.5 + mode: alwaysOn + balancerMigrationsThrottlingMs: 100 + enable_balancer: true diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough_with_config_transitions.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough_with_config_transitions.yml index f75cdc6a4f1..205b7066c91 100644 --- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough_with_config_transitions.yml +++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough_with_config_transitions.yml @@ -64,6 +64,8 @@ selector: - assumes_balancer_off # MoveCollection changes the collection uuid. - assumes_stable_collection_uuid + # MoveCollection run by the config transition can kill open cursors. + - requires_getmore executor: archive: diff --git a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml index 56ae9ece7dd..39af4011aa6 100644 --- a/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml +++ b/etc/evergreen_yml_components/tasks/resmoke/server_divisions/clusters_and_integrations/tasks.yml @@ -1520,6 +1520,15 @@ tasks: - func: "run tests" - <<: *gen_task_template + name: aggregation_unsplittable_collections_on_random_shard_passthrough_with_config_transitions + tags: ["assigned_to_jira_team_server_catalog_and_routing", "default", "unsplittable_collections"] + depends_on: + - name: aggregation + commands: + - func: "do setup" + - func: "run tests" + +- <<: *gen_task_template name: causally_consistent_jscore_passthrough_auth_gen tags: ["assigned_to_jira_team_server_catalog_and_routing", "default", "causally_consistent"] commands: diff --git a/jstests/libs/override_methods/implicitly_retry_on_config_transition_errors.js b/jstests/libs/override_methods/implicitly_retry_on_config_transition_errors.js index 9d06e89ab43..e80a9c3cf07 100644 --- a/jstests/libs/override_methods/implicitly_retry_on_config_transition_errors.js +++ b/jstests/libs/override_methods/implicitly_retry_on_config_transition_errors.js @@ -2,6 +2,7 @@ * Overrides runCommand so operations that encounter errors from a config shard transitioning in and * out of dedicated mode retry. */ +import {getCommandName} from "jstests/libs/cmd_object_utils.js"; import {OverrideHelpers} from "jstests/libs/override_methods/override_helpers.js"; const kTimeout = 20 * 60 * 1000; @@ -19,6 +20,10 @@ const kRetryableErrors = [ code: ErrorCodes.ConflictingOperationInProgress, errmsg: "Another ConfigsvrCoordinator with different arguments is already running" }, + // A query can be killed if it is still selecting a query plan after a config transition has + // completed range deletion and drops the collection. Since orphanCleanUpDelaySecs is set to be + // lower in testing than in production, dropCollection is scheduled almost immediately. + {code: ErrorCodes.QueryPlanKilled}, // TODO SERVER-90609: Stop ignoring this error. Currently an index build may fail because a // concurrent movePrimary triggered by the transition hook drops the collection. {code: ErrorCodes.IndexBuildAborted}, @@ -52,6 +57,12 @@ function shouldRetry(cmdObj, res) { return false; } + // getMore errors cannot be retried since a client may not know if previous getMore advanced the + // cursor. + if (getCommandName(cmdObj) === "getMore") { + return false; + } + if (isRetryableError(res)) { return true; } |