diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-18 17:02:53 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-18 17:02:53 -0300 |
| commit | 959575a5ca598bf5f37fb5cebe7ed1d80d3d71f7 (patch) | |
| tree | acc8d60aedb12b70048e676e8a7349deb0010db8 /etc | |
| parent | 76588293975fc059cf076779e4283e6ffaf8afff (diff) | |
New upstream version 6.0.20upstream
Diffstat (limited to 'etc')
| -rw-r--r-- | etc/backports_required_for_multiversion_tests.yml | 114 | ||||
| -rw-r--r-- | etc/evergreen.yml | 383 | ||||
| -rw-r--r-- | etc/evergreen_nightly.yml | 48 | ||||
| -rw-r--r-- | etc/evergreen_timeouts.yml | 34 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 333 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/tasks/misc_tasks.yml | 1611 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/ibm.yml | 8 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/in_memory.yml | 5 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/misc_release.yml | 315 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/ninja.yml | 10 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/openssl_111.yml | 173 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/sanitizer.yml | 17 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/task_generation.yml | 6 | ||||
| -rw-r--r-- | etc/generate_subtasks_config.yml | 14 | ||||
| -rw-r--r-- | etc/lsan.suppressions | 7 | ||||
| -rw-r--r-- | etc/pip/components/jiraclient.req | 2 | ||||
| -rw-r--r-- | etc/pip/components/testing.req | 3 | ||||
| -rwxr-xr-x | etc/system_perf.yml | 1317 |
18 files changed, 2445 insertions, 1955 deletions
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index b44972fdfd4..419768cc676 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -21,16 +21,30 @@ # last-continuous: all: + - test_file: jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js + ticket: SERVER-88885 + - test_file: jstests/replsets/ignore_dbcheck_in_initial_sync.js + ticket: SERVER-80340 + - test_file: jstests/replsets/ignore_dbcheck_in_startup_recovery.js + ticket: SERVER-30832 + - test_file: jstests/replsets/ignore_dbcheck_in_rollback.js + ticket: SERVER-30832 + - test_file: jstests/replsets/dbcheck_fails_on_write_concern_error.js + ticket: SERVER-88318 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_pauses_watchdog_checks_during_storage_change.js ticket: SERVER-83508 - test_file: jstests/sharding/query/find_and_modify_with_partial_shard_key.js ticket: SERVER-44422 - test_file: jstests/sharding/query/delete_with_partial_shard_key.js ticket: SERVER-44422 + - test_file: jstests/sharding/secondaries_clear_filtering_metadata.js + ticket: SERVER-86669 - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js ticket: SERVER-51943 - test_file: jstests/sharding/query/collation_shard_targeting_hashed_shard_key.js ticket: SERVER-53335 + - test_file: jstests/core/elemmatch_index.js + ticket: SERVER-90747 - test_file: jstests/sharding/cwwc_conflict_add_shard.js ticket: SERVER-56800 - test_file: jstests/sharding/reconfig_fails_no_cwwc_set_sharding.js @@ -317,7 +331,11 @@ last-continuous: ticket: SERVER-84013 - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js ticket: SERVER-82353 - - test_file: jstests/core/notablescan.js + - test_file: jstests/core/timeseries/timeseries_collmod.js + ticket: SERVER-83370 + - test_file: jstests/core/timeseries/timeseries_lookup.js + ticket: SERVER-79637 + - test_file: jstests/noPassthrough/notablescan.js ticket: SERVER-83119 - test_file: jstests/sharding/clustered_coll_scan.js ticket: SERVER-83119 @@ -327,11 +345,57 @@ last-continuous: ticket: SERVER-72703 - test_file: jstests/core/query/bittest.js ticket: SERVER-86419 + - test_file: jstests/core/command_let_variables_expressions.js + ticket: SERVER-89268 - test_file: jstests/core/query/internal_strip_invalid_assignment.js ticket: SERVER-82717 + - test_file: jstests/core/timeseries/timeseries_project.js + ticket: SERVER-87987 + - test_file: jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js + ticket: SERVER-79049 + - test_file: jstests/noPassthrough/server_status_metrics_retryable_writes.js + ticket: SERVER-86253 + - test_file: jstests/replsets/reconstruct_prepared_transactions_initial_sync.js + ticket: SERVER-89618 + - test_file: jstests/core/timeseries/timeseries_geonear_lookup.js + ticket: SERVER-81454 + - test_file: jstests/core/timeseries/timeseries_computed_field.js + ticket: SERVER-88072 + - test_file: jstests/core/bypass_empty_ts_replacement.js + ticket: SERVER-88750 + - test_file: jstests/core/bypass_empty_ts_replacement_timeseries.js + ticket: SERVER-88750 + - test_file: jstests/aggregation/unwind_sort.js + ticket: SERVER-90485 + - test_file: jstests/aggregation/group_by_objectid.js + ticket: SERVER-90173 + - test_file: jstests/aggregation/sources/redact/root_redact.js + ticket: SERVER-91339 + - test_file: jstests/replsets/catchup_ignores_old_heartbeats.js + ticket: SERVER-86674 + - test_file: jstests/core/wildcard_index_validindex.js + ticket: SERVER-93105 + - test_file: jstests/sharding/refresh_sessions.js + ticket: SERVER-94635 + - test_file: jstests/change_streams/report_post_batch_resume_token.js + ticket: SERVER-92904 + - test_file: jstests/core/negated_geo_queries.js + ticket: SERVER-92193 + - test_file: jstests/core/query/find_all_values_at_path_expression.js + ticket: SERVER-95670 suites: null last-lts: all: + - test_file: jstests/replsets/dbcheck_skip_applying_batch_on_secondary_parameter.js + ticket: SERVER-88885 + - test_file: jstests/replsets/ignore_dbcheck_in_initial_sync.js + ticket: SERVER-80340 + - test_file: jstests/replsets/dbcheck_fails_on_write_concern_error.js + ticket: SERVER-88318 + - test_file: jstests/replsets/ignore_dbcheck_in_startup_recovery.js + ticket: SERVER-30832 + - test_file: jstests/replsets/ignore_dbcheck_in_rollback.js + ticket: SERVER-30832 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_pauses_watchdog_checks_during_storage_change.js ticket: SERVER-83508 - test_file: jstests/core/query/null_query_semantics.js @@ -392,6 +456,8 @@ last-lts: ticket: SERVER-51259 - test_file: jstests/sharding/mongos_quiesce_mode.js ticket: SERVER-51259 + - test_file: jstests/sharding/secondaries_clear_filtering_metadata.js + ticket: SERVER-86669 - test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js ticket: SERVER-51943 - test_file: jstests/core/txns/errors_on_committed_transaction.js @@ -412,6 +478,8 @@ last-lts: ticket: SERVER-56518 - test_file: jstests/concurrency/fsm_workloads/collmod_writeconflict.js ticket: SERVER-56772 + - test_file: jstests/core/elemmatch_index.js + ticket: SERVER-90747 - test_file: jstests/sharding/cwwc_conflict_add_shard.js ticket: SERVER-56800 - test_file: jstests/sharding/database_versioning_all_commands.js @@ -692,7 +760,11 @@ last-lts: ticket: SERVER-84013 - test_file: jstests/sharding/multi_collection_transaction_placement_conflict_workaround.js ticket: SERVER-82353 - - test_file: jstests/core/notablescan.js + - test_file: jstests/core/timeseries/timeseries_collmod.js + ticket: SERVER-83370 + - test_file: jstests/core/timeseries/timeseries_lookup.js + ticket: SERVER-79637 + - test_file: jstests/noPassthrough/notablescan.js ticket: SERVER-83119 - test_file: jstests/sharding/clustered_coll_scan.js ticket: SERVER-83119 @@ -704,4 +776,42 @@ last-lts: ticket: SERVER-86419 - test_file: jstests/core/query/internal_strip_invalid_assignment.js ticket: SERVER-82717 + - test_file: jstests/core/timeseries/timeseries_project.js + ticket: SERVER-87987 + - test_file: jstests/core/command_let_variables_expressions.js + ticket: SERVER-89268 + - test_file: jstests/sharding/swallow_unnecessary_uuid_mismatch_error.js + ticket: SERVER-79049 + - test_file: jstests/noPassthrough/server_status_metrics_retryable_writes.js + ticket: SERVER-86253 + - test_file: jstests/replsets/log_wt_stats_during_secondary_oplog_application.js + ticket: SERVER-78509 + - test_file: jstests/replsets/reconstruct_prepared_transactions_initial_sync.js + ticket: SERVER-89618 + - test_file: jstests/core/timeseries/timeseries_geonear_lookup.js + ticket: SERVER-81454 + - test_file: jstests/core/timeseries/timeseries_computed_field.js + ticket: SERVER-88072 + - test_file: jstests/core/bypass_empty_ts_replacement.js + ticket: SERVER-88750 + - test_file: jstests/core/bypass_empty_ts_replacement_timeseries.js + ticket: SERVER-88750 + - test_file: jstests/aggregation/unwind_sort.js + ticket: SERVER-90485 + - test_file: jstests/aggregation/group_by_objectid.js + ticket: SERVER-90173 + - test_file: jstests/aggregation/sources/redact/root_redact.js + ticket: SERVER-91339 + - test_file: jstests/replsets/catchup_ignores_old_heartbeats.js + ticket: SERVER-86674 + - test_file: jstests/core/wildcard_index_validindex.js + ticket: SERVER-93105 + - test_file: jstests/sharding/refresh_sessions.js + ticket: SERVER-94635 + - test_file: jstests/change_streams/report_post_batch_resume_token.js + ticket: SERVER-92904 + - test_file: jstests/core/negated_geo_queries.js + ticket: SERVER-92193 + - test_file: jstests/core/query/find_all_values_at_path_expression.js + ticket: SERVER-95670 suites: null diff --git a/etc/evergreen.yml b/etc/evergreen.yml index e8f15610eea..0d87fb6a44e 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -177,6 +177,10 @@ variables: - name: libunwind_tests - name: .logical_session_cache .one_sec - name: .ocsp + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough - name: .replica_sets .multi_oplog @@ -298,6 +302,7 @@ buildvariants: expansions: &linux-64-required-duroff-expansions compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --enable-http-client=off --link-model=dynamic --use-diagnostic-latches=on --modules= multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: targeted # Running WiredTiger with --nojournal in a replica set is no longer supported, so this variant # does not include replica set tests. Since transactions are only supported on replica sets, we @@ -341,13 +346,13 @@ buildvariants: tasks: - name: tla_plus -- &enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-dynamic-v4gcc-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC DEBUG" +- &enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-dynamic-v4gcc-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC DEBUG" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small - expansions: &enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + expansions: &enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --use-diagnostic-latches=on has_packages: false @@ -355,7 +360,7 @@ buildvariants: scons_cache_mode: all large_distro_name: rhel80-medium num_scons_link_jobs_available: 0.99 - tasks: &enterprise-rhel80-dynamic-v4gcc-debug-experimental-tasks + tasks: &enterprise-rhel8-dynamic-v4gcc-debug-experimental-tasks - name: compile_test_and_package_parallel_core_stream_TG distros: - rhel80-xlarge @@ -391,7 +396,6 @@ buildvariants: - name: jsCore_txns_large_txns_format - name: json_schema - name: libunwind_tests - - name: mqlrun - name: .multi_shard !.multiversion - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .read_write_concern .large !.multiversion @@ -428,29 +432,29 @@ buildvariants: - name: vector_search_auth - name: vector_search_ssl -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-dynamic-v4clang-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang DEBUG" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-dynamic-v4clang-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang DEBUG" expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --use-diagnostic-latches=on -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-dynamic-v4gcc-cxx20-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC C++20 DEBUG" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-dynamic-v4gcc-cxx20-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC C++20 DEBUG" expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --use-diagnostic-latches=on -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-dynamic-v4clang-cxx20-debug-experimental - display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang C++20 DEBUG" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-dynamic-v4clang-cxx20-debug-experimental + display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang C++20 DEBUG" expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --use-diagnostic-latches=on -- name: enterprise-rhel-80-64-bit-coverage - display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage" +- name: enterprise-rhel-8-64-bit-coverage + display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage" run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -461,6 +465,7 @@ buildvariants: compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on large_distro_name: rhel80-medium multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # The gcov instrumentation saves the path the .gcno files were created in as the default path @@ -474,7 +479,7 @@ buildvariants: use_scons_cache: false gcov_tool: /opt/mongodbtoolchain/v3/bin/gcov num_scons_link_jobs_available: 0.99 - tasks: &enterprise-rhel-80-64-bit-coverage-tasks + tasks: &enterprise-rhel-8-64-bit-coverage-tasks - name: compile_test_and_package_parallel_core_stream_TG distros: - rhel80-large @@ -530,8 +535,8 @@ buildvariants: #- name: snmp #- name: update_fuzzer_gen -- name: enterprise-rhel-80-64-bit-coverage-clang - display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (clang)" +- name: enterprise-rhel-8-64-bit-coverage-clang + display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (clang)" run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -542,6 +547,7 @@ buildvariants: compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --link-model=dynamic --use-diagnostic-latches=on large_distro_name: rhel80-medium multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # The gcov instrumentation saves the path the .gcno files were created in as the default path @@ -559,10 +565,10 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v3/bin/gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-80-64-bit-coverage-tasks + *enterprise-rhel-8-64-bit-coverage-tasks -- name: enterprise-rhel-80-64-bit-coverage-experimental - display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v4)" +- name: enterprise-rhel-8-64-bit-coverage-experimental + display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (v4)" run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -573,6 +579,7 @@ buildvariants: compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_gcc.vars --link-model=dynamic --use-diagnostic-latches=on large_distro_name: rhel80-medium multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # The gcov instrumentation saves the path the .gcno files were created in as the default path @@ -587,10 +594,10 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v4/bin/gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-80-64-bit-coverage-tasks + *enterprise-rhel-8-64-bit-coverage-tasks -- name: enterprise-rhel-80-64-bit-coverage-clang-experimental - display_name: "~ Enterprise RHEL 8.0 DEBUG Code Coverage (v4 clang)" +- name: enterprise-rhel-8-64-bit-coverage-clang-experimental + display_name: "~ Enterprise RHEL 8 DEBUG Code Coverage (v4 clang)" run_on: - rhel80-medium cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -601,6 +608,7 @@ buildvariants: compile_flags: --dbg=on --gcov --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v4_clang.vars --link-model=dynamic --use-diagnostic-latches=on large_distro_name: rhel80-medium multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.5 # Avoid starting too many mongod's # The gcov instrumentation saves the path the .gcno files were created in as the default path @@ -615,16 +623,17 @@ buildvariants: gcov_tool: /opt/mongodbtoolchain/v4/bin/llvm-cov gcov num_scons_link_jobs_available: 0.99 tasks: - *enterprise-rhel-80-64-bit-coverage-tasks + *enterprise-rhel-8-64-bit-coverage-tasks -- name: enterprise-rhel-80-64-bit-bson-column - display_name: "~ Shared Library Enterprise RHEL 8.0 (BSONColumn validate)" +- name: enterprise-rhel-8-64-bit-bson-column + display_name: "~ Shared Library Enterprise RHEL 8 (BSONColumn validate)" run_on: - rhel80-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -687,7 +696,6 @@ buildvariants: - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - - name: mqlrun - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .query_fuzzer - name: .read_write_concern .large @@ -728,36 +736,36 @@ buildvariants: - name: vector_search_auth - name: vector_search_ssl -#- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template -# name: enterprise-rhel80-dynamic-v4gcc-cxx20-debug-pm-1328-experimental -# display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain GCC C++20 DEBUG + PM-1328" +#- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template +# name: enterprise-rhel8-dynamic-v4gcc-cxx20-debug-pm-1328-experimental +# display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain GCC C++20 DEBUG + PM-1328" # cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # expansions: -# <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions +# <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions # compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --disable-warnings-as-errors -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-dynamic-v4clang-cxx20-debug-pm-1328-experimental - display_name: "~ Shared Library Enterprise RHEL 8.0 v4 Toolchain Clang C++20 DEBUG + PM-1328" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-dynamic-v4clang-cxx20-debug-pm-1328-experimental + display_name: "~ Shared Library Enterprise RHEL 8 v4 Toolchain Clang C++20 DEBUG + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --link-model=dynamic --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --use-diagnostic-latches=on -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-v4gcc-cxx20-pm-1328-experimental - display_name: "~ Enterprise RHEL 8.0 v4 Toolchain GCC C++20 + PM-1328" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-v4gcc-cxx20-pm-1328-experimental + display_name: "~ Enterprise RHEL 8 v4 Toolchain GCC C++20 + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_gcc.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --disable-warnings-as-errors --use-diagnostic-latches=on -- <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-template - name: enterprise-rhel80-v4clang-cxx20-pm-1328-experimental - display_name: "~ Enterprise RHEL 8.0 v4 Toolchain Clang C++20 + PM-1328" +- <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-template + name: enterprise-rhel8-v4clang-cxx20-pm-1328-experimental + display_name: "~ Enterprise RHEL 8 v4 Toolchain Clang C++20 + PM-1328" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: - <<: *enterprise-rhel80-dynamic-v4gcc-debug-experimental-expansions + <<: *enterprise-rhel8-dynamic-v4gcc-debug-experimental-expansions compile_flags: --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_testing_clang.vars --cxx-std=20 --experimental-optimization=* --experimental-runtime-hardening=* --use-diagnostic-latches=on - name: stm-daily-cron @@ -767,6 +775,7 @@ buildvariants: - rhel80-small expansions: multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise stepback: false tasks: @@ -905,7 +914,6 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common - - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - name: .replica_sets .common @@ -982,7 +990,6 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common - - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough @@ -1097,7 +1104,7 @@ buildvariants: display_name: "* macOS DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - - macos-1100 + - macos-11 expansions: &macos-debug-expansions test_flags: --excludeWithAnyTags=incompatible_with_macos --enableEnterpriseTests=off resmoke_jobs_max: 6 @@ -1125,7 +1132,7 @@ buildvariants: display_name: "Enterprise macOS Via Rosetta 2" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app @@ -1141,7 +1148,6 @@ buildvariants: - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl - name: .jscore .common !.decimal !.sharding - name: .logical_session_cache .one_sec - - name: mqlrun # TODO(SERVER-64009): Re-enable replica_sets_auth_gen. # - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen @@ -1152,7 +1158,7 @@ buildvariants: display_name: "Enterprise macOS C++20 DEBUG" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app @@ -1168,7 +1174,6 @@ buildvariants: - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore !.ssl - name: .jscore .common !.decimal !.sharding - name: .logical_session_cache .one_sec - - name: mqlrun - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough_gen - name: sasl @@ -1177,7 +1182,7 @@ buildvariants: - name: enterprise-macos-arm64 display_name: "~ Enterprise macOS arm64" run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm compile_env: DEVELOPER_DIR=/Applications/Xcode13.app @@ -1194,7 +1199,6 @@ buildvariants: - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec - - name: mqlrun - name: replica_sets_auth_gen - name: replica_sets_jscore_passthrough - name: sasl @@ -1209,7 +1213,7 @@ buildvariants: - name: embedded-sdk-macos display_name: "Embedded SDK - macOS" run_on: - - macos-1100 + - macos-11 cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter expansions: test_flags: --excludeWithAnyTags=uses_transactions,incompatible_with_macos --enableEnterpriseTests=off @@ -1261,17 +1265,18 @@ buildvariants: # Redhat buildvariants # ########################################### -- &enterprise-rhel-80-64-bit-dynamic-required-template - name: enterprise-rhel-80-64-bit-dynamic-required - display_name: "! Shared Library Enterprise RHEL 8.0" +- &enterprise-rhel-8-64-bit-dynamic-required-template + name: enterprise-rhel-8-64-bit-dynamic-required + display_name: "! Shared Library Enterprise RHEL 8" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-dynamic-required-expansions + expansions: &enterprise-rhel-8-64-bit-dynamic-required-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -1346,11 +1351,14 @@ buildvariants: - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - name: .multiversion_sanity_check - - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -1405,18 +1413,19 @@ buildvariants: - name: vector_search_ssl - name: generate_buildid_to_debug_symbols_mapping -- &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-template - name: enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required - display_name: "! Shared Library Enterprise RHEL 8.0 (all feature flags)" +- &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-template + name: enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required + display_name: "! Shared Library Enterprise RHEL 8 (all feature flags)" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter stepback: false run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-expansions + expansions: &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -1436,7 +1445,7 @@ buildvariants: # WARNING! Task splitting is not supported for burn-in tasks. Large unsplitted `_gen` tasks may # run too long and hit execution timeouts. # burn_in_task_name: jsCore - tasks: &enterprise-rhel-80-64-bit-dynamic-all-feature-flags-tasks + tasks: &enterprise-rhel-8-64-bit-dynamic-all-feature-flags-tasks - name: cqf - name: cqf_parallel - name: compile_test_and_package_parallel_core_stream_TG @@ -1483,12 +1492,15 @@ buildvariants: - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: powercycle_smoke - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -1530,6 +1542,10 @@ buildvariants: - name: .serverless - name: .updatefuzzer - name: secondary_reads_passthrough_gen + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough # Disabling as the following tests are not aware of feature flags. # - name: server_discovery_and_monitoring_json_test_TG # distros: @@ -1542,17 +1558,18 @@ buildvariants: - name: vector_search_auth - name: vector_search_ssl -- &enterprise-rhel-80-64-bit-dynamic-classic-engine - name: enterprise-rhel-80-64-bit-dynamic-classic-engine - display_name: "Shared Library Enterprise RHEL 8.0 (Classic Engine)" +- &enterprise-rhel-8-64-bit-dynamic-sbe + name: enterprise-rhel-8-64-bit-dynamic-sbe + display_name: "Shared Library Enterprise RHEL 8 (SBE)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. stepback: false run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-dynamic-classic-engine-expansions + expansions: &enterprise-rhel-8-64-bit-dynamic-sbe-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -1564,7 +1581,7 @@ buildvariants: large_distro_name: rhel80-medium num_scons_link_jobs_available: 0.99 test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --mongodSetParameters="{internalQueryForceClassicEngine: false}" tasks: - name: .aggfuzzer - name: .aggregation @@ -1628,10 +1645,13 @@ buildvariants: - name: jsCore_txns_large_txns_format - name: json_schema - name: lint_fuzzer_sanity_patch - - name: mqlrun - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .multiversion_sanity_check + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: replica_sets_api_version_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_stepdown_passthrough_gen @@ -1661,8 +1681,8 @@ buildvariants: - name: vector_search_auth - name: vector_search_ssl -- name: enterprise-rhel-80-64-bit-large-txns-format - display_name: "Enterprise RHEL 8.0 (large transactions format)" +- name: enterprise-rhel-8-64-bit-large-txns-format + display_name: "Enterprise RHEL 8 (large transactions format)" run_on: - rhel80-small cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -1676,6 +1696,7 @@ buildvariants: --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise repo_edition: enterprise scons_cache_mode: all @@ -1742,13 +1763,13 @@ buildvariants: # This build variant is used to run multiversion tests as part of burn_in_tags as these tests are # currently only run on our daily builders. -- &enterprise-rhel-80-64-bit-multiversion-template - name: enterprise-rhel-80-64-bit-multiversion - display_name: "Enterprise RHEL 8.0 (implicit multiversion)" +- &enterprise-rhel-8-64-bit-multiversion-template + name: enterprise-rhel-8-64-bit-multiversion + display_name: "Enterprise RHEL 8 (implicit multiversion)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-multiversion-expansions-template + expansions: &enterprise-rhel-8-64-bit-multiversion-expansions-template compile_flags: >- -j$(grep -c ^processor /proc/cpuinfo) --ssl @@ -1757,6 +1778,7 @@ buildvariants: --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise repo_edition: enterprise scons_cache_scope: shared @@ -1780,12 +1802,12 @@ buildvariants: - name: .multiversion_passthrough - name: .random_multiversion_ds -- <<: *enterprise-rhel-80-64-bit-multiversion-template - name: enterprise-rhel-80-64-bit-multiversion-all-feature-flags - display_name: "Enterprise RHEL 8.0 (implicit multiversion & all feature flags)" +- <<: *enterprise-rhel-8-64-bit-multiversion-template + name: enterprise-rhel-8-64-bit-multiversion-all-feature-flags + display_name: "Enterprise RHEL 8 (implicit multiversion & all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: - <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template + <<: *enterprise-rhel-8-64-bit-multiversion-expansions-template # No feature flag tests since they aren't compatible with the older binaries. test_flags: >- --runNoFeatureFlagTests @@ -1805,8 +1827,8 @@ buildvariants: - name: .multiversion_passthrough - name: .random_multiversion_ds -- name: enterprise-rhel-80-64-bit-future-git-tag-multiversion - display_name: "Enterprise RHEL 8.0 (future git tag multiversion)" +- name: enterprise-rhel-8-64-bit-future-git-tag-multiversion + display_name: "Enterprise RHEL 8 (future git tag multiversion)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -1819,6 +1841,7 @@ buildvariants: --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise repo_edition: enterprise scons_cache_scope: shared @@ -1847,8 +1870,8 @@ buildvariants: - name: .multiversion !.future_git_tag_incompatible - name: .multiversion_future_git_tag -#- name: rhel-80-64-bit-nossl -# display_name: "RHEL 8.0 Shared Library (No SSL)" +#- name: rhel-8-64-bit-nossl +# display_name: "RHEL 8 Shared Library (No SSL)" # run_on: # - rhel80-small # cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. @@ -1860,6 +1883,7 @@ buildvariants: # MONGO_DISTMOD=rhel80 # --link-model=dynamic # multiversion_platform: rhel80 +# multiversion_platform_50_or_later: rhel8 # multiversion_edition: enterprise # scons_cache_scope: shared # scons_cache_mode: all @@ -1878,9 +1902,9 @@ buildvariants: # - rhel80-large # - name: jsCore -- &enterprise-rhel-80-64-bit-suggested-template - name: enterprise-rhel-80-64-bit-suggested - display_name: "* Enterprise RHEL 8.0" +- &enterprise-rhel-8-64-bit-suggested-template + name: enterprise-rhel-8-64-bit-suggested + display_name: "* Enterprise RHEL 8" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - rhel80-build @@ -1888,6 +1912,7 @@ buildvariants: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: true packager_script: packager_enterprise.py @@ -2078,7 +2103,7 @@ buildvariants: - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel80-dynamic-clang-tidy-required +- name: enterprise-rhel8-dynamic-clang-tidy-required display_name: "! Enterprise Clang Tidy" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: @@ -2099,8 +2124,8 @@ buildvariants: - name: clang_tidy_TG -- name: rhel80-debug-suggested - display_name: "* Shared Library RHEL 8.0 DEBUG" +- name: rhel8-debug-suggested + display_name: "* Shared Library RHEL 8 DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - rhel80-small @@ -2141,8 +2166,8 @@ buildvariants: # storage engine buildvariants # ################################ -- name: enterprise-rhel-80-benchmarks - display_name: Enterprise RHEL 8.0 (Benchmarks) +- name: enterprise-rhel-8-benchmarks + display_name: Enterprise RHEL 8 (Benchmarks) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-medium @@ -2152,66 +2177,12 @@ buildvariants: - name: compile_benchmarks - name: .benchmarks -- name: linux-64-ephemeralForTest - display_name: Linux (ephemeralForTest) - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - rhel80-medium - expansions: - # Transactions are not explicitly supported on the ephemeralForTest storage engine. - # Speculative majority reads are currently only allowed for change streams, which are only supported on WiredTiger. - # We also relax oplog constraints so that applying oplog entries after a rollbackViaRefetch works correctly. - # TODO (SERVER-47022): Re-enable oplog constraint enforcement once we set the application mode - # correctly after rollbackViaRefetch. - test_flags: >- - --storageEngine=ephemeralForTest - --excludeWithAnyTags=requires_persistence,requires_fsync,requires_journaling,requires_wiredtiger,uses_transactions,uses_speculative_majority,requires_snapshot_read,requires_majority_read_concern,uses_change_streams,requires_sharding,incompatible_with_eft - --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false}" - --enableEnterpriseTests=off - compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --modules= --use-diagnostic-latches=on - multiversion_platform: rhel80 - multiversion_edition: targeted - scons_cache_mode: all - scons_cache_scope: shared - num_scons_link_jobs_available: 0.99 - tasks: - - name: compile_test_and_package_parallel_core_stream_TG - - name: compile_test_and_package_parallel_unittest_stream_TG - - name: compile_test_and_package_parallel_dbtest_stream_TG - # TODO (SERVER-58125): Re-enable the timeseries fuzzer for EFT - - name: .aggfuzzer .common !.timeseries - - name: aggregation - - name: .auth !.multiversion !.audit !.sharding - # SERVER-58597: Re-enable parallel - - name: .misc_js !.parallel - # SERVER-50296: Investigate concurrency test failures. - # - name: concurrency_gen - # - name: concurrency_replication_gen - # - name: concurrency_replication_causal_consistency_gen - # - name: concurrency_simultaneous_gen - # - name: concurrency_simultaneous_replication - - name: .integration !.audit - - name: .jscore .common !.txns !.decimal - # SERVER-50295: Investigate jstestfuzz time outs. - # - name: .jstestfuzz .common - # SERVER-59095: Investigate the EFT failure and re-enable the failing jstest suite on evergreen. - # - name: .logical_session_cache .one_sec - - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen - - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen - - name: .read_write_concern .linearize - - name: replica_sets_gen - - name: .replica_sets .common - # SERVER-49428: Disabled due to writeConcernMajorityJournalDefault is not off - # rollback_fuzzer_gen - - name: .updatefuzzer - - ########################################### # Experimental buildvariants # ########################################### -- name: rhel80-debug-asan-all-feature-flags - display_name: "~ Shared Library ASAN Enterprise RHEL 8.0 DEBUG (all feature flags)" +- name: rhel8-debug-asan-all-feature-flags + display_name: "~ Shared Library ASAN Enterprise RHEL 8 DEBUG (all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -2224,6 +2195,7 @@ buildvariants: san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. hang_analyzer_dump_core: false @@ -2278,14 +2250,18 @@ buildvariants: - name: snmp - name: .serverless - name: .watchdog + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough # Disabling the following tests as they are not aware of feature flags. # - name: .benchmarks # - name: unittest_shell_hang_analyzer_gen # - name: server_discovery_and_monitoring_json_test_TG # - name: server_selection_json_test_TG -- name: rhel80-debug-asan-classic-engine - display_name: ~ ASAN Enterprise RHEL 8.0 DEBUG (Classic Engine) +- name: rhel8-debug-asan-sbe + display_name: ~ ASAN Enterprise RHEL 8 DEBUG (SBE) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -2298,10 +2274,11 @@ buildvariants: san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --mongodSetParameters="{internalQueryForceClassicEngine: false}" --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling --enableEnterpriseTests=off multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. hang_analyzer_dump_core: false @@ -2355,8 +2332,8 @@ buildvariants: - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- name: rhel80-debug-ubsan-all-feature-flags - display_name: "~ Shared Library UBSAN Enterprise RHEL 8.0 DEBUG (all feature flags)" +- name: rhel8-debug-ubsan-all-feature-flags + display_name: "~ Shared Library UBSAN Enterprise RHEL 8 DEBUG (all feature flags)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -2376,6 +2353,7 @@ buildvariants: --additionalFeatureFlagsFile all_feature_flags.txt --enableEnterpriseTests=off multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. scons_cache_scope: shared @@ -2420,13 +2398,14 @@ buildvariants: - name: .serverless - name: .updatefuzzer - name: watchdog_wiredtiger + - name: query_stats_passthrough # Disabling the following tests as they are not aware of feature flags. # - name: .benchmarks # - name: server_discovery_and_monitoring_json_test_TG # - name: server_selection_json_test_TG -- name: rhel80-debug-ubsan-classic-engine - display_name: "~ UBSAN Enterprise RHEL 8.0 DEBUG (Classic Engine)" +- name: rhel8-debug-ubsan-sbe + display_name: "~ UBSAN Enterprise RHEL 8 DEBUG (SBE)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -2439,10 +2418,11 @@ buildvariants: san_options: UBSAN_OPTIONS="print_stacktrace=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --modules= --use-diagnostic-latches=on test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --mongodSetParameters="{internalQueryForceClassicEngine: false}" --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits --enableEnterpriseTests=off multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. scons_cache_scope: shared @@ -2490,9 +2470,9 @@ buildvariants: - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- &rhel80-debug-aubsan-lite-required-template - name: rhel80-debug-aubsan-lite-required - display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG" +- &rhel8-debug-aubsan-lite-required-template + name: rhel8-debug-aubsan-lite-required + display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8 DEBUG" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter run_on: - rhel80-build @@ -2528,9 +2508,9 @@ buildvariants: - name: unittest_shell_hang_analyzer_gen - name: generate_buildid_to_debug_symbols_mapping -- <<: *rhel80-debug-aubsan-lite-required-template - name: rhel80-debug-aubsan-lite-all-feature-flags-required - display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8.0 DEBUG (all feature flags)" +- <<: *rhel8-debug-aubsan-lite-required-template + name: rhel8-debug-aubsan-lite-all-feature-flags-required + display_name: "! Shared Library {A,UB}SAN Enterprise RHEL 8 DEBUG (all feature flags)" cron: "0 */4 * * *" # From the ${project_required_suggested_cron} parameter expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug @@ -2562,8 +2542,8 @@ buildvariants: # - name: unittest_shell_hang_analyzer_gen - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel80-debug-tsan - display_name: ~ TSAN Enterprise RHEL 8.0 DEBUG +- name: enterprise-rhel8-debug-tsan + display_name: ~ TSAN Enterprise RHEL 8 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-large @@ -2594,8 +2574,8 @@ buildvariants: - name: compile_test_and_package_parallel_unittest_stream_TG - name: jsCore -- name: enterprise-rhel80-unoptimized-64-bit - display_name: "~ Enterprise RHEL 8.0 DEBUG (Unoptimized)" +- name: enterprise-rhel8-unoptimized-64-bit + display_name: "~ Enterprise RHEL 8 DEBUG (Unoptimized)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. expansions: compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --dbg=on --opt=off --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --detect-odr-violations --use-diagnostic-latches=on @@ -2613,8 +2593,8 @@ buildvariants: distros: - rhel80-build -- name: enterprise-rhel80-fixed-service-executor-64-bit - display_name: "~ Enterprise RHEL 8.0 (with FixedServiceExecutor)" +- name: enterprise-rhel8-fixed-service-executor-64-bit + display_name: "~ Enterprise RHEL 8 (with FixedServiceExecutor)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2624,6 +2604,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="{initialServiceExecutorThreadingModel: borrowed}" @@ -2651,8 +2632,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-sdam-replica-set-monitor-64-bit - display_name: "~ Enterprise RHEL 8.0 (with SdamReplicaSetMonitor)" +- name: enterprise-rhel8-sdam-replica-set-monitor-64-bit + display_name: "~ Enterprise RHEL 8 (with SdamReplicaSetMonitor)" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2662,6 +2643,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="{replicaSetMonitorProtocol: sdam}" @@ -2688,8 +2670,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-task-executor-pool-size-64-bit - display_name: "~ Enterprise RHEL 8.0 (with {taskExecutorPoolSize: 4})" +- name: enterprise-rhel8-task-executor-pool-size-64-bit + display_name: "~ Enterprise RHEL 8 (with {taskExecutorPoolSize: 4})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2699,6 +2681,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: |- # Set the taskExecutorPoolSize for all tests --mongosSetParameters="taskExecutorPoolSize: 4" @@ -2723,8 +2706,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-sharding-task-executor-pool-rsm-matchPrimary-64-bit - display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchPrimaryNode\"})" +- name: enterprise-rhel8-sharding-task-executor-pool-rsm-matchPrimary-64-bit + display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchPrimaryNode\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2734,6 +2717,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="ShardingTaskExecutorPoolReplicaSetMatching: \"matchPrimaryNode\"" @@ -2759,8 +2743,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-sharding-task-executor-pool-rsm-matchBusiest-64-bit - display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchBusiestNode\"})" +- name: enterprise-rhel8-sharding-task-executor-pool-rsm-matchBusiest-64-bit + display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"matchBusiestNode\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2770,6 +2754,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="ShardingTaskExecutorPoolReplicaSetMatching: \"matchBusiestNode\"" @@ -2795,8 +2780,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-sharding-task-executor-pool-rsm-disabled-64-bit - display_name: "~ Enterprise RHEL 8.0 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"disabled\"})" +- name: enterprise-rhel8-sharding-task-executor-pool-rsm-disabled-64-bit + display_name: "~ Enterprise RHEL 8 (with {ShardingTaskExecutorPoolReplicaSetMatching: \"disabled\"})" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -2806,6 +2791,7 @@ buildvariants: num_scons_link_jobs_available: 0.99 compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="ShardingTaskExecutorPoolReplicaSetMatching: \"disabled\"" @@ -2831,8 +2817,8 @@ buildvariants: - name: .sharding .jscore !.wo_snapshot !.multi_stmt - name: .sharding .common !.csrs -- name: enterprise-rhel80-join-ingress-sessions-on-shutdown - display_name: "~ Enterprise RHEL 8.0 (with {joinIngressSessionsOnShutdown: \"true\"})" +- name: enterprise-rhel8-join-ingress-sessions-on-shutdown + display_name: "~ Enterprise RHEL 8 (with {joinIngressSessionsOnShutdown: \"true\"})" activate: false run_on: - rhel80-small @@ -2840,6 +2826,7 @@ buildvariants: scons_cache_scope: shared compile_flags: MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise test_flags: >- --mongosSetParameters="joinIngressSessionsOnShutdown: \"true\"" @@ -2889,7 +2876,7 @@ buildvariants: cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter stepback: false expansions: - selected_tests_buildvariants: enterprise-windows-required linux-64-debug-required enterprise-ubuntu-dynamic-1604-clang rhel80-debug-aubsan-lite-required + selected_tests_buildvariants: enterprise-windows-required linux-64-debug-required enterprise-ubuntu-dynamic-1604-clang rhel8-debug-aubsan-lite-required tasks: - name: selected_tests_gen @@ -2913,26 +2900,26 @@ buildvariants: - windows-vsCurrent-large ### QO & QE Patch-Specific Build Variants ### -- <<: *enterprise-rhel-80-64-bit-dynamic-classic-engine - name: enterprise-rhel-80-64-bit-dynamic-classic-engine-query-patch-only - display_name: "~ Shared Library Enterprise RHEL 8.0 Query Patch Only (Classic Engine)" +- <<: *enterprise-rhel-8-64-bit-dynamic-sbe + name: enterprise-rhel-8-64-bit-dynamic-sbe-query-patch-only + display_name: "~ Shared Library Enterprise RHEL 8 Query Patch Only (SBE)" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # This is a patch-only variant but we run on mainline to pick up task history. expansions: - <<: *enterprise-rhel-80-64-bit-dynamic-classic-engine-expansions + <<: *enterprise-rhel-8-64-bit-dynamic-sbe-expansions jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 target_resmoke_time: 30 max_sub_suites: 3 test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --mongodSetParameters="{internalQueryForceClassicEngine: false}" --excludeWithAnyTags=resource_intensive -- <<: *enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-template - name: enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required-query-patch-only - display_name: "~ Shared Library Enterprise RHEL 8.0 Query Patch Only (all feature flags)" +- <<: *enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-template + name: enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required-query-patch-only + display_name: "~ Shared Library Enterprise RHEL 8 Query Patch Only (all feature flags)" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter # This is a patch-only variant but we run on mainline to pick up task history. expansions: - <<: *enterprise-rhel-80-64-bit-dynamic-all-feature-flags-expansions + <<: *enterprise-rhel-8-64-bit-dynamic-all-feature-flags-expansions jstestfuzz_num_generated_files: 20 jstestfuzz_concurrent_num_files: 5 target_resmoke_time: 30 @@ -2993,8 +2980,8 @@ buildvariants: distros: - windows-vsCurrent-large -- name: rhel80-debug-aubsan-lite_fuzzer - display_name: "{A,UB}SAN Enterprise RHEL 8.0 FUZZER" +- name: rhel8-debug-aubsan-lite_fuzzer + display_name: "{A,UB}SAN Enterprise RHEL 8 FUZZER" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build diff --git a/etc/evergreen_nightly.yml b/etc/evergreen_nightly.yml index 19dcec48264..5df73217f7b 100644 --- a/etc/evergreen_nightly.yml +++ b/etc/evergreen_nightly.yml @@ -5,7 +5,6 @@ include: - filename: etc/evergreen_yml_components/variants/task_generation.yml - filename: etc/evergreen_yml_components/variants/atlas.yml - filename: etc/evergreen_yml_components/variants/misc_release.yml -- filename: etc/evergreen_yml_components/variants/openssl_111.yml ### Comment out when using this file for a LTS or Rapid release branch. ### - filename: etc/evergreen_yml_components/variants/ibm.yml ### Uncomment when using this file for a LTS release branch. ### @@ -144,7 +143,6 @@ buildvariants: - name: jsCore_ese - name: jsCore_txns_large_txns_format - name: .jstestfuzz .common - - name: mqlrun - name: noPassthrough_gen - name: noPassthroughWithMongod_gen - name: .replica_sets .common !.ignore_non_generated_replica_sets_jscore_passthrough @@ -154,6 +152,7 @@ buildvariants: - windows-vsCurrent-large - name: replica_sets_ese_gen - name: sasl + - name: sasl_windows_cyrussasl - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG - name: .sharding .txns @@ -163,17 +162,18 @@ buildvariants: - name: snmp - name: unittest_shell_hang_analyzer_gen -- &enterprise-rhel-80-64-bit-dynamic-required-template - name: enterprise-rhel-80-64-bit-dynamic-required - display_name: "! Shared Library Enterprise RHEL 8.0" +- &enterprise-rhel-8-64-bit-dynamic-required-template + name: enterprise-rhel-8-64-bit-dynamic-required + display_name: "! Shared Library Enterprise RHEL 8" cron: "0 */4 * * *" # Every 4 hours starting at midnight run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-dynamic-required-expansions + expansions: &enterprise-rhel-8-64-bit-dynamic-required-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -215,6 +215,7 @@ buildvariants: - name: .auth - name: buildscripts_test - name: resmoke_end2end_tests + - name: resmoke_validation_tests - name: unittest_shell_hang_analyzer_gen - name: .causally_consistent !.sharding - name: .change_streams @@ -249,11 +250,14 @@ buildvariants: - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - name: .multiversion_sanity_check - - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -362,7 +366,7 @@ buildvariants: - name: jsCore_txns - name: unittest_shell_hang_analyzer_gen -- <<: *enterprise-rhel-80-64-bit-dynamic-required-template +- <<: *enterprise-rhel-8-64-bit-dynamic-required-template name: commit-queue display_name: "~ Commit Queue" cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter @@ -384,14 +388,15 @@ buildvariants: - name: test_api_version_compatibility - name: validate_commit_message - name: check_feature_flag_tags + - name: resmoke_validation_tests # Temporary variant to stress a component and gather more data. -- <<: *enterprise-rhel-80-64-bit-dynamic-required-template - name: enterprise-rhel-80-64-bit-dynamic-stress-initialsyncid # For BF-25011 - display_name: "~ Shared Library Enterprise RHEL 8.0 (initialSyncId stress test)" +- <<: *enterprise-rhel-8-64-bit-dynamic-required-template + name: enterprise-rhel-8-64-bit-dynamic-stress-initialsyncid # For BF-25011 + display_name: "~ Shared Library Enterprise RHEL 8 (initialSyncId stress test)" cron: "0 */1 * * *" # Every hour starting at midnight expansions: - <<: *enterprise-rhel-80-64-bit-dynamic-required-expansions + <<: *enterprise-rhel-8-64-bit-dynamic-required-expansions max_sub_suites: 10 # Set a low target time to run only 1 test per generated Evergreen task. Increase timeout. target_resmoke_time: 1 @@ -406,8 +411,8 @@ buildvariants: - rhel80-xlarge - name: .sharding -- name: enterprise-rhel-80-64-bit-large-txns-format - display_name: "Enterprise RHEL 8.0 (large transactions format)" +- name: enterprise-rhel-8-64-bit-large-txns-format + display_name: "Enterprise RHEL 8 (large transactions format)" run_on: - rhel80-small cron: "0 4 * * 0" @@ -420,6 +425,7 @@ buildvariants: --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --link-model=dynamic multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise repo_edition: enterprise scons_cache_mode: all @@ -478,9 +484,9 @@ buildvariants: - rhel80-medium - name: generate_buildid_to_debug_symbols_mapping -- &enterprise-rhel-80-64-bit-dynamic-classic-engine - name: enterprise-rhel-80-64-bit-dynamic-classic-engine - display_name: "Shared Library Enterprise RHEL 8.0 (Classic Engine)" +- &enterprise-rhel-8-64-bit-dynamic-sbe + name: enterprise-rhel-8-64-bit-dynamic-sbe + display_name: "Shared Library Enterprise RHEL 8 (SBE)" cron: "0 4 * * 0" run_on: - rhel80-small @@ -488,6 +494,7 @@ buildvariants: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --link-model=dynamic multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: false scons_cache_scope: shared @@ -499,7 +506,7 @@ buildvariants: large_distro_name: rhel80-medium num_scons_link_jobs_available: 0.99 test_flags: >- - --mongodSetParameters="{internalQueryForceClassicEngine: true}" + --mongodSetParameters="{internalQueryForceClassicEngine: false}" --excludeWithAnyTags=requires_latch_analyzer tasks: - name: .aggfuzzer @@ -518,6 +525,10 @@ buildvariants: - name: .misc_js - name: .multi_shard - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_only - name: .read_write_concern !.large @@ -551,7 +562,6 @@ buildvariants: - name: jsCore_txns_large_txns_format - name: json_schema - name: lint_fuzzer_sanity_patch - - name: mqlrun - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .multiversion_sanity_check diff --git a/etc/evergreen_timeouts.yml b/etc/evergreen_timeouts.yml index 071fb427808..136d0118f76 100644 --- a/etc/evergreen_timeouts.yml +++ b/etc/evergreen_timeouts.yml @@ -19,18 +19,34 @@ overrides: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - enterprise-rhel-80-64-bit-coverage: + enterprise-rhel-8-64-bit-coverage: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours. - enterprise-rhel80-debug-complete-tsan: + enterprise-rhel8-debug-complete-tsan: - task: aggregation_timeseries_fuzzer exec_timeout: 150 # 2.5 hours - enterprise-rhel80-debug-tsan: + enterprise-rhel8-debug-tsan: - task: aggregation_timeseries_fuzzer exec_timeout: 150 # 2.5 hours + windows-debug-suggested: + - task: buildscripts_test + exec_timeout: 15 + + enterprise-windows-wtdevelop: + - task: buildscripts_test + exec_timeout: 15 + + windows: + - task: buildscripts_test + exec_timeout: 15 + + enterprise-windows: + - task: buildscripts_test + exec_timeout: 15 + macos: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours @@ -39,27 +55,27 @@ overrides: - task: replica_sets_jscore_passthrough exec_timeout: 150 # 2.5 hours - rhel80-asan: + rhel8-asan: - task: aggregation_timeseries_fuzzer exec_timeout: 150 # 2.5 hours - rhel80-debug-ubsan: + rhel8-debug-ubsan: - task: update_timeseries_fuzzer exec_timeout: 150 # 2.5 hours - rhel80-debug-suggested: + rhel8-debug-suggested: - task: replica_sets_jscore_passthrough exec_timeout: 180 # 3 hours. - rhel80-debug-ubsan-all-feature-flags: + rhel8-debug-ubsan-all-feature-flags: - task: update_timeseries_fuzzer exec_timeout: 150 # 2.5 hours - rhel80-debug-ubsan-classic-engine: + rhel8-debug-ubsan-sbe-engine: - task: update_timeseries_fuzzer exec_timeout: 150 # 2.5 hours - rhel80-debug-v4ubsan-all-feature-flags-experimental: + rhel8-debug-v4ubsan-all-feature-flags-experimental: - task: update_timeseries_fuzzer exec_timeout: 150 # 2.5 hours diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index faa6ba5290e..d80d00b6813 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -14,22 +14,6 @@ pre_error_fails_task: true stepback: true -# Files that match an ignore-list pattern will not trigger a build, if they're the only modified -# files in the patch. -ignore: -- ".*" -- "!.clang-format" -- "!.eslintrc.yml" -- "*.md" -- "*.rst" -- "*.txt" -- "/distsrc/**" -- "/docs/**" -- "/etc/*.yml" -- "!/etc/evergreen.yml" -- "README" - - ## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) parameters: - key: patch_compile_flags @@ -294,9 +278,7 @@ variables: # If a variant is listed here and has a push task, mongocryptd is pushed - mongocryptd_variants: &mongocryptd_variants - enterprise-amazon2 - - enterprise-amazon2-openssl111 - enterprise-amazon2-arm64 - - enterprise-amazon2-arm64-openssl111 - enterprise-amazon2023 - enterprise-amazon2023-arm64 - enterprise-debian10-64 @@ -310,12 +292,12 @@ variables: - enterprise-rhel-71-ppc64le - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - - enterprise-rhel-80-64-bit - - enterprise-rhel-80-64-bit-coverage - - enterprise-rhel-80-64-bit-inmem - - enterprise-rhel-80-64-bit-multiversion - - enterprise-rhel-80-64-bit-suggested - - enterprise-rhel-82-arm64 + - enterprise-rhel-8-64-bit + - enterprise-rhel-8-64-bit-coverage + - enterprise-rhel-8-64-bit-inmem + - enterprise-rhel-8-64-bit-multiversion + - enterprise-rhel-8-64-bit-suggested + - enterprise-rhel-8-arm64 - enterprise-rhel-83-s390x - enterprise-rhel-90-64-bit - enterprise-rhel-90-arm64 @@ -343,10 +325,10 @@ variables: - mh_variants: &mh_variants - enterprise-macos - enterprise-macos-arm64 - - enterprise-rhel-80-64-bit - - enterprise-rhel-80-64-bit-dynamic-required + - enterprise-rhel-8-64-bit + - enterprise-rhel-8-64-bit-dynamic-required - enterprise-rhel-70-64-bit - - enterprise-rhel-82-arm64 + - enterprise-rhel-8-arm64 - enterprise-rhel-90-64-bit - enterprise-rhel-90-arm64 - enterprise-amazon2-arm64 @@ -365,9 +347,7 @@ variables: - amazon2023 - amazon2023-arm64 - enterprise-amazon2 - - enterprise-amazon2-openssl111 - enterprise-amazon2-arm64 - - enterprise-amazon2-arm64-openssl111 - enterprise-amazon2023 - enterprise-amazon2023-arm64 - debian10 @@ -381,13 +361,13 @@ variables: - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - ubi8 - - rhel80 - - rhel-82-arm64 + - rhel8 + - rhel-8-arm64 - rhel90 - rhel90-arm64 - - enterprise-rhel-80-64-bit - - enterprise-rhel-80-64-bit-suggested # For testing selinux. - - enterprise-rhel-82-arm64 + - enterprise-rhel-8-64-bit + - enterprise-rhel-8-64-bit-suggested # For testing selinux. + - enterprise-rhel-8-arm64 - enterprise-rhel-83-s390x - enterprise-rhel-90-64-bit - enterprise-rhel-90-arm64 @@ -636,7 +616,7 @@ functions: "get buildnumber": &get_buildnumber command: keyval.inc params: - key: "${build_variant}_v6.0" + key: "${build_variant}_${project}" destination: "builder_num" "run diskstats": &run_diskstats @@ -1352,16 +1332,36 @@ functions: "do jepsen setup": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: libfaketime + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec params: binary: bash args: - - "./src/evergreen/do_jepsen_setup/build_libfaketime.sh" + - "./src/evergreen/do_jepsen_setup/build_libfaketime.sh" + include_expansions_in_env: + - github_token + - command: github.generate_token + params: + owner: 10gen + repo: jepsen + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec params: binary: bash args: - "./src/evergreen/do_jepsen_setup/install_jepsen.sh" + include_expansions_in_env: + - github_token - command: subprocess.exec params: binary: bash @@ -1375,12 +1375,22 @@ functions: "do jepsen docker setup": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: jepsen-io-mongodb + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec type: system params: binary: bash args: - - "./src/evergreen/jepsen_docker/setup.sh" + - "./src/evergreen/jepsen_docker/setup.sh" + include_expansions_in_env: + - github_token "run jepsen docker test": - *f_expansions_write - command: subprocess.exec @@ -1455,11 +1465,22 @@ functions: "setup jstestfuzz": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: jstestfuzz + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec + display_name: "jstestfuzz setup sh" params: binary: bash args: - - "./src/evergreen/jstestfuzz_setup.sh" + - "./src/evergreen/jstestfuzz_setup.sh" + include_expansions_in_env: + - github_token "lint fuzzer sanity patch": - *f_expansions_write @@ -1492,11 +1513,21 @@ functions: # Used by generator "run jstestfuzz": - *f_expansions_write + - command: github.generate_token + params: + owner: 10gen + repo: QA + expansion_name: github_token + permissions: + metadata: read + contents: read - command: subprocess.exec params: binary: bash args: - "./src/evergreen/run_jstestfuzz/clone_repos.sh" + include_expansions_in_env: + - github_token - *f_expansions_write - command: subprocess.exec type: test @@ -1582,7 +1613,9 @@ functions: params: files: - wiki_page_running_minimized_test_location.json + file: src/compile_expansions.yml + # Used by generator "run idl tests": - *f_expansions_write - command: subprocess.exec @@ -1658,7 +1691,7 @@ functions: params: provider: ec2 distro: ${distro_id} - timeout_teardown_secs: 604800 # 7 days + timeout_teardown_secs: 86400 # 24 hours security_group_ids: - sg-097bff6dd0d1d31d0 @@ -2209,6 +2242,13 @@ functions: content_type: text/plain display_name: Resmoke.py Invocation for Local Usage + "cleanup external auth test artifacts": + - *f_expansions_write + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/sasl_windows_cyrussasl_teardown.sh" # Pre task steps pre: @@ -2239,6 +2279,7 @@ post: - func: "cleanup FUSE watchdog" - func: "cleanup environment" - func: "cleanup jepsen docker test" +- func: "cleanup external auth test artifacts" # Timeout steps timeout: @@ -2246,7 +2287,6 @@ timeout: - func: "run hang analyzer" - func: "wait for resmoke to shutdown" - ####################################### # Tasks # ####################################### @@ -2635,7 +2675,7 @@ tasks: compiling_for_test: true - name: compile_and_run_unittests_first_quarter - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours depends_on: - name: compile_dist_test commands: @@ -2652,10 +2692,10 @@ tasks: vars: suite: unittests_first_quarter install_dir: build/install/bin - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours - name: compile_and_run_unittests_second_quarter - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours depends_on: - name: compile_dist_test commands: @@ -2672,10 +2712,10 @@ tasks: vars: suite: unittests_second_quarter install_dir: build/install/bin - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours - name: compile_and_run_unittests_third_quarter - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours depends_on: - name: compile_dist_test commands: @@ -2692,10 +2732,10 @@ tasks: vars: suite: unittests_third_quarter install_dir: build/install/bin - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours - name: compile_and_run_unittests_fourth_quarter - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours depends_on: - name: compile_dist_test commands: @@ -2712,7 +2752,7 @@ tasks: vars: suite: unittests_fourth_quarter install_dir: build/install/bin - exec_timeout_secs: 10800 # 3 hours + exec_timeout_secs: 21600 # 6 hours ## clang_tidy - run clang_tidy - name: clang_tidy @@ -3672,6 +3712,7 @@ tasks: num_files: 5 num_tasks: 5 resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + jstestfuzz_vars: --numPipelines=3000 npm_command: agg-fuzzer run_no_feature_flag_tests: "true" @@ -3703,7 +3744,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --diffTestingMode optimization + jstestfuzz_vars: --diffTestingMode optimization --numPipelines=3000 suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-expr-fuzzer @@ -3718,7 +3759,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --diffTestingMode optimization + jstestfuzz_vars: --diffTestingMode optimization --numPipelines=3000 suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-fuzzer @@ -3733,7 +3774,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --diffTestingMode wildcard + jstestfuzz_vars: --diffTestingMode wildcard --numPipelines=3000 npm_command: agg-fuzzer suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -3748,7 +3789,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --diffTestingMode timeseries + jstestfuzz_vars: --diffTestingMode timeseries --numPipelines=750 npm_command: agg-fuzzer suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -3763,7 +3804,7 @@ tasks: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - jstestfuzz_vars: --diffTestingMode optimization + jstestfuzz_vars: --diffTestingMode optimization --numPipelines=3000 npm_command: change-stream-fuzzer suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" @@ -4443,6 +4484,7 @@ tasks: - name: sync_repo_with_copybara tags: [] + patchable: false commands: - command: manifest.load - func: "git get project and add git tag" @@ -5014,6 +5056,20 @@ tasks: resmoke_args: --includeWithAnyTags=requires_domain_controller - <<: *task_template + name: sasl_windows_cyrussasl + tags: ["assigned_to_jira_team_server_security", "sasl", "experimental"] + commands: + - func: "f_expansions_write" + - func: "do setup" + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/sasl_windows_cyrussasl_setup.sh" + - func: "run tests" + +- <<: *task_template name: jsCore tags: ["jscore", "common"] commands: @@ -5376,7 +5432,13 @@ tasks: - <<: *gen_task_template name: noPassthrough_gen - tags: ["misc_js"] + tags: + [ + "incompatible_mac", + "incompatible_ppc", + "incompatible_s390x", + "misc_js" + ] commands: - func: "generate resmoke tasks" vars: @@ -5521,6 +5583,34 @@ tasks: vars: use_large_distro: "true" +- <<: *task_template + name: query_stats_passthrough + tags: [] + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template + name: query_stats_aggregation_passthrough + tags: [] + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template + name: query_stats_mongos_passthrough + tags: [] + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template + name: query_stats_mongos_aggregation_passthrough + tags: [] + commands: + - func: "do setup" + - func: "run tests" + - <<: *gen_task_template name: tenant_migration_jscore_passthrough_gen tags: ["serverless"] @@ -5612,14 +5702,6 @@ tasks: use_large_distro: "true" - <<: *gen_task_template - name: sharding_batched_deletes_passthrough_gen - tags: ["sharding", "large", "batched_deletes"] - commands: - - func: "generate resmoke tasks" - vars: - use_large_distro: "true" - -- <<: *gen_task_template name: change_streams_batched_deletes_passthrough_gen tags: ["change_streams", "large", "batched_deletes"] commands: @@ -6537,14 +6619,14 @@ tasks: task_names: >- powercycle_smoke_skip_compile num_tasks: 20 - exec_timeout_secs: 604800 # 7 days - timeout_secs: 604800 # 7 days + exec_timeout_secs: 86400 # 24 hours + timeout_secs: 86400 # 24 hours set_up_retry_count: 1800 run_powercycle_args: --sshAccessRetryCount=1800 - name: powercycle_sentinel tags: [] - exec_timeout_secs: 604800 # 7 days + exec_timeout_secs: 86400 # 24 hours commands: - func: "run powercycle sentinel" vars: @@ -6853,6 +6935,14 @@ tasks: - func: "do setup" - func: "run tests" +- <<: *task_template + name: resmoke_validation_tests + tags: [] + depends_on: [] + commands: + - func: "do non-compile setup" + - func: "run tests" + - name: package tags: [] depends_on: @@ -6862,7 +6952,6 @@ tasks: - func: "scons compile" vars: targets: >- - distsrc-${ext|tgz} archive-dist archive-dist-debug archive-jstestshell @@ -6886,19 +6975,6 @@ tasks: - "src/evergreen/package.sh" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/distsrc.${ext|tgz} - remote_file: ${project}/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Source tarball - # We only need to upload the source tarball from one of the build variants - # because it should be the same everywhere, so just use rhel70/windows. - build_variants: [rhel70, windows] - - command: s3.put - params: optional: true aws_key: ${aws_key} aws_secret: ${aws_secret} @@ -7079,14 +7155,6 @@ tasks: local_file: src/mh.tgz build_variants: *mh_variants # Fetch the sources (on relevant variants only) - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${project}/${build_variant}/${revision}/sources/mongo-src-${build_id}.${ext|tgz} - bucket: mciuploads - local_file: src/distsrc.${ext|tgz} - build_variants: [rhel70, windows] - func: "generate compile expansions" - func: "apply compile expansions" - func: "fetch dist debugsymbols" @@ -7151,17 +7219,6 @@ tasks: content_type: ${content_type|application/gzip} remote_file: ${push_path}-STAGE/${push_name}/mh-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz} build_variants: *mh_variants - # Put the source tarball - - command: s3.put - params: - aws_secret: ${aws_secret} - local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz} - aws_key: ${aws_key} - bucket: build-push-testing - permissions: public-read - content_type: ${content_type|application/gzip} - remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz} - build_variants: [rhel70, windows] # Put the debug symbols # push directly to repo due to limitations in file size SERVER-63432 @@ -7199,18 +7256,6 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sig build_variants: *mongocryptd_variants - # Put the source tarball signature - - command: s3.put - params: - aws_secret: ${aws_secret} - local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig - aws_key: ${aws_key} - bucket: build-push-testing - permissions: public-read - content_type: ${content_type|application/gzip} - remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig - build_variants: [rhel70, windows] - # Put the debug symbols signature - command: s3.put params: @@ -7258,18 +7303,6 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1 build_variants: *mongocryptd_variants - # Put the source tarball sha1 - - command: s3.put - params: - aws_secret: ${aws_secret} - local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1 - aws_key: ${aws_key} - permissions: public-read - bucket: build-push-testing - content_type: text/plain - remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1 - build_variants: [rhel70, windows] - # Put the debug symbols sha1 - command: s3.put params: @@ -7317,18 +7350,6 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha256 build_variants: *mongocryptd_variants - # Put the source tarball sha256 - - command: s3.put - params: - aws_secret: ${aws_secret} - local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256 - permissions: public-read - aws_key: ${aws_key} - bucket: build-push-testing - content_type: text/plain - remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256 - build_variants: [rhel70, windows] - # Put the debug symbols sha256 - command: s3.put params: @@ -7376,18 +7397,6 @@ tasks: remote_file: ${push_path}-STAGE/${push_name}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.md5 build_variants: *mongocryptd_variants - # Put the source tarball md5 - - command: s3.put - params: - aws_secret: ${aws_secret} - local_file: src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5 - aws_key: ${aws_key} - bucket: build-push-testing - permissions: public-read - content_type: text/plain - remote_file: ${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5 - build_variants: [rhel70, windows] - # Put the debug symbols md5 - command: s3.put params: @@ -7431,11 +7440,6 @@ tasks: 'destination': {'path': '${push_path}/mh-${push_name}-${push_arch}-${suffix}.${ext|tgz}', 'bucket': '${push_bucket}'}, 'build_variants': *mh_variants} - #Source tarball - - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}', 'bucket': 'build-push-testing'}, - 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}', 'bucket': '${push_bucket}'}, - 'build_variants': ['rhel70', 'windows']} - #MSI (Windows only) - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi', 'bucket': '${push_bucket}'}, @@ -7450,11 +7454,6 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sig', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} - #Source tarball signature - - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sig', 'bucket': 'build-push-testing'}, - 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sig', 'bucket': '${push_bucket}'}, - 'build_variants': ['rhel70', 'windows']} - #SHA1 for binaries - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}.${ext|tgz}.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}} @@ -7464,11 +7463,6 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha1', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} - #SHA1 for source tarball - - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha1', 'bucket': 'build-push-testing'}, - 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha1', 'bucket': '${push_bucket}'}, - 'build_variants': ['rhel70', 'windows']} - #SHA1 for MSI - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha1', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1', 'bucket': '${push_bucket}'}, @@ -7483,11 +7477,6 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.sha256', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} - #SHA256 for source tarball - - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.sha256', 'bucket': 'build-push-testing'}, - 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.sha256', 'bucket': '${push_bucket}'}, - 'build_variants': ['rhel70', 'windows']} - #SHA256 for MSI files - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha256', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256', 'bucket': '${push_bucket}'}, @@ -7502,11 +7491,6 @@ tasks: 'destination': {'path': '${push_path}/mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}, 'build_variants': *mongocryptd_variants} - #MD5 for source tarball - - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-src-${src_suffix}-${task_id}.${ext|tar.gz}.md5', 'bucket': 'build-push-testing'}, - 'destination': {'path': 'src/mongodb-src-${src_suffix}.${ext|tar.gz}.md5', 'bucket': '${push_bucket}'}, - 'build_variants': ['rhel70', 'windows']} - #MD5 for MSIs - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5', 'bucket': '${push_bucket}'}, @@ -7801,13 +7785,6 @@ tasks: args: - "./src/evergreen/todos_check.sh" -- <<: *task_template - name: mqlrun - tags: [] - commands: - - func: "do setup" - - func: "run tests" - - name: check_feature_flag_tags tags: [] patch_only: true diff --git a/etc/evergreen_yml_components/tasks/misc_tasks.yml b/etc/evergreen_yml_components/tasks/misc_tasks.yml new file mode 100644 index 00000000000..b217c0fcb19 --- /dev/null +++ b/etc/evergreen_yml_components/tasks/misc_tasks.yml @@ -0,0 +1,1611 @@ +# This file contains miscellaneous tasks + +################################################ +# Variable # +################################################ +variables: + - &antithesis_task_template + name: antithesis_task_template + depends_on: + name: archive_dist_test_debug + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: suite_name + resmoke_args: >- + --arg=val + --flag + + - &jepsen_config_vars + jepsen_key_time_limit: --key-time-limit 15 + jepsen_protocol_version: --protocol-version 1 + jepsen_read_concern: "" + jepsen_read_with_find_and_modify: "" + jepsen_storage_engine: "" + jepsen_test_name: "" + # Empirically, we've had greater success in reproducing the issues found in MongoDB versions + # 3.4.0-rc3 and 3.4.0-rc4 when running Jepsen with at least --time-limit=600. + jepsen_time_limit: --time-limit 1200 + jepsen_write_concern: "" + mongod_conf: --mongod-conf mongod_verbose.conf + + # Template for running Jepsen tests + - &run_jepsen_template + name: run_jepsen_template + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + + # Templates used by powercycle + - &powercycle_remote_credentials + private_key_file: src/powercycle.pem + private_key_remote: ${__project_aws_ssh_key_value} + +################################################ +# Tasks # +################################################ +tasks: + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_and_config_transitions + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_replication_with_balancer_and_config_transitions + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_replication_with_balancer_and_config_transitions + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_jstestfuzz_sharded_with_config_transitions + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: jstestfuzz_sharded_with_config_transitions + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_multi_stmt_txn_stepdown_terminate_kill_primary + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_stepdown_terminate_kill_primary_with_balancer + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_stepdown_terminate_kill_primary_with_balancer + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_initial_sync + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_initial_sync + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_initsync_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_initsync_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_concurrency_sharded_stepdown_terminate_kill_primary_with_balancer_config_shard + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_stepdown_terminate_kill_primary_with_balancer + resmoke_args: >- + --configShard=0 + --numShards=2 + --excludeWithAnyTags=config_shard_incompatible + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_config_fuzzer_stress_concurrency_sharded_replication + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: concurrency_sharded_replication + resmoke_args: >- + --fuzzMongodConfigs=stress + --fuzzMongosConfigs=normal + --fuzzRuntimeParams + --excludeWithAnyTags=does_not_support_config_fuzzer + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_core + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: core + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_fcv_upgrade_downgrade_sharded_collections_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: fcv_upgrade_downgrade_sharded_collections_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_jstestfuzz_sharded + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: jstestfuzz_sharded + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_lag_oplog_application_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_lag_oplog_application_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_kill_primary_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_kill_primary_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_kill_secondaries_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_kill_secondaries_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - <<: *antithesis_task_template + name: antithesis_replica_sets_initsync_lagged_sync_source_jscore_passthrough + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "antithesis", + ] + commands: + - func: "do setup" + - func: "do setup for antithesis" + - func: "antithesis image build and push" + vars: + suite: replica_sets_initsync_lagged_sync_source_jscore_passthrough + resmoke_args: >- + --runAllFeatureFlagTests + + - name: build_metrics_tasks_gen + tags: ["assigned_to_jira_team_devprod_build", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "do scons setup" + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/evergreen_gen_build_metrics_tasks.py" + - command: archive.targz_pack + params: + target: build_metrics_task_gen.tgz + source_dir: "src" + include: + - "build_metrics_task_gen.json" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: build_metrics_task_gen.tgz + remote_file: ${project}/${build_variant}/${revision}/build-metrics/${task_name}-${build_id}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/gzip + display_name: Build Metrics Task Gen JSON + - command: generate.tasks + params: + files: + - src/build_metrics_task_gen.json + + - name: check_feature_flag_tags + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + ] + patch_only: true + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "./src/evergreen/feature_flag_tags_check.sh" + + - name: check_for_todos + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + ] + exec_timeout_secs: 600 # 10 minute timeout + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "configure evergreen api credentials" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "./src/evergreen/todos_check.sh" + + - name: daily_task_scanner + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/daily_task_scan.py" + - "-f=task_errors.txt" + + - name: determine_patch_tests + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - func: "f_expansions_write" + - command: github.generate_token + params: + owner: 10gen + repo: coredb-patchbuild-optimizer + expansion_name: github_token + permissions: + metadata: read + contents: read + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/gen_patch_test_tags.sh" + include_expansions_in_env: + - "github_token" + + - name: generate_buildid_to_debug_symbols_mapping + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "release_critical", + "incompatible_mac", + "incompatible_windows", + "requires_compile_variant", + "symbolizer", + ] + stepback: false + patchable: true + depends_on: + - archive_dist_test_debug + commands: + - func: "f_expansions_write" + - func: "do setup" + - func: "configure evergreen api credentials" + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/generate_buildid_debug_symbols_mapping.sh" + + - name: idl_tests + tags: ["assigned_to_jira_team_server_servicearch", "default"] + depends_on: + - name: archive_dist_test + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "run idl tests" + + - name: iwyu_self_test + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + ] + commands: + - func: "do scons setup" + - func: "f_expansions_write" + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/iwyu/test/run_tests.py" + + # Smoke test to ensure the Server still works with Jepsen + - <<: *run_jepsen_template + name: jepsen-config_fuzzer_smoke + tags: ["assigned_to_jira_team_server_repl", "experimental"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + jepsen_time_limit: --time-limit 120 + + # Smoke test to ensure the Server still works with Jepsen + - <<: *run_jepsen_template + name: jepsen-smoke + tags: ["assigned_to_jira_team_server_repl", "experimental"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + jepsen_time_limit: --time-limit 120 + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_list-append + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen_docker"] + commands: + - func: "do setup" + - func: "do jepsen docker setup" + - func: "setup jepsen docker config fuzzer" + - func: "run jepsen docker test" + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_read-concern-majority + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_read-concern-majority_w1 + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + jepsen_write_concern: --write-concern w1 + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_register_findAndModify + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_read_with_find_and_modify: --read-with-find-and-modify + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: register + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_register_linearizableRead + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_read_concern: --read-concern linearizable + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: register + + - <<: *run_jepsen_template + name: jepsen_config_fuzzer_set_linearizableRead + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "setup jepsen config fuzzer" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + mongod_conf: --mongod-conf mongod.conf + jepsen_read_concern: --read-concern linearizable + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: set + + - <<: *run_jepsen_template + name: jepsen_list-append + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen_docker"] + commands: + - func: "do setup" + - func: "do jepsen docker setup" + - func: "run jepsen docker test" + + - <<: *run_jepsen_template + name: jepsen_read-concern-majority + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + + - <<: *run_jepsen_template + name: jepsen_read-concern-majority_w1 + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: read-concern-majority + jepsen_write_concern: --write-concern w1 + + - <<: *run_jepsen_template + name: jepsen_register_findAndModify + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_read_with_find_and_modify: --read-with-find-and-modify + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: register + + - <<: *run_jepsen_template + name: jepsen_register_linearizableRead + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_read_concern: --read-concern linearizable + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: register + + - <<: *run_jepsen_template + name: jepsen_set_linearizableRead + tags: ["assigned_to_jira_team_server_repl", "experimental", "jepsen"] + commands: + - func: "do setup" + - func: "do jepsen setup" + - func: "run jepsen test" + vars: + <<: *jepsen_config_vars + jepsen_read_concern: --read-concern linearizable + jepsen_storage_engine: --storage-engine wiredTiger + jepsen_test_name: set + + # TODO: rename if display_name appears on the evergreen UI + - name: bazel_run_//:format + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + "bazel_check", + ] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: timeout.update + params: + # 40 minutes + exec_timeout_secs: 2400 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. + - func: "fetch bazel" + - func: "bazel run" + vars: + target: >- + //:format -- --check + + # TODO: rename if display_name appears on the evergreen UI + - name: bazel_run_//:codeowners + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + "bazel_check", + ] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: timeout.update + params: + # 40 minutes + exec_timeout_secs: 2400 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. + - func: "fetch bazel" + - func: "bazel run" + vars: + target: >- + //:codeowners -- --check + + - name: lint_clang_format + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 2 hours + exec_timeout_secs: 7200 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + - func: "scons lint" + vars: + targets: lint-clang-format + + - name: lint_cpplint + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 2 hours + exec_timeout_secs: 7200 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + - func: "scons lint" + vars: + targets: lint-lint.py + + - name: lint_errorcodes + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + - func: "scons lint" + vars: + targets: lint-errorcodes + + - name: lint_eslint + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 2 hours + exec_timeout_secs: 7200 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + - func: "scons lint" + vars: + targets: lint-eslint + + # Check that the mutational fuzzer can parse all JS filess. + - name: lint_fuzzer_sanity_all + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "setup jstestfuzz" + - func: "lint fuzzer sanity all" + + # Check that the mutational fuzzer can parse JS files modified in a patch build. + - name: lint_fuzzer_sanity_patch + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + ] + patch_only: true + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "setup jstestfuzz" + - func: "lint fuzzer sanity patch" + + - name: lint_large_files_check + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + ] + exec_timeout_secs: 600 # 10 minute timeout + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "./src/evergreen/run_python_script.sh" + - "buildscripts/large_file_check.py" + - "--exclude" + - "src/third_party/*" + + # Confirm that the poetry.lock file is up to date with pyproject.toml + - name: lint_poetry_lock + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/functions/poetry_lock_check.py" + + - name: lint_bazel + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 40 minutes + exec_timeout_secs: 2400 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/download_buildifier.py" + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/buildifier.py" + - "--generate-report" + - "--binary-dir=./" + - "lint-all" + + # TODO: DEVPROD-9047 Enable after the issue with getting third party file is resolved + - name: lint_sbom + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 40 minutes + exec_timeout_secs: 2400 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/run_python_script.sh" + - "buildscripts/sbom_linter.py" + + - name: lint_pylinters + tags: + [ + "assigned_to_jira_team_devprod_build", + "development_critical_single_variant", + "lint", + ] + commands: + - command: timeout.update + params: + # 2 hours + exec_timeout_secs: 7200 + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + - func: "scons lint" + vars: + targets: lint-pylinters + + - name: lint_shellscripts + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + "lint", + ] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/lint_shellscripts.sh" + + - name: lint_yaml + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + "lint", + ] + depends_on: [] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "f_expansions_write" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/lint_yaml.sh" + + - name: bazel_coverage + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: timeout.update + params: + exec_timeout_secs: 10800 # 3 hours + - func: "f_expansions_write" + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "get engflow creds" + # TODO SERVER-81038: Remove "fetch bazel" once bazelisk is self-hosted. + - func: "fetch bazel" + - func: "bazel coverage" + vars: + # Running something like "bazel test //src/..." has to do a lot of preprocessing on *all* + # targets before it actually runs the much smaller fraction of test targets, unless we're + # certain that a full build has already been cached. + # It might be optimized by doing a query upfront, which uses minimal analysis: + # targets=$(bazel query 'kind(".*_test", //src/...)') + # bazel test $targets + target: >- + //src/... + args: >- + --test_output=all + --config=local + --combined_report=lcov + + - name: monitor_build_status + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/monitor_build_status_run.sh" + env: + JIRA_AUTH_ACCESS_TOKEN: ${jira_auth_access_token} + JIRA_AUTH_ACCESS_TOKEN_SECRET: ${jira_auth_access_token_secret} + JIRA_AUTH_CONSUMER_KEY: ${jira_auth_consumer_key} + JIRA_AUTH_KEY_CERT: ${jira_auth_key_cert} + + - name: monitor_mongo_fork_10gen + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "monitor mongo fork 10gen" + + - name: powercycle + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_gen + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "incompatible_windows", + ] + commands: + - func: "generate powercycle tasks" + vars: + task_names: >- + powercycle + powercycle_kill_mongod + powercycle_replication_smalloplog + powercycle_syncdelay + num_tasks: 1 + + - name: powercycle_kill_mongod + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_last_lts_fcv + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_replication + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_replication_smalloplog + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_sentinel + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "incompatible_windows", + ] + exec_timeout_secs: 86400 # 24 hours + commands: + - func: "run powercycle sentinel" + vars: + gen_task: powercycle_smoke_skip_compile_gen + + - name: powercycle_smoke + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "default", + "incompatible_tsan", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_smoke_skip_compile_gen + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "experimental", + "incompatible_windows", + ] + commands: + - func: "generate powercycle tasks" + vars: + task_names: >- + powercycle_smoke_skip_compile + num_tasks: 20 + exec_timeout_secs: 86400 # 24 hours + timeout_secs: 86400 # 24 hours + set_up_retry_count: 1800 + run_powercycle_args: --sshAccessRetryCount=1800 + + - name: powercycle_syncdelay + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: powercycle_write_concern_majority + tags: + [ + "assigned_to_jira_team_server_storage_engines", + "experimental", + "powercycle", + ] + exec_timeout_secs: 7200 # 2 hour timeout for the task overall + depends_on: + - name: archive_dist_test + commands: + - func: "do setup" + - func: "set up remote credentials" + vars: + <<: *powercycle_remote_credentials + - func: "set up EC2 instance" + vars: + <<: *powercycle_remote_credentials + - func: "run powercycle test" + vars: + <<: *powercycle_remote_credentials + timeout_secs: 1800 # 30 minute timeout for no output + + - name: selinux_rhel8_enterprise + tags: ["assigned_to_jira_team_server_security", "experimental"] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel8.8-selinux + test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js + + - name: selinux_rhel8_org + tags: ["assigned_to_jira_team_server_security", "experimental"] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel8.8-selinux + test_list: jstests/selinux/*.js + + - name: selinux_rhel9_enterprise + tags: ["assigned_to_jira_team_server_security", "experimental"] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel90-selinux + test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js + + - name: selinux_rhel9_org + tags: ["assigned_to_jira_team_server_security", "experimental"] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel90-selinux + test_list: jstests/selinux/*.js + + - name: sync_repo_with_copybara + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + patchable: false + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "configure evergreen api credentials" + - func: "sync repo with copybara" + + - name: test_api_version_compatibility + tags: + [ + "assigned_to_jira_team_server_repl", + "development_critical_single_variant", + ] + depends_on: + - name: archive_dist_test + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "do setup" + - func: "f_expansions_write" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/check_idl_compat.sh" + + - name: test_packages + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + depends_on: + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "run package test" + vars: + docker_username: ${dockerhub_username} + docker_password: ${dockerhub_password} + + - name: test_packages_release + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "run release package test" + vars: + docker_username: ${dockerhub_username} + docker_password: ${dockerhub_password} + + - name: validate_commit_message + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + ] + exec_timeout_secs: 600 # 10 minute timeout + commands: + - command: manifest.load + - func: "git get project and add git tag" + - func: "f_expansions_write" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "configure evergreen api credentials" + - func: "f_expansions_write" + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "./src/evergreen/run_python_script.sh" + - "buildscripts/validate_commit_message.py" + env: + JIRA_AUTH_ACCESS_TOKEN: ${jira_auth_access_token} + JIRA_AUTH_ACCESS_TOKEN_SECRET: ${jira_auth_access_token_secret} + JIRA_AUTH_CONSUMER_KEY: ${jira_auth_consumer_key} + JIRA_AUTH_KEY_CERT: ${jira_auth_key_cert} + IS_COMMIT_QUEUE: ${is_commit_queue} + BRANCH_NAME: ${branch_name} + + - name: version_burn_in_gen + run_on: ubuntu2004-medium + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - command: manifest.load + - func: "git get shallow project" + - func: "f_expansions_write" + - func: "restore git history and tags" + - func: "add git tag" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate version burn in" + + - name: version_gen + run_on: ubuntu2004-medium + tags: ["assigned_to_jira_team_devprod_correctness", "auxiliary"] + commands: + - command: manifest.load + - func: "git get shallow project" + - func: "f_expansions_write" + - func: "restore git history and tags" + - func: "add git tag" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate version" + + - name: version_gen_validation + run_on: ubuntu2004-medium + tags: + [ + "assigned_to_jira_team_devprod_correctness", + "development_critical_single_variant", + ] + commands: + - command: manifest.load + - func: "git get shallow project" + - func: "f_expansions_write" + - func: "restore git history and tags" + - func: "add git tag" + - func: "kill processes" + - func: "cleanup environment" + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate version validation" diff --git a/etc/evergreen_yml_components/variants/ibm.yml b/etc/evergreen_yml_components/variants/ibm.yml index 11eef6ae85d..a402008a3d9 100644 --- a/etc/evergreen_yml_components/variants/ibm.yml +++ b/etc/evergreen_yml_components/variants/ibm.yml @@ -26,7 +26,9 @@ buildvariants: multiversion_architecture: ppc64le multiversion_edition: enterprise test_flags: >- - --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer + --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible + timeout_secs: 7200 # 2 hours idle timeout + exec_timeout_secs: 43200 # 12 hours exec timeout tasks: - name: compile_test_and_package_serial_TG distros: @@ -80,7 +82,9 @@ buildvariants: multiversion_architecture: ppc64le multiversion_edition: enterprise test_flags: >- - --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer + --excludeWithAnyTags=requires_increased_memlock_limits,requires_latch_analyzer,ppc64le_incompatible + timeout_secs: 7200 # 2 hours idle timeout + exec_timeout_secs: 43200 # 12 hours exec timeout tasks: - name: compile_test_and_package_serial_TG distros: diff --git a/etc/evergreen_yml_components/variants/in_memory.yml b/etc/evergreen_yml_components/variants/in_memory.yml index d042152ba2b..a55e4cd7979 100644 --- a/etc/evergreen_yml_components/variants/in_memory.yml +++ b/etc/evergreen_yml_components/variants/in_memory.yml @@ -1,8 +1,8 @@ # InMemory build variants that need to be run on LTS but not Rapid releases. buildvariants: -- name: enterprise-rhel-80-64-bit-inmem - display_name: Enterprise RHEL 8.0 (inMemory) +- name: enterprise-rhel-8-64-bit-inmem + display_name: Enterprise RHEL 8 (inMemory) cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small @@ -11,6 +11,7 @@ buildvariants: test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise scons_cache_scope: shared large_distro_name: rhel80-large diff --git a/etc/evergreen_yml_components/variants/misc_release.yml b/etc/evergreen_yml_components/variants/misc_release.yml index b6c383172d5..717b47ebed0 100644 --- a/etc/evergreen_yml_components/variants/misc_release.yml +++ b/etc/evergreen_yml_components/variants/misc_release.yml @@ -5,7 +5,7 @@ buildvariants: display_name: Amazon Linux 2 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-test + - amazon2-latest-small expansions: test_flags: >- --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_latch_analyzer --enableEnterpriseTests=off @@ -22,14 +22,14 @@ buildvariants: packager_distro: amazon2 repo_edition: org scons_cache_scope: shared - large_distro_name: amazon2-large + large_distro_name: amazon2-latest-large tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-build + - amazon2-latest-large - name: compile_test_and_package_parallel_unittest_stream_TG distros: - - amazon2-build + - amazon2-latest-large - name: .aggfuzzer .common - name: aggregation - name: .auth !.audit !.multiversion @@ -57,7 +57,7 @@ buildvariants: display_name: "Enterprise Amazon Linux 2" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-test + - amazon2-latest-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug # TODO BUILD-13887 should fix uses_pykmip incompatibility. @@ -80,13 +80,13 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-build + - amazon2-latest-large - name: compile_test_and_package_parallel_unittest_stream_TG distros: - - amazon2-build + - amazon2-latest-large - name: compile_integration_and_test_parallel_stream_TG distros: - - amazon2-build + - amazon2-latest-large - name: .aggfuzzer .common - name: aggregation - name: audit @@ -115,7 +115,7 @@ buildvariants: display_name: Amazon Linux 2 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-arm64-small + - amazon2-arm64-latest-small expansions: push_path: linux push_bucket: downloads.mongodb.org @@ -129,21 +129,21 @@ buildvariants: packager_distro: amazon2 repo_edition: org scons_cache_scope: shared - large_distro_name: amazon2-arm64-large + large_distro_name: amazon2-arm64-latest-large tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: compile_test_and_package_parallel_unittest_stream_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: aggregation - name: .auth !.audit !.multiversion - name: causally_consistent_jscore_txns_passthrough - name: .misc_js - name: .concurrency .common distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: concurrency_replication_causal_consistency_gen - name: disk_wiredtiger - name: .jscore .common @@ -162,7 +162,7 @@ buildvariants: display_name: "Enterprise Amazon Linux 2 arm64" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - amazon2-arm64-small + - amazon2-arm64-latest-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: linux @@ -187,13 +187,13 @@ buildvariants: tasks: - name: compile_test_and_package_serial_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: compile_test_and_package_parallel_unittest_stream_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: compile_integration_and_test_parallel_stream_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: test_api_version_compatibility - name: .aggfuzzer - name: .aggregation @@ -206,48 +206,51 @@ buildvariants: - name: .concurrency !.large !.ubsan !.no_txns !.debug_only - name: .concurrency .large !.ubsan !.no_txns !.debug_only distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: config_fuzzer_concurrency - name: config_fuzzer_simulate_crash_concurrency_replication distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: config_fuzzer_concurrency_replication distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: config_fuzzer_jsCore - name: config_fuzzer_replica_sets_jscore_passthrough distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: disk_wiredtiger - name: .encrypt - name: idl_tests - name: initial_sync_fuzzer_gen - name: jsCore distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: .jscore .common !jsCore - name: jsCore_minimum_batch_size - name: jsCore_txns_large_txns_format - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .read_write_concern .large distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: .read_write_concern !.large - name: .replica_sets !.encrypt !.auth distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: replica_sets_api_version_jscore_passthrough_gen - name: replica_sets_reconfig_jscore_passthrough_gen - name: retryable_writes_jscore_passthrough_gen - name: retryable_writes_jscore_stepdown_passthrough_gen distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: .read_only - name: .rollbackfuzzer - name: sasl @@ -263,16 +266,16 @@ buildvariants: - name: snmp - name: .crypt distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: .publish_crypt - name: secondary_reads_passthrough_gen - name: server_discovery_and_monitoring_json_test_TG - name: .serverless distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: server_selection_json_test_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: .publish - name: generate_buildid_to_debug_symbols_mapping - name: vector_search @@ -563,9 +566,6 @@ buildvariants: - name: audit - name: causally_consistent_jscore_txns_passthrough - name: .encrypt !.replica_sets !.aggregation !.sharding !.jscore - - name: .jepsen_docker - distros: - - debian10-large - name: .jscore .common !.decimal !.sharding - name: .jstestfuzz .common - name: .logical_session_cache .one_sec @@ -593,8 +593,10 @@ buildvariants: test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer multiversion_platform: linux_x86_64 multiversion_edition: base - multiversion_platform_42_or_later: debian11 + multiversion_platform_42_or_later: debian10 multiversion_edition_42_or_later: targeted + multiversion_platform_last_lts: debian11 + multiversion_edition_last_lts: targeted has_packages: true packager_script: packager.py packager_arch: x86_64 @@ -646,8 +648,10 @@ buildvariants: crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: linux_x86_64 multiversion_edition: base - multiversion_platform_42_or_later: debian11 + multiversion_platform_42_or_later: debian10 multiversion_edition_42_or_later: enterprise + multiversion_platform_last_lts: debian11 + multiversion_edition_last_lts: enterprise has_packages: true packager_script: packager_enterprise.py packager_arch: x86_64 @@ -769,8 +773,8 @@ buildvariants: - name: sharding_gen - name: replica_sets_gen -- name: rhel80 - display_name: RHEL 8.0 +- name: rhel8 + display_name: RHEL 8 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -778,10 +782,11 @@ buildvariants: push_path: linux push_bucket: downloads.mongodb.org push_name: linux - push_arch: x86_64-rhel80 + push_arch: x86_64-rhel8 compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= test_flags: --enableEnterpriseTests=off --excludeWithAnyTags=requires_latch_analyzer multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: targeted has_packages: true packager_script: packager.py @@ -820,20 +825,21 @@ buildvariants: - name: .publish - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel-80-64-bit - display_name: "Enterprise RHEL 8.0" +- name: enterprise-rhel-8-64-bit + display_name: "Enterprise RHEL 8" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-small - expansions: &enterprise-rhel-80-64-bit-expansions + expansions: &enterprise-rhel-8-64-bit-expansions additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux - push_arch: x86_64-enterprise-rhel80 + push_arch: x86_64-enterprise-rhel8 compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise has_packages: true packager_script: packager_enterprise.py @@ -880,11 +886,14 @@ buildvariants: - name: json_schema - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - name: libunwind_tests - - name: mqlrun - name: .multi_shard - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - name: multiversion_gen - name: .query_fuzzer + - name: query_stats_aggregation_passthrough + - name: query_stats_mongos_aggregation_passthrough + - name: query_stats_mongos_passthrough + - name: query_stats_passthrough - name: .random_multiversion_ds - name: .read_write_concern .large distros: @@ -938,8 +947,8 @@ buildvariants: - name: vector_search_auth - name: vector_search_ssl -- name: rhel-82-arm64 - display_name: RHEL 8.2 arm64 +- name: rhel-8-arm64 + display_name: RHEL 8 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel82-arm64-small @@ -947,7 +956,7 @@ buildvariants: push_path: linux push_bucket: downloads.mongodb.org push_name: linux - push_arch: aarch64-rhel82 + push_arch: aarch64-rhel8 compile_flags: --ssl MONGO_DISTMOD=rhel82 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars --modules= has_packages: true packager_script: packager.py @@ -985,8 +994,8 @@ buildvariants: - name: .publish - name: generate_buildid_to_debug_symbols_mapping -- name: enterprise-rhel-82-arm64 - display_name: "Enterprise RHEL 8.2 arm64" +- name: enterprise-rhel-8-arm64 + display_name: "Enterprise RHEL 8 arm64" cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel82-arm64-small @@ -995,7 +1004,7 @@ buildvariants: push_path: linux push_bucket: downloads.10gen.com push_name: linux - push_arch: aarch64-enterprise-rhel82 + push_arch: aarch64-enterprise-rhel8 compile_flags: --ssl MONGO_DISTMOD=rhel82 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" has_packages: true @@ -1166,7 +1175,6 @@ buildvariants: - name: snmp - name: .crypt - name: .publish_crypt - - name: unittest_shell_hang_analyzer_gen - name: vector_search - name: vector_search_auth - name: vector_search_ssl @@ -1355,6 +1363,7 @@ buildvariants: repo_edition: enterprise scons_cache_scope: shared test_flags: --excludeWithAnyTags=requires_latch_analyzer + large_distro_name: suse12-sp5-large tasks: - name: compile_test_and_package_serial_TG distros: @@ -1887,6 +1896,31 @@ buildvariants: - name: .publish - name: generate_buildid_to_debug_symbols_mapping + +- name: enterprise-ubuntu2204-jepsen + display_name: Jepsen Tests on Enterprise Ubuntu 22.04 + tags: ["bazel_check"] + cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. + run_on: + - ubuntu2204-small + stepback: false + expansions: + compile_flags: >- + --ssl + MONGO_DISTMOD=ubuntu2204 + -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + scons_cache_scope: shared + large_distro_name: ubuntu2204-large + compile_variant: enterprise-ubuntu2204-64 + tasks: + - name: compile_test_and_package_serial_TG + distros: + - ubuntu2204-large + - name: .jepsen_docker + distros: + - ubuntu2204-large + - name: enterprise-ubuntu2004-arm64 display_name: Enterprise Ubuntu 20.04 arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. @@ -2183,7 +2217,7 @@ buildvariants: display_name: macOS cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_latch_analyzer --enableEnterpriseTests=off push_path: osx @@ -2213,7 +2247,7 @@ buildvariants: display_name: macOS arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_latch_analyzer --enableEnterpriseTests=off push_path: osx @@ -2243,7 +2277,7 @@ buildvariants: display_name: Enterprise macOS cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100 + - macos-11 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_latch_analyzer additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug @@ -2262,7 +2296,6 @@ buildvariants: - name: auth_audit_gen - name: .encrypt !.replica_sets !.sharding !.aggregation !.jscore - name: .logical_session_cache .one_sec - - name: mqlrun - name: replica_sets_auth_gen - name: sasl - name: push @@ -2273,7 +2306,7 @@ buildvariants: display_name: Enterprise macOS arm64 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - - macos-1100-arm64 + - macos-11-arm64 expansions: test_flags: --excludeWithAnyTags=incompatible_with_macos,requires_gcm,requires_latch_analyzer additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug @@ -2294,7 +2327,6 @@ buildvariants: - name: fle - name: fle2 - name: .logical_session_cache .one_sec - - name: mqlrun - name: replica_sets_auth_gen - name: sasl - name: push @@ -2305,18 +2337,25 @@ buildvariants: display_name: Amazon Linux 2 arm64 Query Patch Only (trySbeEngine) cron: "0 4 * * 0" # From the ${project_weekly_cron} parameter. run_on: - - amazon2-arm64-small + - amazon2-arm64-latest-small expansions: additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug push_path: linux push_bucket: downloads.10gen.com push_name: linux push_arch: aarch64-enterprise-amazon2 - compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + # Diagnostic latches are disabled by default, and we match that behavior here. We also need to + # exclude tests with 'requires_latch_analyzer'. + compile_flags: >- + --ssl + MONGO_DISTMOD=amazon2 + -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + --use-diagnostic-latches=off crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" test_flags: >- - --mongodSetParameters="{featureFlagSbeFull: true}" - --excludeWithAnyTags=resource_intensive,incompatible_with_amazon_linux,sbe_incompatible + --mongodSetParameters="{featureFlagSbeFull: true, internalQueryForceClassicEngine: false}" + --excludeWithAnyTags=resource_intensive,incompatible_with_amazon_linux,sbe_incompatible,requires_latch_analyzer has_packages: true multiversion_platform: amazon2 multiversion_edition: enterprise @@ -2328,11 +2367,11 @@ buildvariants: packager_distro: amazon2 repo_edition: enterprise scons_cache_scope: shared - large_distro_name: amazon2-arm64-large + large_distro_name: amazon2-arm64-latest-large tasks: - name: compile_and_archive_dist_test_then_package_TG distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: aggregation_mongos_passthrough - name: aggregation_sharded_collections_passthrough - name: aggregation_one_shard_sharded_collections @@ -2342,7 +2381,7 @@ buildvariants: - name: noPassthroughWithMongod_gen - name: jsCore distros: - - amazon2-arm64-large + - amazon2-arm64-latest-large - name: replica_sets_jscore_passthrough_gen - name: sharded_collections_jscore_passthrough_gen - name: sharding_jscore_passthrough @@ -2350,3 +2389,155 @@ buildvariants: - name: concurrency_gen - name: concurrency_replication_gen - name: concurrency_sharded_replication_gen + +# This task is responsible for creating a AL2023 x86 server binary candidate for 10gen/mongot. +# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting +# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. +- name: amazon-linux-2023-x86-mongot-integration-cron-only + display_name: "AL2023 x86 mongot integration tasks cron only" + cron: "0 */4 * * *" # Run these tasks every 4 hours + patchable: false + run_on: + - amazon2023.0-small + expansions: + push_path: linux + push_bucket: downloads.10gen.com + push_name: linux + push_arch: x86_64-enterprise-amazon2023 + compile_flags: --ssl MONGO_DISTMOD=amazon2023 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" + # TODO BUILD-13887 should fix uses_pykmip incompatibility. + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,uses_pykmip,requires_v4_0,requires_external_data_source,requires_latch_analyzer + has_packages: false + multiversion_platform: amazon2023 + multiversion_edition: enterprise + multiversion_architecture: x86_64 + packager_script: packager_enterprise.py + packager_arch: x86_64 + packager_distro: amazon2023 + repo_edition: enterprise + scons_cache_scope: shared + compile_variant: amazon-linux-2023-x86-mongot-integration-cron-only + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon2023.0-large + - name: search + - name: search_auth + - name: search_pinned_connections_auth + - name: search_ssl + - name: vector_search + - name: vector_search_auth + - name: vector_search_ssl + +# This task is responsible for creating a AL2023 arm64 server binary candidate for 10gen/mongot. +# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting +# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. +- name: amazon2023-arm64-mongot-integration-cron-only + display_name: "AL2023 arm64 mongot integration tasks cron only" + cron: "0 */4 * * *" # Run these tasks every 4 hours + patchable: false + run_on: + - amazon2023.0-arm64-small + expansions: + compile_flags: >- + --ssl + MONGO_DISTMOD=amazon2023 + -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars --enterprise-features="audit,encryptdb,fcbis,fips,fle,hot_backups,inmemory,kerberos,vector_search,ldap,live_import,mongohouse,queryable,sasl,search,serverless" + test_flags: >- + --excludeWithAnyTags=incompatible_with_amazon_linux,requires_external_data_source,requires_latch_analyzer + --enableEnterpriseTests=off + has_packages: false + multiversion_platform: amazon2023 + multiversion_edition: enterprise + multiversion_architecture: aarch64 + packager_script: packager_enterprise.py + packager_arch: aarch64 + packager_distro: amazon2023 + repo_edition: enterprise + scons_cache_scope: shared + compile_variant: amazon2023-arm64-mongot-integration-cron-only + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon2023.0-arm64-large + - name: search + - name: search_auth + - name: search_pinned_connections_auth + - name: search_ssl + - name: vector_search + - name: vector_search_auth + - name: vector_search_ssl + +# This task is responsible for creating a AL2 arm64 server binary candidate for 10gen/mongot. +# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting +# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests. +- name: amazon-linux2-arm64-mongot-integration-cron-only + display_name: "AL2 arm64 mongot integration tasks cron only" + cron: "0 */4 * * *" # Run these tasks every 4 hours + patchable: false + run_on: + - amazon2-arm64-small + stepback: true + expansions: + compile_flags: >- + --ssl + MONGO_DISTMOD=amazon2 + -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + scons_cache_scope: shared + scons_cache_mode: all + has_packages: false + target_resmoke_time: 10 + idle_timeout_factor: 1.5 + exec_timeout_factor: 1.5 + compile_variant: amazon-linux2-arm64-mongot-integration-cron-only + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon2-arm64-large + - name: search + - name: search_auth + - name: search_pinned_connections_auth + - name: search_ssl + - name: vector_search + - name: vector_search_auth + - name: vector_search_ssl + +# This task is responsible for creating a AL2 x86 server binary candidate for 10gen/mongot. +# A project trigger defined on 10gen/mongot runs when this variant succeeds, to check if the resulting +# binary candidate passes their tests and is therefore safe to use in their pre-commit integration tests +- name: amazon2-x86-mongot-integration-cron-only + display_name: "AL2 x86 mongot integration tasks cron only" + cron: "0 */4 * * *" # Run these tasks every 4 hours + patchable: false + run_on: + - amazon2-large + expansions: + compile_flags: >- + --ssl + MONGO_DISTMOD=amazon2 + -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars + test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source,requires_latch_analyzer + has_packages: false + multiversion_platform: amazon2 + multiversion_edition: enterprise + multiversion_architecture: x86_64 + packager_script: packager_enterprise.py + packager_arch: x86_64 + packager_distro: amazon2 + repo_edition: enterprise + scons_cache_scope: shared + compile_variant: amazon2-x86-mongot-integration-cron-only + tasks: + - name: compile_test_and_package_serial_TG + distros: + - amazon2-large + - name: search + - name: search_auth + - name: search_pinned_connections_auth + - name: search_ssl + - name: vector_search + - name: vector_search_auth + - name: vector_search_ssl diff --git a/etc/evergreen_yml_components/variants/ninja.yml b/etc/evergreen_yml_components/variants/ninja.yml index b19d9729c9c..4dbe6fb576f 100644 --- a/etc/evergreen_yml_components/variants/ninja.yml +++ b/etc/evergreen_yml_components/variants/ninja.yml @@ -17,7 +17,7 @@ buildvariants: display_name: "Ninja Build: macOS Enterprise" cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds run_on: - - macos-1100 + - macos-11 expansions: compile_env: DEVELOPER_DIR=/Applications/Xcode13.app compile_flags: --ssl -j$(sysctl -n hw.logicalcpu) --libc++ --variables-files=etc/scons/xcode_macosx.vars @@ -50,8 +50,8 @@ buildvariants: distros: - ubuntu1804-xlarge -- name: enterprise-rhel-80-64-bit-dynamic-required-ninja - display_name: "Ninja Build: Enterprise RHEL 8.0" +- name: enterprise-rhel-8-64-bit-dynamic-required-ninja + display_name: "Ninja Build: Enterprise RHEL 8" cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds run_on: - rhel80-small @@ -66,8 +66,8 @@ buildvariants: distros: - rhel80-xlarge -- name: enterprise-rhel80-ninja - display_name: "Ninja Build: RHEL 8.0" +- name: enterprise-rhel8-ninja + display_name: "Ninja Build: RHEL 8" cron: "0 4 * * 0" # Run once a week to ensure no failures introduced to ninja builds run_on: - rhel80-small diff --git a/etc/evergreen_yml_components/variants/openssl_111.yml b/etc/evergreen_yml_components/variants/openssl_111.yml deleted file mode 100644 index 08836cd736b..00000000000 --- a/etc/evergreen_yml_components/variants/openssl_111.yml +++ /dev/null @@ -1,173 +0,0 @@ -buildvariants: -- name: enterprise-amazon2-openssl111 - display_name: "Enterprise Amazon Linux 2 (openssl 1.1.1)" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2-openssl111-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug - test_flags: >- - --excludeWithAnyTags=SERVER-34286,incompatible_with_amazon_linux,requires_external_data_source,requires_latch_analyzer - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: x86_64-enterprise-amazon2-openssl111 - compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - multiversion_platform: amazon2 - multiversion_edition: enterprise - has_packages: true - packager_script: packager_enterprise.py - packager_arch: x86_64 - packager_distro: amazon2 - repo_edition: enterprise - scons_cache_scope: shared - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2-openssl111-large - - name: compile_integration_and_test_parallel_stream_TG - distros: - - amazon2-openssl111-large - - name: .aggfuzzer .common - - name: aggregation - - name: audit - - name: .auth !.multiversion - - name: causally_consistent_jscore_txns_passthrough - - name: .encrypt !.aggregation - - name: .jscore .common - - name: .jstestfuzz .common - - name: libunwind_tests - - name: .logical_session_cache .one_sec - - name: noPassthrough_gen - - name: noPassthroughWithMongod_gen - - name: .replica_sets .common - - name: sasl - - name: serial_run - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: .sharding .txns !.csrs - - name: slow1_gen - - name: snmp - - name: .stitch - - name: .crypt - - name: crypt_push - - name: push - - name: generate_buildid_to_debug_symbols_mapping - -- name: enterprise-amazon2-arm64-openssl111 - display_name: "Enterprise Amazon Linux 2 arm64 (openssl 1.1.1)" - cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. - run_on: - - amazon2-arm64-openssl111-small - expansions: - additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug - push_path: linux - push_bucket: downloads.10gen.com - push_name: linux - push_arch: aarch64-enterprise-amazon2-openssl111 - compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - crypt_task_compile_flags: SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique" CCFLAGS="-fno-gnu-unique" - test_flags: --excludeWithAnyTags=incompatible_with_amazon_linux,requires_ldap_pool,requires_v4_0,requires_external_data_source,requires_latch_analyzer - has_packages: true - multiversion_platform: amazon2 - multiversion_edition: enterprise - multiversion_architecture: arm64 - multiversion_architecture_42_or_later: aarch64 - multiversion_skip_install_for_40: "true" - packager_script: packager_enterprise.py - packager_arch: aarch64 - packager_distro: amazon2 - repo_edition: enterprise - scons_cache_scope: shared - tasks: - - name: compile_test_and_package_serial_TG - distros: - - amazon2-arm64-openssl111-large - - name: compile_integration_and_test_parallel_stream_TG - distros: - - amazon2-arm64-openssl111-large - - name: test_api_version_compatibility - - name: .aggfuzzer - - name: .aggregation - - name: audit - - name: .auth - - name: .causally_consistent !.sharding - - name: .change_streams - - name: .change_stream_fuzzer - - name: .misc_js - - name: .concurrency !.large !.ubsan !.no_txns !.debug_only - - name: .concurrency .large !.ubsan !.no_txns !.debug_only - distros: - - amazon2-arm64-openssl111-large - - name: config_fuzzer_concurrency - - name: config_fuzzer_simulate_crash_concurrency_replication - distros: - - amazon2-arm64-openssl111-large - - name: config_fuzzer_concurrency_replication - distros: - - amazon2-arm64-openssl111-large - - name: config_fuzzer_jsCore - - name: config_fuzzer_replica_sets_jscore_passthrough - distros: - - amazon2-arm64-openssl111-large - - name: disk_wiredtiger - - name: .encrypt - - name: idl_tests - - name: initial_sync_fuzzer_gen - - name: jsCore - distros: - - amazon2-arm64-openssl111-large - - name: .jscore .common !jsCore - - name: jsCore_minimum_batch_size - - name: jsCore_txns_large_txns_format - - name: json_schema - - name: .jstestfuzz !.flow_control # Flow control jstestfuzz take longer. - - name: libunwind_tests - - name: mqlrun - - name: .multi_shard - - name: multi_stmt_txn_jscore_passthrough_with_migration_gen - - name: .query_fuzzer - - name: .read_write_concern .large - distros: - - amazon2-arm64-openssl111-large - - name: .read_write_concern !.large - - name: .replica_sets !.encrypt !.auth - distros: - - amazon2-arm64-openssl111-large - - name: replica_sets_api_version_jscore_passthrough_gen - - name: replica_sets_reconfig_jscore_passthrough_gen - - name: retryable_writes_jscore_passthrough_gen - - name: retryable_writes_jscore_stepdown_passthrough_gen - distros: - - amazon2-arm64-openssl111-large - - name: .read_only - - name: .rollbackfuzzer - - name: sasl - - name: search - - name: search_auth - - name: search_pinned_connections_auth - - name: search_ssl - - name: session_jscore_passthrough - - name: .sharding .jscore !.wo_snapshot !.multi_stmt - - name: sharding_api_version_jscore_passthrough_gen - - name: .sharding .txns - - name: .sharding .common - - name: snmp - - name: .stitch - - name: .crypt - distros: - - amazon2-arm64-openssl111-large - - name: secondary_reads_passthrough_gen - - name: server_discovery_and_monitoring_json_test_TG - - name: .serverless - distros: - - amazon2-arm64-openssl111-large - - name: server_selection_json_test_TG - distros: - - amazon2-arm64-openssl111-large - - name: crypt_push - - name: push - - name: generate_buildid_to_debug_symbols_mapping - - name: vector_search - - name: vector_search_auth - - name: vector_search_ssl diff --git a/etc/evergreen_yml_components/variants/sanitizer.yml b/etc/evergreen_yml_components/variants/sanitizer.yml index 60ff0935d59..14a982998ff 100644 --- a/etc/evergreen_yml_components/variants/sanitizer.yml +++ b/etc/evergreen_yml_components/variants/sanitizer.yml @@ -3,8 +3,8 @@ buildvariants: -- name: rhel80-asan - display_name: ~ ASAN RHEL 8.0 +- name: rhel8-asan + display_name: ~ ASAN RHEL 8 cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -14,6 +14,7 @@ buildvariants: san_options: LSAN_OPTIONS="suppressions=etc/lsan.suppressions:report_objects=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=true:strict_init_order=true:abort_on_error=1:disable_coredump=0:handle_abort=1:external_symbolizer_path=/opt/mongodbtoolchain/v3/bin/llvm-symbolizer" compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. hang_analyzer_dump_core: false @@ -26,8 +27,8 @@ buildvariants: - name: .aggfuzzer .common - name: .jstestfuzz !.initsync -- name: rhel80-debug-asan - display_name: ~ ASAN Enterprise RHEL 8.0 DEBUG +- name: rhel8-debug-asan + display_name: ~ ASAN Enterprise RHEL 8 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -41,6 +42,7 @@ buildvariants: compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --allocator=system --sanitize=address --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) test_flags: --excludeWithAnyTags=requires_fast_memory,requires_ocsp_stapling,requires_increased_memlock_limits,requires_latch_analyzer multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under ASAN build. hang_analyzer_dump_core: false @@ -98,8 +100,8 @@ buildvariants: - name: server_discovery_and_monitoring_json_test_TG - name: server_selection_json_test_TG -- name: rhel80-debug-ubsan - display_name: ~ UBSAN Enterprise RHEL 8.0 DEBUG +- name: rhel8-debug-ubsan + display_name: ~ UBSAN Enterprise RHEL 8 DEBUG cron: "0 4 * * *" # From the ${project_nightly_cron} parameter. run_on: - rhel80-build @@ -113,6 +115,7 @@ buildvariants: compile_flags: --variables-files=etc/scons/mongodbtoolchain_v3_clang.vars --dbg=on --opt=on --sanitize=undefined --ssl --ocsp-stapling=off -j$(grep -c ^processor /proc/cpuinfo) --use-diagnostic-latches=on test_flags: --excludeWithAnyTags=requires_ocsp_stapling,requires_increased_memlock_limits multiversion_platform: rhel80 + multiversion_platform_50_or_later: rhel8 multiversion_edition: enterprise resmoke_jobs_factor: 0.3 # Avoid starting too many mongod's under UBSAN build. scons_cache_scope: shared @@ -130,7 +133,7 @@ buildvariants: - name: .causally_consistent !.wo_snapshot - name: .change_streams - name: .misc_js - - name: .concurrency !.no_txns !.repl !.kill_terminate + - name: .concurrency !.no_txns !.kill_terminate - name: disk_wiredtiger - name: .encrypt - name: initial_sync_fuzzer_gen diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml index fa010996291..1f7a0294a28 100644 --- a/etc/evergreen_yml_components/variants/task_generation.yml +++ b/etc/evergreen_yml_components/variants/task_generation.yml @@ -1,8 +1,14 @@ # Build variant to generate tasks for evergreen versions. +# +# Updates to this file may also need to appear in etc/system_perf_yml_components/variants/task_generation.yml, +# which is the same but excludes resmoke task generation tasks. +# buildvariants: - name: generate-tasks-for-version display_name: "Generate tasks for evergreen version" + batchtime: 0 + activate: true run_on: - rhel80-medium tasks: diff --git a/etc/generate_subtasks_config.yml b/etc/generate_subtasks_config.yml index 07113ca91e3..7e8384e5e6e 100644 --- a/etc/generate_subtasks_config.yml +++ b/etc/generate_subtasks_config.yml @@ -25,10 +25,10 @@ build_variant_large_distro_exceptions: - enterprise-rhel-71-ppc64le-inmem - enterprise-rhel-72-s390x - enterprise-rhel-72-s390x-inmem - - enterprise-rhel-80-64-bit - - enterprise-rhel-80-64-bit-coverage + - enterprise-rhel-8-64-bit + - enterprise-rhel-8-64-bit-coverage - enterprise-rhel-81-ppc64le - - enterprise-rhel-82-arm64 + - enterprise-rhel-8-arm64 - enterprise-rhel-83-s390x - enterprise-rhel-90-64-bit - enterprise-rhel-90-arm64 @@ -46,8 +46,8 @@ build_variant_large_distro_exceptions: - macos-arm64 - macos-debug-suggested - rhel70 - - rhel80 - - rhel-82-arm64 + - rhel8 + - rhel-8-arm64 - rhel90 - rhel90-arm64 - suse12 @@ -58,8 +58,8 @@ build_variant_large_distro_exceptions: - ubuntu1604-debug - ubuntu1804-debug-asan - ubuntu1804-debug-asan-all-feature-flags - - ubuntu1804-debug-asan-classic-engine + - ubuntu1804-debug-asan-sbe-engine - ubuntu1804-debug-aubsan-lite-required - ubuntu1804-debug-ubsan - ubuntu1804-debug-ubsan-all-feature-flags - - ubuntu1804-debug-ubsan-classic-engine + - ubuntu1804-debug-ubsan-sbe-engine diff --git a/etc/lsan.suppressions b/etc/lsan.suppressions index 43e72a8ee44..6eb6406e5db 100644 --- a/etc/lsan.suppressions +++ b/etc/lsan.suppressions @@ -18,3 +18,10 @@ leak:__cxa_thread_atexit_impl leak:mongo::ThreadPool::Impl::_startWorkerThread_inlock() leak:glob64 + +# Allocations from this function in libresolv are stored in thread-locals. +# Threads that are not joined by the time the leak-detector runs (like those spawned by ASIO) may +# still have pointers to these allocated objects but LSAN isn't able to identify them and add them +# to do the root-set, resulting in false-positive leak reports. +# TODO SERVER-65936: Re-evaluate if this is needed with a clean shutdown that joins all threads. +leak:__res_context_send diff --git a/etc/pip/components/jiraclient.req b/etc/pip/components/jiraclient.req index 760a5c46684..106e78f9de3 100644 --- a/etc/pip/components/jiraclient.req +++ b/etc/pip/components/jiraclient.req @@ -1,5 +1,5 @@ # TIG jira integration jira <= 3.1.1 requests-oauth <= 0.4.1 -PyJWT <= 2.3.0 # https://github.com/pycontribs/jira/issues/247 +PyJWT == 2.4.0 # We are omitting pycrypto based on https://github.com/pycontribs/jira/pull/629 diff --git a/etc/pip/components/testing.req b/etc/pip/components/testing.req index 120dbbfd183..e2d8e1170e2 100644 --- a/etc/pip/components/testing.req +++ b/etc/pip/components/testing.req @@ -11,7 +11,7 @@ itsdangerous == 2.0.0 ocspbuilder == 0.10.2 Werkzeug == 2.0.3 blackduck == 1.0.1 -PyGithub == 1.53 +PyGithub == 1.58 urllib3 >= 1.26.0, <= 1.26.7 distro == 1.5.0 dnspython == 2.1.0 @@ -20,3 +20,4 @@ pkce == 1.0.3 oauthlib == 3.1.1 requests-oauthlib == 1.3.0 packaging <= 21.3 +retry == 0.9.2 diff --git a/etc/system_perf.yml b/etc/system_perf.yml index 9218d3e3f5e..752891122c8 100755 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -1,6 +1,15 @@ command_type: system stepback: false +exec_timeout_secs: &exec_timeout_secs 21600 +timeout_secs: &timeout_secs 7200 +include: + - filename: evergreen/system_perf/6.0/variants.yml + module: dsi + - filename: evergreen/system_perf/shared_tasks.yml + module: dsi + - filename: evergreen/system_perf/6.0/genny_tasks.yml + module: genny ## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds) parameters: @@ -9,46 +18,18 @@ parameters: # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects - key: reuse_compile_from description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_6.0_90c65f9cc8fc4e6664a5848230abaa9b3f3b02f7" + - key: exec_timeout_secs_override + value: *exec_timeout_secs + description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task" + - key: timeout_secs_override + value: *timeout_secs + description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task" variables: - ### - # Leave this section uncommented to enable compile. - _real_compile_amazon2: &_compile_amazon2 - - name: compile - variant: compile-amazon2 - - name: schedule_global_auto_tasks - variant: task_generation - _real_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64 - - name: compile - variant: compile-amazon-linux2-arm64 - - name: schedule_global_auto_tasks - variant: task_generation - _real_expansions: &_expansion_updates - [] - ### - -### -# **Or**: Leave this section uncommented to bypass/skip compile. -# _skip_compile_amazon2: &_compile_amazon2 -# - name: schedule_global_auto_tasks -# variant: task_generation -# _skip_compile_amazon_linux2_arm64: &_compile_amazon_linux2_arm64 -# - name: schedule_global_auto_tasks -# variant: task_generation -# _skip_expansions: &_expansion_updates -# # This is the normal (amazon2) "compile" artifact from https://evergreen.mongodb.com/version/sys_perf_6.0_198cd42e240e7ae8d2cc06250dccb67deda169ad -# - key: mdb_binary_for_client -# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_6.0_198cd42e240e7ae8d2cc06250dccb67deda169ad/198cd42e240e7ae8d2cc06250dccb67deda169ad/linux/mongodb-sys_perf_6.0_198cd42e240e7ae8d2cc06250dccb67deda169ad.tar.gz -# - key: mdb_binary_for_server -# value: https://mciuploads.s3.amazonaws.com/dsi/sys_perf_6.0_198cd42e240e7ae8d2cc06250dccb67deda169ad/198cd42e240e7ae8d2cc06250dccb67deda169ad/linux/mongodb-sys_perf_6.0_198cd42e240e7ae8d2cc06250dccb67deda169ad.tar.gz - -### - _src_dir: &src_dir src/mongo _modules: &modules - mongo-tools - # - mongo - dsi - genny - workloads @@ -62,12 +43,18 @@ variables: - flamegraph + _shared_compile_expansions: &shared_compile_expansions + platform: linux + project_dir: dsi + scons_cache_scope: shared + scons_cache_mode: all + has_packages: false + +### +# Same in every DSI project. Ensure that this block is synchronized with +# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here) +# in the dsi repo, and etc/system_perf.yml in mongodb/mongo. modules: - ### - # Same in every DSI project. Ensure that this block is synchronized with - # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py - # (search update-repos-here) in this repo, and etc/system_perf.yml and - # etc/perf.yml in mongodb/mongo - name: dsi owner: 10gen repo: dsi @@ -107,7 +94,7 @@ modules: owner: mongodb-labs repo: YCSB prefix: ${workdir}/src - branch: production + branch: main - name: py-tpcc owner: mongodb-labs repo: py-tpcc @@ -118,11 +105,6 @@ modules: repo: flamegraph prefix: ${workdir}/src branch: master -# - name: mongo -# repo: git@github.com:mongodb/mongo.git -# prefix: ${workdir}/src -# branch: master - ### - name: mongo-tools owner: mongodb repo: mongo-tools @@ -133,6 +115,7 @@ modules: repo: PrivateWorkloads prefix: ${workdir}/src branch: production +### ### # Same in every DSI project @@ -149,103 +132,6 @@ timeout: functions: - ### - # Same in every DSI project - f_dsi_pre_run: - - command: manifest.load - - command: expansions.update - params: - updates: *_expansion_updates - f_dsi_run_workload: &dsi_run_func - - command: git.get_project - params: - directory: *src_dir - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - mongo-perf: ${mongo-perf_rev} - YCSB: ${YCSB_rev} - py-tpcc: ${py-tpcc_rev} - flamegraph: ${flamegraph_rev} - # mongo: ${mongo_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - redacted: true - - command: shell.exec - params: - script: ./src/dsi/run-dsi run_workload - - command: shell.exec - type: system - params: - script: ./src/dsi/run-dsi determine_failure -m SYSTEM - - command: shell.exec - type: setup - params: - script: ./src/dsi/run-dsi determine_failure -m SETUP - - command: shell.exec - type: test - params: - script: ./src/dsi/run-dsi determine_failure -m TEST - f_run_dsi_workload: *dsi_run_func # Do not use this function. It is deprecated. - - f_dsi_post_run: - - command: shell.exec - params: - script: ./src/dsi/run-dsi post_run - - command: perf.send - params: - file: ./build/CedarReports/cedar_report.json - aws_key: ${terraform_key} - aws_secret: ${terraform_secret} - bucket: genny-metrics - region: us-east-1 - prefix: ${task_id}_${execution} - - command: attach.results - params: - file_location: ./build/EvergreenResultsJson/results.json - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ./build/Artifacts/DSIArtifacts.tgz - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/x-gzip - display_name: DSI Artifacts - Execution ${execution} - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ./build/Documentation/index.html - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: Documentation - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: bootstrap.yml - remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/bootstrap-${task_name}-${build_id}-${execution}.yml - bucket: mciuploads - permissions: public-read - content_type: text/plain - display_name: Task Bootstrap Config - f_dsi_timeout: - - command: shell.exec - params: - script: ./src/dsi/run-dsi on_timeout - ### - f_other_post_ops: - command: shell.exec params: @@ -459,103 +345,7 @@ functions: display_name: mongo-debugsymbols.tgz ### - ## Schedule Tasks ## - f_schedule_tasks: - - command: git.get_project - params: - directory: *src_dir - clone_depth: 1000 - revisions: - dsi: ${dsi_rev} - genny: ${genny_rev} - linkbench: ${linkbench_rev} - linkbench2: ${linkbench2_rev} - tsbs: ${tsbs_rev} - workloads: ${workloads_rev} - mongo-perf: ${mongo-perf_rev} - YCSB: ${YCSB_rev} - py-tpcc: ${py-tpcc_rev} - PrivateWorkloads: ${PrivateWorkloads_rev} - - command: expansions.write - params: - file: ./expansions.yml - - command: shell.exec - params: - script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks} - - command: generate.tasks - params: - files: - - build/TaskJSON/Tasks.json - - tasks: - ### - # Same in every DSI project - - name: schedule_global_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: all_tasks - - name: schedule_variant_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: variant_tasks - - name: schedule_patch_auto_tasks - priority: 5 - commands: - - func: f_schedule_tasks - vars: - tasks: patch_tasks - - name: smoke_test - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - - name: canaries_only - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: canaries - - name: smoke_test_ssl - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-ssl - infrastructure_provisioning: replica - - name: smoke_test_standalone_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: standalone-auth - infrastructure_provisioning: single - - name: smoke_test_replset_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: replica-auth - infrastructure_provisioning: replica - - name: smoke_test_shard_lite_auth - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: short - mongodb_setup: shard-lite-auth - infrastructure_provisioning: shard-lite - - ### - - name: compile commands: - command: manifest.load @@ -566,809 +356,7 @@ tasks: mongo-tools: ${mongo-tools_rev} - func: "compile mongodb" - - name: linkbench - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench" - - - name: linkbench_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_stepdowns" - - - name: linkbench_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_rolling_restarts" - - - name: linkbench_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_stepdowns" - - - name: linkbench_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench_non_retryable_writes_rolling_restarts" - - - name: linkbench2 - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "linkbench2" - additional_tfvars: "tags: {expire-on-delta: 12}" - - - name: tsbs_load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_load" - - - name: tsbs_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query" - - - name: tsbs_query_manual_bucketing - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tsbs_query_manual_bucketing" - - - name: tpcc - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc" - - - name: tpcc_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpcc_majority" - - - name: industry_benchmarks - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb" - - - name: ycsb_60GB - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB" - - - name: industry_benchmarks_secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads" - - - name: industry_benchmarks_wmajority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-wmajority" - - - name: industry_benchmarks_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_stepdowns" - - - name: industry_benchmarks_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_rolling_restarts" - - - name: industry_benchmarks_non_retryable_writes_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_stepdowns" - - - name: industry_benchmarks_non_retryable_writes_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb_non_retryable_writes_rolling_restarts" - - - name: ycsb.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.2023-09" - - - name: ycsb_w1.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-w1.2023-09" - - - name: ycsb_60GB.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.2023-09" - - - name: ycsb_60GB.long.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-60GB.long.2023-09" - - - name: ycsb_secondary_reads.2023-09 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb-secondary-reads.2023-09" - - - name: ycsb.load - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "ycsb.load" - - - name: crud_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads" - - - name: crud_workloads_majority - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_majority" - - - name: crud_workloads_w1 - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "crud_workloads_w1.2023-02" - - - name: cursor_manager - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "cursor_manager" - - - name: mixed_workloads_genny_stepdowns - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_stepdowns" - - - name: mixed_workloads_genny_rolling_restarts - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mixed_workloads_genny_rolling_restarts" - - - name: big_update_10k - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "BigUpdate10k" - - - name: misc_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "misc_workloads" - - - - name: map_reduce_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "map_reduce_workloads" - - - name: genny_canaries - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "genny_canaries" - - - name: retryable_writes_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "retryable_writes" - - - name: snapshot_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "snapshot_reads" - - - name: secondary_reads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "secondary_reads" - - - name: bestbuy_agg - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg" - - - name: bestbuy_agg_merge_same_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_same_db" - - - name: bestbuy_agg_merge_different_db - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_different_db" - - - name: bestbuy_agg_merge_target_hashed - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_target_hashed" - - - name: bestbuy_agg_merge_wordcount - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_agg_merge_wordcount" - - - name: bestbuy_query - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "bestbuy_query" - - - name: tpch_1_normalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: normalized} - - - name: tpch_1_denormalized - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 1, - schema: denormalized} - - - name: tpch_10_normalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: normalized} - - - name: tpch_10_denormalized - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - vars: - test_control: "tpch" - test_control_params: | - {scale: 10, - schema: denormalized} - - - name: non_sharded_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "non_sharded" - - - name: mongos_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos" - - - name: mongos_large_catalog_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "mongos_large_catalog" - - - name: move_chunk_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk" - - - name: move_chunk_waiting_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_waiting" - - - name: move_chunk_large_chunk_map_workloads - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "move_chunk_large_chunk_map" - - - name: refine_shard_key_transaction_stress - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "refine_shard_key_transaction_stress" - - - name: secondary_performance - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - # Unfortunately the dash/underscore style is different for mongodb_setup and test_control - test_control: "secondary_performance" - mongodb_setup: "secondary-performance" - - - name: initialsync - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - - - name: initialsync-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync" - mongodb_setup: "replica-2node-fcbis" - - - name: initialsync-logkeeper - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - - - name: initialsync-logkeeper-fcbis - priority: 5 - exec_timeout_secs: 43200 # 12 hours - commands: - - func: f_dsi_run_workload - timeout_secs: 43200 # 12 hours - vars: - test_control: "initialsync-logkeeper" - mongodb_setup: "initialsync-logkeeper-fcbis" - - # The following two initial sync logkeeper automation tasks are only used in the commented-out - # "Linux ReplSet Initial Sync LogKeeper Snapshot Update" variant below and are only intended to be - # run in patch builds to update FCV for logkeeper datasets. - - - name: initialsync-logkeeper-snapshot-update - priority: 5 - exec_timeout_secs: 216000 # 2.5 days - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-logkeeper-snapshot-update" - - - name: initialsync-large - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - - - name: initialsync-large-fcbis - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "initialsync-large" - mongodb_setup: "replica-2node-fcbis" - - - name: change_streams_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_latency" - - - name: change_streams_preimage_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_throughput" - - - name: change_streams_preimage_latency - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_preimage_latency" - - - name: change_streams_listen_throughput - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_listen_throughput" - - - name: change_streams_multi_mongos - priority: 5 - commands: - - func: f_dsi_run_workload - vars: - test_control: "change_streams_multi_mongos" - - - name: genny_execution_UserAcquisition - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/execution/UserAcquisition.yml - - name: genny_scale_InsertRemove - commands: - - func: f_dsi_run_workload - vars: - test_control: auto_genny_workload - auto_workload_path: ./src/genny/dist/etc/genny/workloads/scale/InsertRemove.yml - - name: query_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: query_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_large_dataset, - include_filter_2: regression, - exclude_filter: none, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: big_collection - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query, - include_filter_2: getmore, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-query - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: query_identityview, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: views-aggregation - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_identityview, - include_filter_2: regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: where_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: where, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: update_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: update, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: insert_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: insert, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - # - name: compound-wildcard-index-insert - # commands: - # - func: f_dsi_run_workload - # vars: - # test_control: mongo-perf.2023-02 - # test_control_params: | - # {include_filter_1: compound-wildcard-insert, - # include_filter_2: core regression, - # exclude_filter: single_threaded, - # threads: "1 2 4 8", - # read_cmd: 'true'} - - name: wildcard-index-read_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_read, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: wildcard-index-write_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: wildcard_write, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: geo_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: geo, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_default_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-default.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_slow_or_sample_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-slow-or-sample.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_complex_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-complex.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_custom_filter_whole_doc_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-custom-filter-whole-doc.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: misc_slowms_everything_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - mongodb_setup: mongo-perf-standalone-slowms-everything.2023-02 - test_control_params: | - {include_filter_1: command multi remove mixed, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: singleThreaded_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: single_threaded, - include_filter_2: core regression, - exclude_filter: none, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation, - include_filter_2: regression, - exclude_filter: js, - threads: "1", - read_cmd: 'true'} - - name: aggregation_read_commands_large_dataset - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: aggregation_large_dataset, - include_filter_2: regression, - exclude_filter: js, - threads: "1 4", - read_cmd: 'true', - share_dataset: 'true'} - - name: agg-query-comparison_read_commands - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: agg_query_comparison, - include_filter_2: core regression, - exclude_filter: single_threaded, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: pipeline-updates - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: pipeline-updates, - include_filter_2: regression, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - - name: javascript - commands: - - func: f_dsi_run_workload - vars: - test_control: mongo-perf.2023-02 - test_control_params: | - {include_filter_1: js, - include_filter_2: aggregation, - exclude_filter: none, - threads: "1 2 4 8", - read_cmd: 'true'} - buildvariants: - - name: task_generation - display_name: Task Generation - modules: *modules - expansions: - platform: linux - project_dir: dsi - run_on: - - amazon2-build - tasks: - - name: schedule_global_auto_tasks - - &compile-amazon2 name: compile-amazon2 display_name: Compile @@ -1404,252 +392,3 @@ buildvariants: - "amazon2-arm64-large" tasks: - name: compile - - - name: perf-atlas-M60-real.arm.aws.2023-11 - display_name: PERF M60-Atlas ReplSet ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl - use_custom_build: true - infrastructure_provisioning: workload_client_arm.2023-04 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-atlas-large" - depends_on: - - name: compile - variant: compile-amazon2 - - name: schedule_global_auto_tasks - variant: task_generation - - name: compile - variant: compile-amazon-linux2-arm64 - - name: schedule_global_auto_tasks - variant: task_generation - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - - - name: perf-atlas-M60-real.intel.azure.2023-11 - display_name: PERF M60-Atlas ReplSet Intel Azure 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: atlas - canaries: none - atlas_setup: M60-repl-azure - use_custom_build_azure: true - infrastructure_provisioning: workload_client_intel.2023-11 - infrastructure_provisioning_release: 2023-09 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - storageEngine: wiredTiger - run_on: - - "rhel70-perf-atlas-large" - depends_on: *_compile_amazon2 - tasks: # Cannot use *3nodetasks because secondary_performance uses a special mongodb setup - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: tpcc - - name: tpcc_majority - - name: linkbench - - name: linkbench2 - - - name: perf-3-shard.arm.aws.2023-11 - display_name: PERF 3-Shard Cluster ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: shard - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: shard - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-shard" - depends_on: *_compile_amazon_linux2_arm64 - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: smoke_test - - name: mongos_workloads - - name: mongos_large_catalog_workloads - - name: change_streams_latency - - name: change_streams_multi_mongos - - - name: perf-3-node-replSet.arm.aws.2023-11 - display_name: PERF 3-Node ReplSet ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - compile_variant: "-arm64" - run_on: - - "rhel70-perf-replset" - depends_on: *_compile_amazon_linux2_arm64 - tasks: &3nodetasks - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: ycsb.2023-09 - - name: ycsb_w1.2023-09 - - name: ycsb_60GB.2023-09 - - name: ycsb.load - - name: ycsb_60GB.long.2023-09 - - name: ycsb_secondary_reads.2023-09 - - name: crud_workloads_majority - - name: crud_workloads_w1 - - name: misc_workloads - - name: map_reduce_workloads - - name: refine_shard_key_transaction_stress - - name: smoke_test - - name: secondary_performance # Uses a special 2 node mongodb setup - - name: non_sharded_workloads - - name: bestbuy_agg - - name: bestbuy_agg_merge_different_db - - name: bestbuy_agg_merge_same_db - - name: bestbuy_agg_merge_wordcount - - name: bestbuy_query - - name: change_streams_latency - - name: snapshot_reads - - name: secondary_reads - - name: tpcc - - name: linkbench - - name: linkbench2 - - - name: perf-3-node-replSet-intel.intel.aws.2023-11 - display_name: PERF 3-Node ReplSet Intel AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-intel.2023-11 - workload_setup: 2022-11 - platform: linux - project_dir: *project_dir - authentication: enabled - storageEngine: wiredTiger - run_on: - - "rhel70-perf-replset" - depends_on: *_compile_amazon2 - tasks: &3nodetasks - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: industry_benchmarks - - name: ycsb.2023-09 - - name: ycsb_60GB.2023-09 - - name: ycsb_60GB.long.2023-09 - - name: crud_workloads_majority - - name: smoke_test - - name: linkbench - - name: linkbench2 - - - - name: perf-2-node-replSet-initialsync.arm.aws.2023-11 - display_name: PERF 2-Node ReplSet Initial Sync ARM AWS 2023-11 - modules: *modules - expansions: - mongodb_setup_release: 2022-11 - mongodb_setup: replica-2node - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: replica-2node - workload_setup: 2022-11 - platform: linux - authentication: disabled - storageEngine: wiredTiger - compile_variant: "-arm64" - project_dir: *project_dir - depends_on: *_compile_amazon_linux2_arm64 - run_on: - - "rhel70-perf-replset" - tasks: - - name: schedule_patch_auto_tasks - - name: schedule_variant_auto_tasks - - name: initialsync-large - - - &linux-microbenchmarks-standalone-arm - name: perf-mongo-perf-standalone.arm.aws.2023-11 - display_name: PERF Monogo-Perf Standalone inMemory ARM AWS 2023-11 - cron: "0 0 * * 0" # Weekly, Sundays at 12 AM - modules: *modules - expansions: &standalone-arm-expansions - mongodb_setup_release: 2022-11 - mongodb_setup: mongo-perf-standalone.2023-02 - infrastructure_provisioning_release: 2023-09 - infrastructure_provisioning: workload_client_mongod_combined.2023-01 - workload_setup: 2022-11 - use_scons_cache: true - platform: linux - canaries: none - storageEngine: inMemory - project_dir: *project_dir - compile_variant: "-arm64" - run_on: - - "rhel70-perf-microbenchmarks" - depends_on: *_compile_amazon_linux2_arm64 - tasks: - - name: genny_scale_InsertRemove - - name: genny_execution_UserAcquisition - - name: aggregation_read_commands - - name: agg-query-comparison_read_commands - - name: query_read_commands - - name: views-aggregation - - name: views-query - - name: where_read_commands - - name: update_read_commands - - name: insert_read_commands - - name: wildcard-index-read_read_commands - - name: wildcard-index-write_read_commands - - name: geo_read_commands - - name: misc_read_commands - - name: singleThreaded_read_commands - - name: pipeline-updates - - name: javascript - - - &linux-microbenchmarks-standalone-intel - <<: *linux-microbenchmarks-standalone-arm - name: perf-mongo-perf-standalone.intel.aws.2023-11 - display_name: PERF Mongo-Perf Standalone inMemory Intel AWS 2023-11 - cron: "0 0 * * 0" # Weekly, Sundays at 12 AM - expansions: &standalone-intel-expansions - <<: *standalone-arm-expansions - infrastructure_provisioning: workload_client_mongod_combined_intel.2023-01 - compile_variant: "" - run_on: - - "rhel70-perf-microbenchmarks" - depends_on: *_compile_amazon2 - # yaml does not nicely merge arrays, so DO NOT ADD INDIVIDUAL TASKS HERE. - # Add tasks to the anchor that this variant references - # If diverging from that list, add the entire list of desired tasks here |
