diff options
Diffstat (limited to 'etc/evergreen_yml_components/definitions.yml')
| -rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 999 |
1 files changed, 740 insertions, 259 deletions
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index b0db6165d9b..3778bc1dbc2 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -107,6 +107,19 @@ variables: vars: resmoke_args: --help +- &gen_burn_in_task_template + name: gen_burn_in_task_template + depends_on: + - name: version_gen + variant: generate-tasks-for-version + - name: version_burn_in_gen + variant: generate-tasks-for-version + - name: archive_dist_test_debug + commands: + - func: "generate resmoke tasks" + vars: + resmoke_args: --help + - &benchmark_template name: benchmark_template depends_on: @@ -118,7 +131,7 @@ variables: resmoke_args: --help resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" vars: suite: benchmark_suite @@ -150,7 +163,12 @@ variables: is_jstestfuzz: true num_files: 15 num_tasks: 5 # Upperbound by `max_sub_suites` specified through the variant with each task still running `num_files` files. - resmoke_args: --help # resmoke_args needs to be overridden to specify one of the jstestfuzz suites + # It is error prone to require each fuzzer-related Evergreen task to need to override the + # resmoke_args variable. However, the resmoke_args variable must be defined as a string in the + # task generation configuration to satisfy mongodb/mongo-task-generator. We therefore specify an + # empty string for the variable to reflect there are no additional arguments provided to resmoke + # by default for the fuzzer-related tasks. + resmoke_args: "" resmoke_jobs_max: 1 should_shuffle: false continue_on_failure: false @@ -290,6 +308,7 @@ variables: - enterprise-rhel-70-64-bit # - enterprise-rhel-70-64-bit-kitchen-sink - enterprise-rhel-70-64-bit-no-libunwind + - enterprise-rhel-71-ppc64le - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - enterprise-rhel-80-64-bit @@ -299,12 +318,16 @@ variables: - enterprise-rhel-80-64-bit-suggested - enterprise-rhel-82-arm64 - enterprise-rhel-83-s390x + - enterprise-rhel-90-64-bit + - enterprise-rhel-90-arm64 - enterprise-suse12-64 - enterprise-suse15-64 - enterprise-ubuntu1804-64 - enterprise-ubuntu1804-arm64 - enterprise-ubuntu2004-arm64 + - enterprise-ubuntu2204-arm64 - enterprise-ubuntu2004-64 + - enterprise-ubuntu2204-64 - enterprise-windows - enterprise-windows-debug-unoptimized - enterprise-windows-inmem @@ -325,8 +348,11 @@ variables: - enterprise-rhel-80-64-bit-dynamic-required - enterprise-rhel-70-64-bit - enterprise-rhel-82-arm64 + - enterprise-rhel-90-64-bit + - enterprise-rhel-90-arm64 - enterprise-amazon2-arm64 - enterprise-ubuntu1804-64 + - enterprise-ubuntu2204-64 - enterprise-windows - enterprise-windows-required @@ -344,15 +370,20 @@ variables: - rhel70 - rhel76_compile_rhel70 - enterprise-rhel-70-64-bit + - enterprise-rhel-71-ppc64le - enterprise-rhel-81-ppc64le - enterprise-rhel-72-s390x - ubi8 - rhel80 - rhel-82-arm64 + - rhel90 + - rhel90-arm64 - enterprise-rhel-80-64-bit - enterprise-rhel-80-64-bit-suggested # For testing selinux. - enterprise-rhel-82-arm64 - enterprise-rhel-83-s390x + - enterprise-rhel-90-64-bit + - enterprise-rhel-90-arm64 - suse12 - enterprise-suse12-64 - suse15 @@ -360,10 +391,13 @@ variables: - ubuntu1804 - ubuntu1804-arm64 - ubuntu2004-arm64 + - ubuntu2204-arm64 - ubuntu2004 + - ubuntu2204 - enterprise-ubuntu1804-64 - enterprise-ubuntu1804-arm64 - enterprise-ubuntu2004-arm64 + - enterprise-ubuntu2204-arm64 - enterprise-ubuntu2004-64 - enterprise-windows - windows @@ -510,12 +544,23 @@ functions: args: - "src/evergreen/functions/binaries_extract.sh" - "check binary version": &check_binary_version - command: subprocess.exec + "get version expansions": &get_version_expansions + command: s3.get params: - binary: bash - args: - - "src/evergreen/functions/binary_version_check.sh" + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${project}/${version_id}/${bv_future_git_tag|version}_expansions.yml + bucket: mciuploads + local_file: src/version_expansions.yml + + "apply version expansions": &apply_version_expansions + command: expansions.update + params: + file: src/version_expansions.yml + + "get and apply version expansions": + - *get_version_expansions + - *apply_version_expansions "fetch benchmarks": &fetch_benchmarks command: s3.get @@ -795,15 +840,10 @@ functions: remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: atext-plain + content_type: text/plain display_name: Pip Requirements "send benchmark results": - - command: json.send - params: - name: perf - file: src/perf.json - - command: perf.send params: aws_key: ${aws_key} @@ -846,7 +886,6 @@ functions: - *adjust_venv - *fetch_binaries - *extract_binaries - - *check_binary_version - *get_buildnumber - *f_expansions_write - *set_up_credentials @@ -1058,7 +1097,7 @@ functions: files: - src/generated_resmoke_config/*.json - "generate burn in tags": + "generate version burn in": - *f_expansions_write - *configure_evergreen_api_credentials - command: subprocess.exec @@ -1066,33 +1105,42 @@ functions: params: binary: bash args: - - "./src/evergreen/burn_in_tests_generate.sh" + - "./src/evergreen/generate_version_burn_in.sh" - command: archive.targz_pack params: - target: burn_in_tags_gen.tgz - source_dir: src/generated_burn_in_tags_config + target: generate_tasks_config.tgz + source_dir: src/generated_resmoke_config include: - "*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: burn_in_tags_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz + local_file: generate_tasks_config.tgz + remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz bucket: mciuploads permissions: public-read - content_type: application/gzip - display_name: Burn_in_tags Task Config - Execution ${execution} + content_type: ${content_type|application/gzip} + display_name: Generated Burn In Task Config - Execution ${execution} + optional: true - command: generate.tasks params: + optional: true files: - - src/generated_burn_in_tags_config/burn_in_tags_gen.json + - src/generated_resmoke_config/*.json + + "initialize multiversion tasks": &initialize_multiversion_tasks + - command: shell.exec + params: + script: "echo 'noop'" "generate resmoke tasks": - *fetch_artifacts - *f_expansions_write - *kill_processes - *cleanup_environment + - *get_version_expansions + - *apply_version_expansions - *fetch_venv - *adjust_venv - *f_expansions_write @@ -1224,6 +1272,8 @@ functions: - "src/evergreen/scons_lint.sh" "scons compile": + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -1232,6 +1282,38 @@ functions: args: - "src/evergreen/scons_compile.sh" + "ninja compile": + - *f_expansions_write + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "./src/evergreen/ninja_compile.sh" + + "generate version expansions": &generate_version_expansions + command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/functions/version_expansions_generate.sh" + + "upload version expansions": &upload_version_expansions + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/version_expansions.yml + remote_file: ${project}/${version_id}/${bv_future_git_tag|version}_expansions.yml + bucket: mciuploads + permissions: public-read + content_type: application/x-yaml + display_name: version expansions + + "generate and upload version expansions": + - *generate_version_expansions + - *upload_version_expansions + "generate compile expansions": - *f_expansions_write - command: subprocess.exec @@ -1506,6 +1588,8 @@ functions: - "./src/evergreen/powercycle_exit.sh" "run packager.py": + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec params: @@ -1747,11 +1831,43 @@ functions: display_name: Jepsen Test Results - ${execution} optional: true + "upload jepsen_log": &upload_jepsen_log + command: s3.put + params: + optional: true + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/jepsen-mongodb/jepsen_${task_name}_${execution}.log + remote_file: ${project}/${build_variant}/${revision}/jepsen_${task_name}_${execution}.log + bucket: mciuploads + permissions: public-read + content_type: text/plain + display_name: Jepsen Test Log + "save jepsen artifacts": - *tar_jepsen_logs - *archive_jepsen_logs - *tar_jepsen_results - *archive_jepsen_results + - *upload_jepsen_log + + "save blackduck artifacts": + - command: archive.targz_pack + params: + target: blackduck.tgz + source_dir: /home/ec2-user/blackduck/runs/ + include: + - "**" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: blackduck.tgz + remote_file: ${project}/${build_variant}/${version_id}/blackduck-${task_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: BlackDuck logs ### Process & archive mongo coredumps ### "gather mongo coredumps": &gather_mongo_coredumps @@ -2076,7 +2192,7 @@ functions: remote_file: ${project}/${build_variant}/${revision}/local-resmoke-invocation-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: atext-plain + content_type: text/plain display_name: Resmoke.py Invocation for Local Usage @@ -2099,6 +2215,7 @@ post: - func: "save local client logs" - func: "save code coverage data" - func: "save jepsen artifacts" +- func: "save blackduck artifacts" - func: "save mongo coredumps" - func: "save failed unittests" - func: "save hang analyzer debugger files" @@ -2125,7 +2242,9 @@ tasks: ## compile - build all scons targets except unittests ## - name: compile_dist_test tags: [] - depends_on: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2135,6 +2254,21 @@ tasks: task_compile_flags: >- PREFIX=dist-test +## compile - build all scons targets except unittests ## +- name: compile_dist_test_half + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + targets: >- + compile_first_half_non_test_source + ${additional_compile_targets|} + task_compile_flags: >- + PREFIX=dist-test + - name: determine_patch_tests commands: - *f_expansions_write @@ -2150,13 +2284,20 @@ tasks: depends_on: - name: compile_dist_test commands: - - *f_expansions_write - func: "scons compile" vars: targets: >- archive-dist-test task_compile_flags: >- PREFIX=dist-test + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/macos_notary.py" + - "mongodb-binaries.${ext|tgz}" - command: s3.put params: @@ -2206,7 +2347,6 @@ tasks: - "./etc/repo_config.yaml" - "./etc/scons/**" - "buildscripts/**" - - "compile_expansions.yml" - "all_feature_flags.txt" # Must correspond to the definition in buildscripts/idl/lib.py. - "jstests/**" - "patch_files.txt" @@ -2271,47 +2411,120 @@ tasks: - name: compile_ninja tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: generating_for_ninja: true separate_debug: off - task_install_action: - default task_compile_flags: >- --ninja - targets: - generate-ninja - *f_expansions_write - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/ninja_compile.sh" + - func: "ninja compile" + vars: + ninja_file: "build.ninja" + +- name: compile_ninja_default_profile + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + generating_for_ninja: true + separate_debug: off + task_compile_flags: >- + --build-profile=default + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + --ninja + - *f_expansions_write + - func: "ninja compile" + vars: + ninja_file: "build.ninja" + +- name: compile_ninja_opt_profile + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + generating_for_ninja: true + separate_debug: off + task_compile_flags: >- + --build-profile=opt + CCACHE= + ICECC= + - *f_expansions_write + - func: "ninja compile" + vars: + ninja_file: "opt.ninja" + +- name: compile_ninja_fast_profile + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + generating_for_ninja: true + separate_debug: off + task_compile_flags: >- + --build-profile=fast + CCACHE= + ICECC= + - *f_expansions_write + - func: "ninja compile" + vars: + ninja_file: "fast.ninja" + +- name: compile_ninja_san_profile + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + generating_for_ninja: true + separate_debug: off + task_compile_flags: >- + --build-profile=san + CCACHE= + ICECC= + - *f_expansions_write + - func: "ninja compile" + vars: + ninja_file: "san.ninja" - name: compile_ninja_next tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: generating_for_ninja: true separate_debug: off - task_install_action: - default task_compile_flags: >- --build-tools=next --ninja - targets: - generate-ninja - *f_expansions_write - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/ninja_compile.sh" + - func: "ninja compile" + vars: + ninja_file: "build.ninja" - name: compile_build_tools_next tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2322,6 +2535,9 @@ tasks: - name: libdeps_graph_linting tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - command: subprocess.exec @@ -2381,10 +2597,14 @@ tasks: - name: clang_tidy tags: [] exec_timeout_secs: 3600 # 1 hour timeout for the task overall + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - - *f_expansions_write - func: "scons compile" vars: + task_compile_flags: >- + --build-profile=compiledb targets: generated-sources compiledb compiling_for_test: true - command: subprocess.exec @@ -2429,7 +2649,7 @@ tasks: ## run_unittests with UndoDB live-record ## #- name: run_unittests_with_recording # depends_on: -# - name: compile_unittests_for_recorded_unittest +# - name: compile_unittests_for_recorded_unittest # commands: # - *f_expansions_write # - func: "run diskstats" @@ -2458,6 +2678,9 @@ tasks: ##compile_and_archive_libfuzzertests - build libfuzzertests ## - name: compile_and_archive_libfuzzertests tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2488,6 +2711,9 @@ tasks: - name: server_discovery_and_monitoring_json_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2500,6 +2726,9 @@ tasks: - name: server_selection_json_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2545,8 +2774,21 @@ tasks: targets: archive-dbtest archive-dbtest-debug compiling_for_test: true +- name: compile_integration_test + tags: [] + depends_on: + - name: compile_dist_test + commands: + - func: "scons compile" + vars: + targets: install-integration-tests + compiling_for_test: true + - name: compile_visibility_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2574,7 +2816,12 @@ tasks: - name: embedded_sdk_build_cdriver tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: + - *get_version_expansions + - *apply_version_expansions - func: f_expansions_write - command: subprocess.exec params: @@ -2593,7 +2840,6 @@ tasks: task_compile_flags: &embedded_sdk_compile_flags >- --allocator=system --dbg=off - --enable-free-mon=off --enable-http-client=off --js-engine=none --opt=size @@ -2609,6 +2855,8 @@ tasks: depends_on: - name: embedded_sdk_install_dev commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec params: @@ -2652,6 +2900,8 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -2678,6 +2928,8 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -2696,6 +2948,8 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: + - *get_version_expansions + - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -2715,6 +2969,8 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: + - *get_version_expansions + - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -2731,6 +2987,9 @@ tasks: - name: stitch_support_create_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2738,7 +2997,6 @@ tasks: targets: install-stitch-support install-stitch-support-debug install-stitch-support-dev task_compile_flags: >- --link-model=dynamic-sdk - --enable-free-mon=off --ssl=off --enable-http-client=off --modules= @@ -2770,7 +3028,6 @@ tasks: targets: install-stitch-support-test compiling_for_test: true task_compile_flags: >- - --enable-free-mon=off --ssl=off --enable-http-client=off --modules= @@ -2781,6 +3038,7 @@ tasks: depends_on: - name: stitch_support_install_tests commands: + - func: "get and apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -2791,6 +3049,9 @@ tasks: - name: crypt_create_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2798,8 +3059,7 @@ tasks: targets: archive-mongo-crypt-dev task_compile_flags: >- --allocator=system - --enable-free-mon=off - --enterprise-features=fle + --enterprise-features=fle,search,vector_search --js-engine=none --link-model=dynamic-sdk --enable-http-client=off @@ -2824,6 +3084,9 @@ tasks: - name: crypt_create_debug_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2833,8 +3096,7 @@ tasks: --dbg=on --opt=off --allocator=system - --enable-free-mon=off - --enterprise-features=fle + --enterprise-features=fle,search,vector_search --js-engine=none --link-model=dynamic-sdk DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' @@ -2869,15 +3131,13 @@ tasks: vars: targets: archive-mongo-crypt-shlib-test compiling_for_test: true - task_install_action: - default task_compile_flags: >- --allocator=system - --enable-free-mon=off - --enterprise-features=fle + --enterprise-features=fle,search,vector_search --js-engine=none --link-model=static DESTDIR='$BUILD_ROOT/crypt-lib-$MONGO_VERSION' + RPATH='$$RPATH_ESCAPED_DOLLAR_ORIGIN/../lib' - command: s3.put params: aws_key: ${aws_key} @@ -2894,6 +3154,7 @@ tasks: depends_on: - name: crypt_install_tests commands: + - func: "get and apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -2904,7 +3165,9 @@ tasks: - name: compile_benchmarks tags: [] - depends_on: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - command: manifest.load - func: "git get project and add git tag" @@ -3094,46 +3357,26 @@ tasks: args: - "src/evergreen/check_idl_compat.sh" -- name: burn_in_tests_gen +- <<: *gen_burn_in_task_template + name: burn_in_tests_gen tags: [] + patch_only: true commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - *kill_processes - - *cleanup_environment - - func: "set up venv" - - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "src/evergreen/burn_in_tests.sh" - env: - BURN_IN_TESTS: ${burn_in_tests} - - command: archive.targz_pack - params: - target: src/burn_in_tests_gen.tgz - source_dir: src - include: - - burn_in_tests_gen.json + - func: "generate resmoke tasks" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/burn_in_tests_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Burn_in_tests Task Config - Execution ${execution} - - command: generate.tasks - params: - files: - - src/burn_in_tests_gen.json +- <<: *gen_burn_in_task_template + name: burn_in_tags_gen + tags: [] + patch_only: true + commands: + - func: "generate resmoke tasks" + +- <<: *gen_burn_in_task_template + name: burn_in_tasks_gen + tags: [] + patch_only: true + commands: + - func: "generate resmoke tasks" - <<: *benchmark_template name: benchmarks_orphaned @@ -3146,7 +3389,7 @@ tasks: exec_timeout_secs: 14400 # 4 hour timeout. resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" vars: suite: benchmarks @@ -3160,7 +3403,7 @@ tasks: suite: benchmarks_sharding resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" # Disabled under SERVER-64949. # - <<: *benchmark_template @@ -3266,14 +3509,18 @@ tasks: name: initial_sync_multiversion_fuzzer_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + initial_sync_fuzzer_last_lts: last_lts + initial_sync_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 10 num_tasks: 5 npm_command: initsync-fuzzer - suite: initial_sync_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + run_no_feature_flag_tests: "true" ## initial sync generational fuzzer ## - <<: *jstestfuzz_template @@ -3294,28 +3541,36 @@ tasks: name: aggregation_multiversion_fuzzer_gen tags: ["aggfuzzer", "common", "multiversion", "require_npm", "random_name", "future_git_tag_incompatible"] commands: + - func: "initialize multiversion tasks" + vars: + aggregation_multiversion_fuzzer_last_lts: last_lts + aggregation_multiversion_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-fuzzer + run_no_feature_flag_tests: "true" ## Standalone generational fuzzer for multiversion aggregation expressions ## - <<: *jstestfuzz_template name: aggregation_expression_multiversion_fuzzer_gen tags: ["aggfuzzer", "multiversion", "require_npm", "random_name"] commands: + - func: "initialize multiversion tasks" + vars: + aggregation_expression_multiversion_fuzzer_last_lts: last_lts + aggregation_expression_multiversion_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 - suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" npm_command: agg-expr-fuzzer + run_no_feature_flag_tests: "true" ## Standalone generational fuzzer for checking optimized and unoptimized expression equivalence - <<: *jstestfuzz_template @@ -3427,14 +3682,18 @@ tasks: name: update_fuzzer_gen tags: ["updatefuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + update_fuzzer_last_lts: last_lts + update_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 npm_command: update-fuzzer - suite: generational_fuzzer resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + run_no_feature_flag_tests: "true" ## jstestfuzz standalone update time-series generational fuzzer ## - <<: *jstestfuzz_template @@ -3456,30 +3715,38 @@ tasks: name: update_fuzzer_replication_gen tags: ["updatefuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + update_fuzzer_replication_last_lts: last_lts + update_fuzzer_replication_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 5 num_tasks: 5 npm_command: update-fuzzer - suite: generational_fuzzer_replication resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" + run_no_feature_flag_tests: "true" ## rollback multiversion fuzzer ## - <<: *jstestfuzz_template name: rollback_multiversion_fuzzer_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + rollback_fuzzer_last_lts: last_lts + rollback_fuzzer_last_continuous: last_continuous - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars num_files: 3 num_tasks: 5 npm_command: rollback-fuzzer - suite: rollback_fuzzer # Rollback suites create indexes with majority of nodes not available for replication. So, disabling # index build commit quorum. resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}, enableIndexBuildCommitQuorum: false}'" + run_no_feature_flag_tests: "true" ## rollback generational fuzzer ## - <<: *jstestfuzz_template @@ -3785,13 +4052,21 @@ tasks: name: jstestfuzz_replication_multiversion_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + jstestfuzz_replication_last_continuous_new_new_old: last_continuous + jstestfuzz_replication_last_continuous_new_old_new: last_continuous + jstestfuzz_replication_last_continuous_old_new_new: last_continuous + jstestfuzz_replication_last_lts_new_new_old: last_lts + jstestfuzz_replication_last_lts_new_old_new: last_lts + jstestfuzz_replication_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - suite: jstestfuzz_replication npm_command: jstestfuzz + run_no_feature_flag_tests: "true" ## jstestfuzz initial sync replica set ## - <<: *jstestfuzz_template @@ -3851,13 +4126,17 @@ tasks: name: jstestfuzz_sharded_multiversion_gen tags: ["multiversion_fuzzer", "require_npm", "random_name", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + jstestfuzz_sharded_last_continuous_new_old_old_new: last_continuous + jstestfuzz_sharded_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: <<: *jstestfuzz_config_vars jstestfuzz_vars: --jsTestsDir ../jstests resmoke_args: "--mongodSetParameters='{logComponentVerbosity: {command: 2}}'" - suite: jstestfuzz_sharded npm_command: jstestfuzz + run_no_feature_flag_tests: "true" ## jstestfuzz sharded cluster causal consistency ## - <<: *jstestfuzz_template @@ -3983,15 +4262,33 @@ tasks: name: multiversion_sanity_check_gen tags: ["multiversion", "multiversion_sanity_check"] commands: + - func: "initialize multiversion tasks" + vars: + multiversion_sanity_check_last_continuous_new_new_old: last_continuous + multiversion_sanity_check_last_continuous_new_old_new: last_continuous + multiversion_sanity_check_last_continuous_old_new_new: last_continuous + multiversion_sanity_check_last_lts_new_new_old: last_lts + multiversion_sanity_check_last_lts_new_old_new: last_lts + multiversion_sanity_check_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" + vars: + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: replica_sets_jscore_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: + - func: "initialize multiversion tasks" + vars: + replica_sets_jscore_passthrough_last_continuous_new_new_old: last_continuous + replica_sets_jscore_passthrough_last_continuous_new_old_new: last_continuous + replica_sets_jscore_passthrough_last_continuous_old_new_new: last_continuous + replica_sets_jscore_passthrough_last_lts_new_new_old: last_lts + replica_sets_jscore_passthrough_last_lts_new_old_new: last_lts + replica_sets_jscore_passthrough_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - suite: replica_sets_jscore_passthrough + run_no_feature_flag_tests: "true" # Check that the mutational fuzzer can parse JS files modified in a patch build. @@ -4145,27 +4442,19 @@ tasks: commands: - func: "generate resmoke tasks" -- name: burn_in_tags_gen - tags: [] - depends_on: - - name: archive_dist_test +- name: version_gen commands: - command: manifest.load - - func: "git get project and add git tag" + - *git_get_project - *f_expansions_write + - *add_git_tag - *kill_processes - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - func: "generate burn in tags" - vars: - max_revisions: 25 - repeat_tests_secs: 600 - repeat_tests_min: 2 - repeat_tests_max: 1000 + - func: "generate version" -- name: version_gen +- name: version_burn_in_gen commands: - command: manifest.load - *git_get_project @@ -4175,7 +4464,19 @@ tasks: - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "generate version" + - func: "generate version burn in" + +- name: version_expansions_gen + commands: + - command: manifest.load + - *git_get_project + - *f_expansions_write + - *add_git_tag + - *kill_processes + - *cleanup_environment + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate and upload version expansions" - name: selected_tests_gen tags: [] @@ -4213,15 +4514,29 @@ tasks: name: change_streams_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: + - func: "initialize multiversion tasks" + vars: + change_streams_last_continuous_new_new_old: last_continuous + change_streams_last_continuous_new_old_new: last_continuous + change_streams_last_continuous_old_new_new: last_continuous + change_streams_last_lts_new_new_old: last_lts + change_streams_last_lts_new_old_new: last_lts + change_streams_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - suite: change_streams + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: change_streams_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + change_streams_downgrade_last_continuous_new_old_old_new: last_continuous + change_streams_downgrade_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" + vars: + run_no_feature_flag_tests: "true" - <<: *task_template name: change_streams_update_v1_oplog @@ -4279,13 +4594,17 @@ tasks: name: change_streams_sharded_collections_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + change_streams_sharded_collections_passthrough_last_continuous_new_old_old_new: last_continuous + change_streams_sharded_collections_passthrough_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - suite: change_streams_sharded_collections_passthrough + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: multiversion_future_git_tag_gen - tags: ["multiversion", "no_version_combination", "multiversion_future_git_tag"] + tags: ["multiversion", "no_multiversion_generate_tasks", "multiversion_future_git_tag"] commands: - func: "generate resmoke tasks" vars: @@ -4293,7 +4612,7 @@ tasks: - <<: *gen_task_template name: multiversion_auth_future_git_tag_gen - tags: ["auth", "multiversion", "no_version_combination", "multiversion_future_git_tag"] + tags: ["auth", "multiversion", "no_multiversion_generate_tasks", "multiversion_future_git_tag"] commands: - func: "generate resmoke tasks" vars: @@ -4437,93 +4756,77 @@ tasks: - <<: *task_template name: integration_tests_standalone tags: ["integration", "standalone"] + depends_on: + - name: compile_integration_test commands: - - command: manifest.load - - func: "git get project and add git tag" - - func: "do setup" - - func: "set up win mount script" - - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" + - *f_expansions_write + - *run_diskstats + - *monitor_process_threads + - *collect_system_resource_info + - *collect_ulimit_info - func: "run tests" + vars: + install_dir: build/install/bin - <<: *task_template name: integration_tests_standalone_audit tags: ["integration", "audit"] + depends_on: + - name: compile_integration_test commands: - - command: manifest.load - - func: "git get project and add git tag" - - func: "do setup" - - func: "set up win mount script" - - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" + - *f_expansions_write + - *run_diskstats + - *monitor_process_threads + - *collect_system_resource_info + - *collect_ulimit_info - func: "run tests" + vars: + install_dir: build/install/bin - <<: *task_template name: integration_tests_replset tags: ["integration"] + depends_on: + - name: compile_integration_test commands: - - command: manifest.load - - func: "git get project and add git tag" - - func: "do setup" - - func: "set up win mount script" - - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" + - *f_expansions_write + - *run_diskstats + - *monitor_process_threads + - *collect_system_resource_info + - *collect_ulimit_info - func: "run tests" + vars: + install_dir: build/install/bin - <<: *task_template name: integration_tests_replset_ssl_auth tags: ["integration"] + depends_on: + - name: compile_integration_test commands: - - command: manifest.load - - func: "git get project and add git tag" - - func: "do setup" - - func: "set up win mount script" - - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" + - *f_expansions_write + - *run_diskstats + - *monitor_process_threads + - *collect_system_resource_info + - *collect_ulimit_info - func: "run tests" + vars: + install_dir: build/install/bin - <<: *task_template name: integration_tests_sharded tags: ["integration", "sharded"] + depends_on: + - name: compile_integration_test commands: - - command: manifest.load - - func: "git get project and add git tag" - *f_expansions_write - - func: "do setup" - - func: "set up win mount script" - - func: "generate compile expansions" # Generate compile expansions needs to be run to mount the shared scons cache. - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-integration-tests - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" + - *run_diskstats + - *monitor_process_threads + - *collect_system_resource_info + - *collect_ulimit_info - func: "run tests" + vars: + install_dir: build/install/bin - <<: *task_template name: external_auth @@ -4592,7 +4895,9 @@ tasks: - func: "run tests" vars: suite: core - resmoke_args: --fuzzMongodConfigs + resmoke_args: >- + --fuzzMongodConfigs + --excludeWithAnyTags=does_not_support_config_fuzzer - <<: *task_template name: config_fuzzer_concurrency @@ -4602,7 +4907,9 @@ tasks: - func: "run tests" vars: suite: concurrency - resmoke_args: --fuzzMongodConfigs + resmoke_args: >- + --fuzzMongodConfigs + --excludeWithAnyTags=does_not_support_config_fuzzer - <<: *task_template name: config_fuzzer_simulate_crash_concurrency_replication @@ -4612,7 +4919,9 @@ tasks: - func: "run tests" vars: suite: simulate_crash_concurrency_replication - resmoke_args: --fuzzMongodConfigs + resmoke_args: >- + --fuzzMongodConfigs + --excludeWithAnyTags=does_not_support_config_fuzzer - <<: *task_template name: config_fuzzer_concurrency_replication @@ -4622,7 +4931,9 @@ tasks: - func: "run tests" vars: suite: concurrency_replication - resmoke_args: --fuzzMongodConfigs + resmoke_args: >- + --fuzzMongodConfigs + --excludeWithAnyTags=does_not_support_config_fuzzer - <<: *task_template name: config_fuzzer_replica_sets_jscore_passthrough @@ -4632,7 +4943,9 @@ tasks: - func: "run tests" vars: suite: replica_sets_jscore_passthrough - resmoke_args: --fuzzMongodConfigs + resmoke_args: >- + --fuzzMongodConfigs + --excludeWithAnyTags=does_not_support_config_fuzzer - <<: *task_template name: jsCore_ese @@ -4899,13 +5212,13 @@ tasks: - <<: *gen_task_template name: multiversion_auth_gen - tags: ["auth", "multiversion", "no_version_combination", "future_git_tag_incompatible"] + tags: ["auth", "multiversion", "no_multiversion_generate_tasks", "future_git_tag_incompatible"] commands: - func: "generate resmoke tasks" - <<: *gen_task_template name: multiversion_gen - tags: ["multiversion", "no_version_combination", "future_git_tag_incompatible"] + tags: ["multiversion", "no_multiversion_generate_tasks", "future_git_tag_incompatible"] commands: - func: "generate resmoke tasks" @@ -4914,7 +5227,7 @@ tasks: # build variants that enable this task. - <<: *gen_task_template name: feature_flag_multiversion_gen - tags: ["multiversion", "no_version_combination"] + tags: ["multiversion", "no_multiversion_generate_tasks"] commands: - func: "generate resmoke tasks" @@ -4973,20 +5286,23 @@ tasks: vars: resmoke_jobs_max: 1 -- <<: *task_template - name: sharded_collections_jscore_passthrough +- <<: *gen_task_template + name: sharded_collections_jscore_passthrough_gen tags: ["sharding", "jscore"] commands: - - func: "do setup" - - func: "run tests" + - func: "generate resmoke tasks" - <<: *gen_task_template name: sharded_collections_jscore_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + sharded_collections_jscore_passthrough_last_continuous_new_old_old_new: last_continuous + sharded_collections_jscore_passthrough_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - suite: sharded_collections_jscore_passthrough + run_no_feature_flag_tests: "true" - <<: *task_template name: sharding_jscore_passthrough @@ -4999,9 +5315,13 @@ tasks: name: sharding_jscore_multiversion_gen tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + sharding_jscore_passthrough_last_lts_new_old_old_new: last_lts + sharding_jscore_passthrough_last_continuous_new_old_old_new: last_continuous - func: "generate resmoke tasks" vars: - suite: sharding_jscore_passthrough + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharding_api_version_jscore_passthrough_gen @@ -5224,9 +5544,17 @@ tasks: name: concurrency_replication_multiversion_gen tags: ["multiversion", "multiversion_passthrough"] commands: + - func: "initialize multiversion tasks" + vars: + concurrency_replication_last_continuous_new_new_old: last_continuous + concurrency_replication_last_continuous_new_old_new: last_continuous + concurrency_replication_last_continuous_old_new_new: last_continuous + concurrency_replication_last_lts_new_new_old: last_lts + concurrency_replication_last_lts_new_old_new: last_lts + concurrency_replication_last_lts_old_new_new: last_lts - func: "generate resmoke tasks" vars: - suite: concurrency_replication + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: concurrency_replication_causal_consistency_gen @@ -5302,9 +5630,13 @@ tasks: name: concurrency_sharded_replication_multiversion_gen tags: ["multiversion_passthrough", "sharded", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + concurrency_sharded_replication_last_continuous_new_old_old_new: last_continuous + concurrency_sharded_replication_last_lts_new_old_old_new: last_lts - func: "generate resmoke tasks" vars: - suite: concurrency_sharded_replication + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: concurrency_sharded_replication_with_balancer_gen @@ -5512,12 +5844,13 @@ tasks: vars: use_large_distro: "true" -- <<: *task_template - name: write_concern_majority_passthrough +- <<: *gen_task_template + name: write_concern_majority_passthrough_gen tags: ["read_write_concern", "large", "write"] commands: - - func: "do setup" - - func: "run tests" + - func: "generate resmoke tasks" + vars: + use_large_distro: "true" - <<: *task_template name: cwrwc_passthrough @@ -5602,9 +5935,13 @@ tasks: name: replica_sets_multiversion_gen tags: ["random_multiversion_ds", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + replica_sets_last_lts: last_lts + replica_sets_last_continuous: last_continuous - func: "generate resmoke tasks" vars: - suite: replica_sets + run_no_feature_flag_tests: "true" - <<: *task_template name: sasl @@ -5632,10 +5969,14 @@ tasks: name: sharding_multiversion_gen tags: ["random_multiversion_ds", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + sharding_last_continuous: last_continuous + sharding_last_lts: last_lts - func: "generate resmoke tasks" vars: use_large_distro: "true" - suite: sharding + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharding_max_mirroring_gen @@ -5696,12 +6037,16 @@ tasks: resmoke_jobs_max: 1 - <<: *gen_task_template - name: sharding_last_lts_mongos_and_mixed_shards_gen - tags: ["sharding", "common", "multiversion", "no_version_combination"] + name: sharding_mongos_and_mixed_shards_gen + tags: ["sharding", "common", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + sharding_last_lts_mongos_and_mixed_shards: last_lts - func: "generate resmoke tasks" vars: use_large_distro: "true" + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharding_update_v1_oplog_gen @@ -5822,25 +6167,28 @@ tasks: vars: use_large_distro: "true" -# Use explicit task definitions for retryable_writes_downgrade suites to avoid running -# with all Repl multiversion combinations. - <<: *gen_task_template - name: retryable_writes_downgrade_last_continuous_gen - tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] - commands: - - func: "generate resmoke tasks" - -- <<: *gen_task_template - name: retryable_writes_downgrade_last_lts_gen - tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] + name: retryable_writes_downgrade_gen + tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + retryable_writes_downgrade_last_lts: last_lts - func: "generate resmoke tasks" + vars: + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: sharded_retryable_writes_downgrade_gen tags: ["multiversion_passthrough", "multiversion"] commands: + - func: "initialize multiversion tasks" + vars: + sharded_retryable_writes_downgrade_last_lts_new_old_old_new: last_lts + sharded_retryable_writes_downgrade_last_continuous_new_old_old_new: last_continuous - func: "generate resmoke tasks" + vars: + run_no_feature_flag_tests: "true" - <<: *gen_task_template name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen @@ -5953,15 +6301,6 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template - name: free_monitoring - tags: [] - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_jobs_max: 1 - -- <<: *task_template name: client_encrypt tags: ["ssl", "encrypt"] commands: @@ -6179,7 +6518,6 @@ tasks: - func: "run powercycle test" timeout_secs: 1800 # 30 minute timeout for no output - - name: selinux_rhel8_org tags: [] depends_on: @@ -6216,6 +6554,42 @@ tasks: distro: rhel80-selinux test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js +- name: selinux_rhel9_org + tags: [] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - *f_expansions_write + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel90-selinux + test_list: jstests/selinux/*.js +- name: selinux_rhel9_enterprise + tags: [] + depends_on: + - name: archive_dist_test + - name: package + commands: + - command: manifest.load + - func: "git get project and add git tag" + - *f_expansions_write + - func: "set up venv" + - func: "fetch packages" + - func: "fetch binaries" + - func: "extract binaries" + - func: "run selinux tests" + vars: + distro: rhel90-selinux + test_list: jstests/selinux/*.js src/mongo/db/modules/enterprise/jstests/selinux/*.js + + - name: selinux_rhel7_org tags: [] depends_on: @@ -6323,26 +6697,6 @@ tasks: - func: "do setup" - func: "run tests" -- name: test_packages - tags: [] - depends_on: - - name: package - commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - *kill_processes - - *cleanup_environment - - *set_up_venv - - func: "fetch packages" - - func: "set up remote credentials" - vars: - private_key_file: ~/.ssh/kitchen.pem - private_key_remote: ${kitchen_private_key} - aws_key_remote: ${kitchen_aws_key} - aws_secret_remote: ${kitchen_aws_secret} - - func: "run kitchen" - - name: package tags: [] depends_on: @@ -6360,6 +6714,14 @@ tasks: ${additional_package_targets|} task_compile_flags: >- --legacy-tarball + - command: subprocess.exec + params: + binary: bash + add_expansions_to_env: true + args: + - "src/evergreen/run_python_script.sh" + - "evergreen/macos_notary.py" + - "mongodb-dist.${ext|tgz}" - func: "f_expansions_write" - command: subprocess.exec params: @@ -6487,6 +6849,7 @@ tasks: - name: publish_packages + run_on: rhel80-small tags: ["publish"] # This should prevent this task from running in patch builds, where we # don't want to publish packages. @@ -6501,6 +6864,7 @@ tasks: commands: - command: manifest.load - func: "git get project and add git tag" + - func: "get and apply version expansions" - func: "f_expansions_write" - func: "kill processes" - func: "cleanup environment" @@ -6521,6 +6885,7 @@ tasks: - "./src/evergreen/packages_publish.sh" - name: push + run_on: rhel8.7-small tags: ["publish"] patchable: false depends_on: @@ -6532,6 +6897,7 @@ tasks: commands: - command: manifest.load - func: "git get project and add git tag" + - func: "get and apply version expansions" - func: "f_expansions_write" - func: "kill processes" - func: "cleanup environment" @@ -6573,13 +6939,28 @@ tasks: aws_key_remote: ${repo_aws_key} aws_secret_remote: ${repo_aws_secret} - func: "f_expansions_write" - - func: "set up notary client credentials" + + # login to container registry + - command: shell.exec + params: + shell: bash + script: | + set -oe + podman login --username ${release_tools_container_registry_username} --password ${release_tools_container_registry_password} ${release_tools_container_registry} + + # signing windows artifacts - command: subprocess.exec - type: test params: binary: bash args: - - "./src/evergreen/notary_client_run.sh" + - "./src/evergreen/garasign_jsign_sign.sh" + + # signing linux artifacts + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/garasign_gpg_sign.sh" # Put the binaries tarball/zipfile - command: s3.put @@ -6692,7 +7073,7 @@ tasks: aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi bucket: build-push-testing content_type: application/x-msi remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi @@ -6751,7 +7132,7 @@ tasks: aws_secret: ${aws_secret} permissions: public-read build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha1 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha1 bucket: build-push-testing content_type: text/plain remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha1 @@ -6809,7 +7190,7 @@ tasks: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.sha256 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.sha256 bucket: build-push-testing permissions: public-read remote_file: ${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-${suffix}-${task_id}-signed.msi.sha256 @@ -6868,7 +7249,7 @@ tasks: aws_key: ${aws_key} aws_secret: ${aws_secret} build_variants: ["enterprise-windows", "windows"] - local_file: src/mongodb-${push_name}-${push_arch}-${suffix}-signed.msi.md5 + local_file: src/mongodb-${push_name}-${push_arch}-${suffix}.msi.md5 bucket: build-push-testing permissions: public-read content_type: text/plain @@ -6999,6 +7380,7 @@ tasks: 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} - name: crypt_push + run_on: rhel8.7-small tags: ["publish_crypt"] patchable: false stepback: false @@ -7016,6 +7398,8 @@ tasks: - func: "upload pip requirements" - func: "get buildnumber" - func: "f_expansions_write" + - func: "get and apply version expansions" + - func: "f_expansions_write" - func: "generate compile expansions" - func: "apply compile expansions" - func: "f_expansions_write" @@ -7033,13 +7417,18 @@ tasks: aws_key_remote: ${repo_aws_key} aws_secret_remote: ${repo_aws_secret} - func: "f_expansions_write" - - func: "set up notary client credentials" + # login to container registry + - command: shell.exec + params: + shell: bash + script: | + set -oe + podman login --username ${release_tools_container_registry_username} --password ${release_tools_container_registry_password} ${release_tools_container_registry} - command: subprocess.exec - type: test params: binary: bash args: - - "./src/evergreen/notary_client_crypt_run.sh" + - "./src/evergreen/garasign_gpg_crypt_sign.sh" # Put the crypt tarball/zipfile - command: s3.put params: @@ -7127,6 +7516,15 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template + name: search_pinned_connections_auth + tags: [] + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_jobs_max: 1 + +- <<: *task_template name: search_ssl tags: [] commands: @@ -7136,6 +7534,33 @@ tasks: resmoke_jobs_max: 1 - <<: *task_template + name: vector_search + tags: [] + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_jobs_max: 1 + +- <<: *task_template + name: vector_search_auth + tags: [] + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_jobs_max: 1 + +- <<: *task_template + name: vector_search_ssl + tags: [] + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_jobs_max: 1 + +- <<: *task_template name: cqf tags: [] commands: @@ -7364,6 +7789,26 @@ task_groups: display_name: build.ninja - <<: *compile_task_group_template + name: compile_ninja_default_profile_TG + tasks: + - compile_ninja_default_profile + +- <<: *compile_task_group_template + name: compile_ninja_opt_profile_TG + tasks: + - compile_ninja_opt_profile + +- <<: *compile_task_group_template + name: compile_ninja_san_profile_TG + tasks: + - compile_ninja_san_profile + +- <<: *compile_task_group_template + name: compile_ninja_fast_profile_TG + tasks: + - compile_ninja_fast_profile + +- <<: *compile_task_group_template name: server_discovery_and_monitoring_json_test_TG tasks: - server_discovery_and_monitoring_json_test @@ -7400,6 +7845,22 @@ task_groups: - compile_all - package +# SERVER-76006 +# This is a compile stream meant for non-cached and/or underpowered systems. +# It joins most of the compile tasks together under a single host spread out +# across different tasks. +- <<: *compile_task_group_template + name: small_compile_test_and_package_serial_TG + tasks: + - compile_dist_test_half + - compile_dist_test + - archive_dist_test + - archive_dist_test_debug + - compile_dbtest + - run_dbtest + - archive_dbtest + - package + - <<: *compile_task_group_template name: compile_and_test_TG tasks: @@ -7448,6 +7909,25 @@ task_groups: - run_dbtest - archive_dbtest +- <<: *compile_task_group_template + name: compile_integration_and_test_parallel_stream_TG + tasks: + - compile_integration_test + - integration_tests_standalone + - integration_tests_standalone_audit + - integration_tests_replset + - integration_tests_replset_ssl_auth + - integration_tests_sharded + +- <<: *compile_task_group_template + name: compile_integration_and_test_no_audit_parallel_stream_TG + tasks: + - compile_integration_test + - integration_tests_standalone + - integration_tests_replset + - integration_tests_replset_ssl_auth + - integration_tests_sharded + - name: clang_tidy_TG setup_group_can_fail_task: true setup_group: @@ -7544,6 +8024,7 @@ task_groups: tags: ["stitch"] tasks: - "stitch_support_create_lib" + - <<: *stitch_support_task_group_template name: stitch_support_lib_build_and_test tags: ["stitch"] |