path: root/etc/system_perf.yml

command_type: system
stepback: false
exec_timeout_secs: &exec_timeout_secs 21600
timeout_secs: &timeout_secs 7200

include:
  - filename: etc/evergreen_yml_components/definitions.yml
  - filename: etc/evergreen_yml_components/tasks/compile_tasks_shared.yml
  ### lint_yaml trim start ###
  #
  # The lines below are trimmed by yamllinters.sh so that `evergreen evaluate` works on this file.
  # This allows us to test that non-dsi imports continue to work.
  #
  - filename: evergreen/system_perf/master/variants.yml
    module: dsi
  - filename: evergreen/system_perf/master/master_variants.yml
    module: dsi
  - filename: evergreen/system_perf/shared_tasks.yml
    module: dsi
  ### lint_yaml trim end ###

## Parameters for parameterized builds (see https://github.com/evergreen-ci/evergreen/wiki/Parameterized-Builds)
parameters:
  - key: patch_compile_flags
    description: "Additional SCons flags to be applied during scons compile invocations in this patch"
  # see https://github.com/10gen/performance-tooling-docs/blob/main/patch_testing.md#skipping-compilation-on-sys-perf-projects
  - key: reuse_compile_from
    description: "Version_id of the commit/patch to reuse the compile artifacts from, e.g., sys_perf_ef8504eefef4c9fe2062b8424e85053b3e1ad8d2"
  - key: exec_timeout_secs_override
    value: *exec_timeout_secs
    description: "Overrides the default exec_timeout_secs defined at the top of this file for the f_dsi_run_workload task"
  - key: timeout_secs_override
    value: *timeout_secs
    description: "Overrides the default timeout_secs defined at the top of this file for the f_dsi_run_workload task"

variables:
  _src_dir: &src_dir src/mongo
  _modules: &modules
    - dsi
    - genny
    - workloads
    - linkbench
    - linkbench2
    - tsbs
    - mongo-perf
    - YCSB
    - py-tpcc
    - PrivateWorkloads
    - flamegraph

  _schedule_variant_auto_tasks_task: &schedule_variant_auto_tasks_task
    name: schedule_variant_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

  _schedule_patch_auto_tasks_task: &schedule_patch_auto_tasks_task
    name: schedule_patch_auto_tasks
    activate: false
    depends_on:
      - name: schedule_global_auto_tasks
        variant: task_generation

  _shared_compile_expansions: &shared_compile_expansions
    platform: linux
    project_dir: dsi
    scons_cache_scope: shared
    scons_cache_mode: all
    has_packages: false

###
# Same in every DSI project. Ensure that this block is synchronized with
# evergreen-dsitest.yml, and src/dsi/onboarding.py (search update-repos-here)
# in the dsi repo, and etc/system_perf.yml in mongodb/mongo.
modules:
  - name: dsi
    owner: 10gen
    repo: dsi
    prefix: ${workdir}/src
    branch: master
  - name: genny
    owner: mongodb
    repo: genny
    prefix: ${workdir}/src
    branch: master
  - name: workloads
    owner: 10gen
    repo: workloads
    prefix: ${workdir}/src
    branch: master
  - name: linkbench
    owner: 10gen
    repo: linkbench
    prefix: ${workdir}/src
    branch: master
  - name: linkbench2
    owner: 10gen
    repo: linkbench2
    prefix: ${workdir}/src
    branch: master
  - name: tsbs
    owner: mongodb-forks
    repo: tsbs
    prefix: ${workdir}/src
    branch: main
  - name: mongo-perf
    owner: mongodb
    repo: mongo-perf
    prefix: ${workdir}/src
    branch: master
  - name: YCSB
    owner: mongodb-labs
    repo: YCSB
    prefix: ${workdir}/src
    branch: production
  - name: py-tpcc
    owner: mongodb-labs
    repo: py-tpcc
    prefix: ${workdir}/src
    branch: production
  - name: flamegraph
    owner: mongodb-forks
    repo: flamegraph
    prefix: ${workdir}/src
    branch: master
  - name: PrivateWorkloads
    owner: 10gen
    repo: PrivateWorkloads
    prefix: ${workdir}/src
    branch: production
###

###
# Same in every DSI project
pre:
  - func: "f_other_pre_ops"
  - func: "f_dsi_pre_run"
  - func: "get version expansions"
  - func: "apply version expansions"
post:
  - func: "f_dsi_post_run"
  - func: "f_other_post_ops"
timeout:
  - func: "f_dsi_timeout"
  - func: "f_other_timeout"
###

functions:
  f_other_post_ops:
    - command: shell.exec
      params:
        script: |
          cp ./build/Artifacts/DSIArtifacts/WorkloadOutput/reports/ycsb_100read/mongod.0/pre_post_task_output/default.profdata.tgz ./default.profdata.tgz || true
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: ./default.profdata.tgz
        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/default.profdata.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: PGO profile data - Execution ${execution}
    - command: shell.exec
      params:
        working_dir: src
        script: |
          # removes files from the (local) scons cache when it's over a
          # threshold, to the $prune_ratio percentage. Ideally override
          # these default values in the distro config in evergreen.

          if [ -d "${scons_cache_path}" ]; then
              /opt/mongodbtoolchain/v4/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
          fi
  f_other_pre_ops:
    - &f_other_pre_ops
      command: shell.exec
      params:
        silent: true
        script: |
          for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
              pkill -9 "$PS"
          done
  f_other_timeout:
    # Can't be empty so just `echo`.
    - command: shell.exec
      params: {script: "echo"}

  ###
  # Package the supplementary DSI data (mongo tools, feature flag list,
  # server params list, jstests, and buildscripts)
  download mongo tools:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o verbose
          set -o errexit

          arch=$(uname -m)
          case $arch in
            "x86_64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-x86_64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-x86_64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            "aarch64")
              if [ -f /etc/os-release ]; then
                . /etc/os-release
                if [ "$ID" == "amzn" ]; then
                  case $VERSION_ID in
                    "2")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2-aarch64-100.9.4.tgz"
                      ;;
                    "2023")
                      binary_url="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-amazon2023-aarch64-100.9.4.tgz"
                      ;;
                    *)
                      echo "Unsupported Amazon Linux version: $VERSION_ID"
                      exit 1
                      ;;
                  esac
                else
                  echo "Unsupported Linux distribution: $ID"
                  exit 1
                fi
              else
                echo "Unable to determine Linux distribution"
                exit 1
              fi
              ;;
            *)
              echo "Unsupported architecture: $arch"
              exit 1
              ;;
          esac

          echo "Downloading and installing MongoDB tools..."
          mkdir -p mongodb/
          wget "$binary_url" -O mongo-tools.tar.gz
          tar -xzvf mongo-tools.tar.gz -C mongodb/ --strip-components=1 "mong*/bin"
          echo "MongoDB tools installed successfully!"
  generate atlas info:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose

          uarch=$(uname -p)
          os=$(uname -r)
          json="{ \"version\": \"${version}\", \"gitVersion\": \"${revision}\", \"uarch\": \"$uarch\", \"os\": \"$os\" }"
          echo $json | jq '.' > mongodb/atlas_info.json
  generate feature flag list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_feature_flag_list.py
          mkdir -p mongodb/feature_flags
          cp ./all_feature_flags.txt mongodb/feature_flags
  generate server params list:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          pushd ..
          . src/evergreen/prelude.sh
          activate_venv
          popd

          python ./buildscripts/idl/gen_all_server_params_list.py
          mkdir -p mongodb/server_params
          cp ./all_server_params.txt mongodb/server_params
  copy jstests:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/jstests/hooks
          if [ -d jstests/hooks ]
          then
            echo "Fetching JS test DB correctness checks from directory jstests"
            cp -a jstests/* mongodb/jstests

            echo "Now adding our own special run_validate_collections.js wrapper"
            mv mongodb/jstests/hooks/run_validate_collections.js mongodb/jstests/hooks/run_validate_collections.actual.js

            cat << EOF > mongodb/jstests/hooks/run_validate_collections.js
            print("NOTE: run_validate_collections.js will skip the oplog!");
            TestData = { skipValidationNamespaces: ['local.oplog.rs'] };
            await import("jstests/hooks/run_validate_collections.actual.js");
          EOF
          fi
  copy buildscripts:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          mkdir -p mongodb/buildscripts
          cp -a buildscripts/* mongodb/buildscripts
  package supplementary data:
    - command: shell.exec
      params:
        working_dir: src
        script: |
          if [ -z "${build_patch_id}" ] || [ -z "${reuse_compile_from}" ] || [ "${is_patch|false}" = "false" ]; then
            tar czf supplementary-data.tgz mongodb
          else
            # Evergreen does not handle nested escaped expansions well
            reuse_compile_from="${reuse_compile_from}"
            build_patch_id="${build_patch_id}"
            version_to_reuse_from=$(if [ -n "${build_patch_id}" ]; then echo "${build_patch_id}"; else echo "${reuse_compile_from}"; fi)

            curl -o supplementary-data.tgz https://s3.amazonaws.com/mciuploads/${project}/${compile_variant}/$${version_to_reuse_from}/dsi/supplementary-data.tgz
          fi
  upload supplementary data:
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/supplementary-data.tgz
        remote_file: ${project}/${compile_variant}/${version_id}/dsi/supplementary-data.tgz
        bucket: mciuploads
        permissions: public-read
        content_type: application/x-gzip
        display_name: supplementary-data.tgz

tasks:
  - name: package_supplementary_data
    depends_on:
      - name: version_expansions_gen
        variant: generate-tasks-for-version
    commands:
      - command: manifest.load
      - command: git.get_project
        params:
          directory: src
          clone_depth: 1000
      - func: "set up venv"
      - func: "get version expansions"
      - func: "apply version expansions"
      - func: "download mongo tools"
      - func: "generate feature flag list"
      - func: "generate atlas info"
      - func: "generate server params list"
      - func: "copy jstests"
      - func: "copy buildscripts"
      - func: "package supplementary data"
      - func: "upload supplementary data"

buildvariants:
  # Replacement for generate-tasks-for-version from
  # evergreen_yml_components/variants/misc/task_generation.yml.
  - name: generate-tasks-for-version
    display_name: "~ Generate tasks for evergreen version"
    activate: true
    run_on:
      - rhel8.8-medium
    tasks:
      - name: version_expansions_gen

  - name: amazon2-x86-compile
    display_name: "* Compile for Amazon Linux 2 x86"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-x86-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  - name: amazon2-arm64-compile
    display_name: "* Compile for Amazon Linux 2 arm64"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  - name: amazon2-arm64-compile-pgo
    display_name: "* Compile for Amazon Linux 2 arm64 PGO"
    expansions:
      platform: linux
      project_dir: dsi
      has_packages: false
      use_scons_cache: false
      pgo_profile_url: https://mciuploads.s3.amazonaws.com/dsi/perf-3-node-replSet.arm.aws.2024-05/b49fc707fb83a45727ad3f9f7cfa4c940a6805fe/sys_perf_perf_3_node_replSet.arm.aws.2024_05_ycsb.100read.2024_05_patch_b49fc707fb83a45727ad3f9f7cfa4c940a6805fe_66b035d75b27a40007f52c27_24_08_05_02_16_15/66b035d75b27a40007f52c27/logs/default.profdata.tgz
      compile_variant: amazon2-arm64-compile-pgo
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        BAZEL_FLAGS=--keep_going
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --thin-lto
        --pgo
        --variables-files=etc/scons/mongodbtoolchain_stable_clang.vars
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  - name: amazon2-arm64-mongocrypt-shlib-compile
    display_name: "* Compile mongo_crypt_v1.so for Amazon Linux 2 arm64"
    modules: *modules
    cron: "0 0 * * 4" # 00:00 on Thursday
    tags: ["bazel_check"]
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      use_scons_cache: true
      compile_flags: >-
        --separate-debug
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --allocator=system
        --enterprise-features=fle
        --js-engine=none
        --link-model=dynamic-sdk
        --enable-http-client=off
        --ssl=off
        SHLINKFLAGS_EXTRA="-Wl,-Bsymbolic -Wl,--no-gnu-unique"
        CCFLAGS="-fno-gnu-unique"
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: .crypt

  - name: amazon2-x86-streams-compile
    display_name: "* Compile Streams for Amazon Linux 2 x86"
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-x86-streams-compile
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
        --streams-release-build
    run_on:
      - amazon2-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
      - name: package_supplementary_data

  # This variant tests the reuse_compile_from feature
  # by reusing compile artifacts from another compile
  # in the same patch.
  - name: test-reuse-compile-from
    display_name: "~ Test reuse_compile_from"
    cron: "0 0 * * *" # Every day at 00:00
    expansions:
      <<: *shared_compile_expansions
      compile_variant: amazon2-arm64-compile
      reuse_compile_from: ${version_id}
      compile_flags: >-
        --ssl
        MONGO_DISTMOD=amazon2
        -j$(grep -c ^processor /proc/cpuinfo)
        --release
        --variables-files=etc/scons/mongodbtoolchain_stable_gcc.vars
    run_on:
      - amazon2-arm64-xlarge
    tasks:
      - name: compile_and_package_serial_no_unittests_TG
        depends_on:
          - name: archive_dist_test
            variant: amazon2-arm64-compile
          - name: archive_dist_test_debug
            variant: amazon2-arm64-compile