summaryrefslogtreecommitdiff
path: root/evergreen
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /evergreen
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'evergreen')
-rwxr-xr-xevergreen/blackduck_hub.sh17
-rwxr-xr-xevergreen/blackduck_setup.sh15
-rwxr-xr-xevergreen/burn_in_tests.sh20
-rw-r--r--evergreen/burn_in_tests_generate.sh12
-rwxr-xr-xevergreen/cleanup_environment.sh11
-rw-r--r--evergreen/commit_message_validate.sh23
-rwxr-xr-xevergreen/compiled_binaries_get.sh9
-rwxr-xr-xevergreen/do_jepsen_setup/install_jepsen.sh5
-rwxr-xr-xevergreen/external_auth_aws_setup.sh6
-rw-r--r--evergreen/failed_unittests_gather.sh6
-rwxr-xr-xevergreen/feature_flag_tags_check.sh12
-rwxr-xr-xevergreen/functions/added_and_modified_patch_files_get.sh5
-rwxr-xr-xevergreen/functions/binary_version_check.sh16
-rw-r--r--evergreen/functions/compile_expansions_generate.sh12
-rwxr-xr-xevergreen/functions/credentials_setup.sh2
-rw-r--r--evergreen/functions/task_timeout_determine.sh9
-rw-r--r--evergreen/functions/venv_setup.sh24
-rwxr-xr-xevergreen/functions/version_expansions_generate.sh24
-rw-r--r--evergreen/garasign_gpg_crypt_sign.sh33
-rw-r--r--evergreen/garasign_gpg_sign.sh65
-rw-r--r--evergreen/garasign_jsign_sign.sh44
-rwxr-xr-xevergreen/gen_feature_flags.sh2
-rw-r--r--evergreen/generate_buildid_debug_symbols_mapping.sh8
-rw-r--r--evergreen/generate_version.sh5
-rw-r--r--evergreen/generate_version_burn_in.sh19
-rw-r--r--evergreen/jepsen_docker/list-append.sh32
-rw-r--r--evergreen/jepsen_docker/setup.sh46
-rw-r--r--evergreen/jepsen_test_run.sh5
-rw-r--r--evergreen/jstestfuzz_minimize.sh4
-rw-r--r--evergreen/jstestfuzz_run.sh9
-rw-r--r--evergreen/jstestfuzz_setup.sh15
-rw-r--r--evergreen/kitchen_run.sh40
-rw-r--r--evergreen/lint_fuzzer_sanity_all.sh10
-rw-r--r--evergreen/lint_fuzzer_sanity_patch.sh15
-rw-r--r--evergreen/macos_notary.py95
-rw-r--r--evergreen/multiversion_setup.sh32
-rw-r--r--evergreen/ninja_compile.sh5
-rw-r--r--evergreen/notary_client_crypt_run.sh20
-rw-r--r--evergreen/notary_client_run.sh23
-rwxr-xr-xevergreen/prelude.sh16
-rw-r--r--evergreen/prelude_db_contrib_tool.sh7
-rw-r--r--evergreen/prelude_mongo_task_generator.sh24
-rw-r--r--evergreen/prelude_venv.sh21
-rwxr-xr-xevergreen/record_mongodb_server_version.sh9
-rw-r--r--evergreen/resmoke_tests_execute.sh25
-rw-r--r--evergreen/resmoke_tests_runtime_validate.sh1
-rwxr-xr-xevergreen/run_jstestfuzz/clone_repos.sh13
-rw-r--r--evergreen/run_python_script.sh11
-rw-r--r--evergreen/sasl_windows_cyrussasl_setup.sh18
-rw-r--r--evergreen/sasl_windows_cyrussasl_teardown.sh18
-rwxr-xr-xevergreen/scons_compile.sh11
-rwxr-xr-xevergreen/selinux_run_test.sh10
-rwxr-xr-xevergreen/selinux_test_executor.sh240
53 files changed, 425 insertions, 754 deletions
diff --git a/evergreen/blackduck_hub.sh b/evergreen/blackduck_hub.sh
new file mode 100755
index 00000000000..f7edf2d486e
--- /dev/null
+++ b/evergreen/blackduck_hub.sh
@@ -0,0 +1,17 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+# activate the virtualenv if it has been set up
+activate_venv
+
+additional_args=
+if [ "$branch_name" != "master" ]; then
+ additional_args="--vulnerabilities_only"
+fi
+
+python buildscripts/blackduck_hub.py -v scan_and_report --build_logger=mci.buildlogger --build_logger_task_id=${task_id} --report_file=report.json $additional_args
diff --git a/evergreen/blackduck_setup.sh b/evergreen/blackduck_setup.sh
new file mode 100755
index 00000000000..8ed396833a3
--- /dev/null
+++ b/evergreen/blackduck_setup.sh
@@ -0,0 +1,15 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+# Create the Evergreen API credentials
+cat > .restconfig.json << END_OF_CREDS
+{
+"baseurl": "${blackduck_url}",
+"username": "${blackduck_username}",
+"password": "${blackduck_password}",
+"debug": false,
+"insecure" : false
+}
+END_OF_CREDS
diff --git a/evergreen/burn_in_tests.sh b/evergreen/burn_in_tests.sh
new file mode 100755
index 00000000000..5f6124a5f47
--- /dev/null
+++ b/evergreen/burn_in_tests.sh
@@ -0,0 +1,20 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+activate_venv
+
+# Capture a list of new and modified tests. The expansion macro burn_in_tests_build_variant
+# is used to for finding the associated tasks from a different build variant than the
+# burn_in_tests_gen task executes on.
+build_variant_opts="--build-variant=${build_variant}"
+if [ -n "${burn_in_tests_build_variant}" ]; then
+ build_variant_opts="--build-variant=${burn_in_tests_build_variant} --run-build-variant=${build_variant}"
+fi
+burn_in_args="$burn_in_args --repeat-tests-min=2 --repeat-tests-max=1000 --repeat-tests-secs=600"
+# Evergreen executable is in $HOME.
+PATH="$PATH:$HOME" eval $python buildscripts/evergreen_burn_in_tests.py --project=${project} $build_variant_opts --distro=${distro_id} --generate-tasks-file=burn_in_tests_gen.json --task_id ${task_id} $burn_in_args --verbose --install-dir "${install_dir}"
diff --git a/evergreen/burn_in_tests_generate.sh b/evergreen/burn_in_tests_generate.sh
new file mode 100644
index 00000000000..6e2499d5e39
--- /dev/null
+++ b/evergreen/burn_in_tests_generate.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+activate_venv
+
+# Evergreen executable is in $HOME.
+PATH=$PATH:$HOME $python buildscripts/burn_in_tags.py --expansion-file ../expansions.yml --install-dir "${install_dir}"
diff --git a/evergreen/cleanup_environment.sh b/evergreen/cleanup_environment.sh
index 4efc9f8d346..ec442defb40 100755
--- a/evergreen/cleanup_environment.sh
+++ b/evergreen/cleanup_environment.sh
@@ -1,13 +1,4 @@
set -o verbose
-rm -rf \
- /data/db/* \
- mongo-diskstats* \
- mongo-*.tgz \
- ~/.aws \
- ~/.boto \
- venv \
- /data/install \
- /data/multiversion
-
+rm -rf /data/db/* mongo-diskstats* mongo-*.tgz ~/.aws ~/.boto venv
exit 0
diff --git a/evergreen/commit_message_validate.sh b/evergreen/commit_message_validate.sh
index 6a66a2113cc..45602695a95 100644
--- a/evergreen/commit_message_validate.sh
+++ b/evergreen/commit_message_validate.sh
@@ -1,25 +1,14 @@
-# Note: This script is called "commit_message_validate", but it does not actually
-# evaluate the commit message. Rather, it leverages the fact that a GitHub merge queue
-# triggered branch will have a single commit message that is the name of the PR.
-# This script ensures the PR name matches a well-known regex (see below for details).
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
+
cd src
+
set -o verbose
set -o errexit
if [ "${is_commit_queue}" = "true" ]; then
- # Fetch the gitlog from the last merge commit to HEAD, which is guaranteed
- # to be the commit message of the PR.
- gitlog=$(git log --pretty=format:'%s' ${branch_name}...HEAD 2>&1)
- # In case there are any special characters that could cause issues (like "). To
- # do this, we will write it out to a file, then read that file into a variable.
- cat > commit_message.txt << END_OF_COMMIT_MSG
-${gitlog}
-END_OF_COMMIT_MSG
- commit_message_content=$(cat commit_message.txt)
activate_venv
- $python buildscripts/validate_commit_message.py "$commit_message_content"
-else
- echo "Not a commit queue PR, skipping commit message validation"
- exit 0
+ $python -m pip --disable-pip-version-check install --upgrade cryptography==36.0.2 || exit 1
+ $python buildscripts/validate_commit_message.py \
+ --evg-config-file ./.evergreen.yml \
+ ${version_id}
fi
diff --git a/evergreen/compiled_binaries_get.sh b/evergreen/compiled_binaries_get.sh
index 42372bb93dd..b2d726452a9 100755
--- a/evergreen/compiled_binaries_get.sh
+++ b/evergreen/compiled_binaries_get.sh
@@ -6,9 +6,8 @@ cd src
set -o errexit
set -o verbose
-# activate_venv will make sure we are using python 3
-activate_venv
-setup_db_contrib_tool
+setup_db_contrib_tool_venv
+activate_db_contrib_tool_venv
rm -rf /data/install /data/multiversion
@@ -49,5 +48,5 @@ db-contrib-tool setup-repro-env \
--architecture $architecture \
$version
-dist_test_dir=$(find /data/install -type d -iname "dist-test")
-mv "$dist_test_dir" "$(pwd)"
+version_dir=$(find /data/install -type d -iname "*$version*")
+mv $version_dir/dist-test $(pwd)
diff --git a/evergreen/do_jepsen_setup/install_jepsen.sh b/evergreen/do_jepsen_setup/install_jepsen.sh
index e92b684935e..7af559692e4 100755
--- a/evergreen/do_jepsen_setup/install_jepsen.sh
+++ b/evergreen/do_jepsen_setup/install_jepsen.sh
@@ -1,7 +1,10 @@
set -o errexit
cd src
-git clone --branch=v0.2.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb
+git clone --branch=jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb
cd jepsen-mongodb
+branch=$(git symbolic-ref --short HEAD)
+commit=$(git show -s --pretty=format:"%h - %an, %ar: %s")
+echo "Git branch: $branch, commit: $commit"
lein install
diff --git a/evergreen/external_auth_aws_setup.sh b/evergreen/external_auth_aws_setup.sh
index 233da94c338..c59b025028c 100755
--- a/evergreen/external_auth_aws_setup.sh
+++ b/evergreen/external_auth_aws_setup.sh
@@ -17,6 +17,10 @@ cat << EOF > aws_e2e_setup.json
"iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
"iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
- "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}"
+ "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
+
+ "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
+ "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
+ "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}"
}
EOF
diff --git a/evergreen/failed_unittests_gather.sh b/evergreen/failed_unittests_gather.sh
index 83b18ab0f0e..9c61c14fef0 100644
--- a/evergreen/failed_unittests_gather.sh
+++ b/evergreen/failed_unittests_gather.sh
@@ -6,11 +6,7 @@ cd src
set -eou pipefail
# Only run on unit test tasks so we don't target mongod binaries from cores.
-if [ "${task_name}" != "run_unittests" ] && [ "${task_name}" != "run_dbtest" ] \
- && [ "${task_name}" != "run_unittests_with_recording" ] \
- && [[ ${task_name} != integration_tests* ]] \
- && [[ "${task_name}" != compile_and_run_unittests_* ]]; then
- echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}"
+if [ "${task_name}" != "run_unittests" ] && [ "${task_name}" != "run_dbtest" ] && [ "${task_name}" != "run_unittests_with_recording" ]; then
exit 0
fi
diff --git a/evergreen/feature_flag_tags_check.sh b/evergreen/feature_flag_tags_check.sh
index 3d2cf818142..5b046843a76 100755
--- a/evergreen/feature_flag_tags_check.sh
+++ b/evergreen/feature_flag_tags_check.sh
@@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise"
diff_file_name="with_base_upstream.diff"
# get the list of feature flags from the patched version
-$python buildscripts/idl/gen_all_feature_flag_list.py
+$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
mv all_feature_flags.txt patch_all_feature_flags.txt
# get the list of feature flags from the base commit
@@ -21,7 +21,15 @@ if [ -s "$diff_file_name" ]; then
git apply -R "$diff_file_name"
fi
-$python buildscripts/idl/gen_all_feature_flag_list.py
+# This script has to be run on an Evergreen variant or local repo with the enterprise module.
+pushd "$enterprise_path"
+git --no-pager diff "$(git merge-base origin/${branch_name} HEAD)" --output="$diff_file_name" --binary
+if [ -s "$diff_file_name" ]; then
+ git apply -R "$diff_file_name"
+fi
+popd
+
+$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src
mv all_feature_flags.txt base_all_feature_flags.txt
# print out the list of tests that previously had feature flag tag, that was
diff --git a/evergreen/functions/added_and_modified_patch_files_get.sh b/evergreen/functions/added_and_modified_patch_files_get.sh
index 7bc89909742..608b53787ef 100755
--- a/evergreen/functions/added_and_modified_patch_files_get.sh
+++ b/evergreen/functions/added_and_modified_patch_files_get.sh
@@ -7,3 +7,8 @@ set -o verbose
set -o errexit
git diff --name-only origin/${branch_name}... --line-prefix="${workdir}/src/" --diff-filter=d >> modified_and_created_patch_files.txt
+if [ -d src/mongo/db/modules/enterprise ]; then
+ pushd src/mongo/db/modules/enterprise
+ git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >> ~1/modified_and_created_patch_files.txt
+ popd
+fi
diff --git a/evergreen/functions/binary_version_check.sh b/evergreen/functions/binary_version_check.sh
new file mode 100755
index 00000000000..ac01374a07e
--- /dev/null
+++ b/evergreen/functions/binary_version_check.sh
@@ -0,0 +1,16 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+mongo_binary=dist-test/bin/mongo${exe}
+activate_venv
+bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
+# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
+mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
+# The versions must match
+if [ "$bin_ver" != "$mongo_ver" ]; then
+ echo "The mongo version is $mongo_ver, expected version is $bin_ver"
+ exit 1
+fi
diff --git a/evergreen/functions/compile_expansions_generate.sh b/evergreen/functions/compile_expansions_generate.sh
index fe06d0fdf7d..dc9642f5cd4 100644
--- a/evergreen/functions/compile_expansions_generate.sh
+++ b/evergreen/functions/compile_expansions_generate.sh
@@ -5,6 +5,14 @@ cd src
set -o errexit
set -o verbose
+# We get the raw version string (r1.2.3-45-gabcdef) from git
+MONGO_VERSION=$(git describe --abbrev=7)
+# If this is a patch build, we add the patch version id to the version string so we know
+# this build was a patch, and which evergreen task it came from
+if [ "${is_patch}" = "true" ]; then
+ MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
+fi
+echo "MONGO_VERSION = ${MONGO_VERSION}"
activate_venv
# shared scons cache testing
# if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true
@@ -35,12 +43,12 @@ if [ ! -z ${scons_cache_scope} ]; then
set -o errexit
fi
echo "Shared Cache with setting: ${scons_cache_scope}"
- SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
+ MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
# Legacy Expansion generation
else
echo "Using legacy expansion generation"
# Proceed with regular expansions generated
# This script converts the generated version string into a sanitized version string for
# use by scons and uploading artifacts as well as information about for the scons cache.
- SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
+ MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml
fi
diff --git a/evergreen/functions/credentials_setup.sh b/evergreen/functions/credentials_setup.sh
index 55a9df5282e..15be8ea1e66 100755
--- a/evergreen/functions/credentials_setup.sh
+++ b/evergreen/functions/credentials_setup.sh
@@ -6,7 +6,7 @@ cd src
cat > mci.buildlogger << END_OF_CREDS
slavename='${slave}'
passwd='${passwd}'
-builder='${build_variant}_${project}'
+builder='MCI_${build_variant}'
build_num=${builder_num}
build_phase='${task_name}_${execution}'
END_OF_CREDS
diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh
index 49dda4dd4a9..79d393bb964 100644
--- a/evergreen/functions/task_timeout_determine.sh
+++ b/evergreen/functions/task_timeout_determine.sh
@@ -25,20 +25,13 @@ if [[ -n "${burn_in_bypass}" ]]; then
build_variant_for_timeout=${burn_in_bypass}
fi
-if [[ -n "${alias}" ]]; then
- evg_alias=${alias}
-else
- evg_alias="evg-alias-absent"
-fi
-
activate_venv
PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_factor \
--install-dir "${install_dir}" \
--task-name ${task_name} \
--suite-name ${suite_name} \
- --project ${project} \
--build-variant $build_variant_for_timeout \
- --evg-alias $evg_alias \
+ --evg-alias '${alias}' \
--timeout ${timeout_secs} \
--exec-timeout ${exec_timeout_secs} \
--evg-api-config ./.evergreen.yml \
diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh
index c6244686084..3d935d87594 100644
--- a/evergreen/functions/venv_setup.sh
+++ b/evergreen/functions/venv_setup.sh
@@ -74,22 +74,10 @@ activate_venv
echo "Upgrading pip to 21.0.1"
python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1
-count=0
-while :; do
- if ! python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log; then
- count=$((count + 1))
- if [[ $count < 5 ]]; then
- rand=$((1 + $RANDOM % 3))
- # delay = 20+[1-3] -> 80+[4-12] -> 180+[9-27] -> 320+[16-48]
- sleep $((count * count * (20 + rand)))
- continue
- fi
- echo "Pip install error"
- cat install.log || true
- exit 1
- else
- break
- fi
-done
-
+python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log
+if [ $? != 0 ]; then
+ echo "Pip install error"
+ cat install.log
+ exit 1
+fi
python -m pip freeze > pip-requirements.txt
diff --git a/evergreen/functions/version_expansions_generate.sh b/evergreen/functions/version_expansions_generate.sh
deleted file mode 100755
index 75c89b94420..00000000000
--- a/evergreen/functions/version_expansions_generate.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/../prelude.sh"
-
-cd src
-
-set -o errexit
-set -o verbose
-# We get the raw version string (r1.2.3-45-gabcdef) from git
-MONGO_VERSION=$(git describe --abbrev=7)
-
-# If the project is sys-perf (or related), add the string -sys-perf to the version
-if [[ "${project}" == sys-perf* ]]; then
- MONGO_VERSION="$MONGO_VERSION-sys-perf"
-fi
-
-# If this is a patch build, we add the patch version id to the version string so we know
-# this build was a patch, and which evergreen task it came from
-if [ "${is_patch}" = "true" ]; then
- MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
-fi
-echo "MONGO_VERSION = ${MONGO_VERSION}"
-
-activate_venv
-MONGO_VERSION=${MONGO_VERSION} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_version_expansions.py --out version_expansions.yml
diff --git a/evergreen/garasign_gpg_crypt_sign.sh b/evergreen/garasign_gpg_crypt_sign.sh
deleted file mode 100644
index 9ae7d568ac9..00000000000
--- a/evergreen/garasign_gpg_crypt_sign.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60}" >> "signing-envfile"
-echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60}" >> "signing-envfile"
-
-set -o errexit
-set -o verbose
-
-ext="${ext:-tgz}"
-
-crypt_file_name=mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}
-mv "mongo_crypt_shared_v1.$ext" $crypt_file_name
-
-# generating checksums
-shasum -a 1 $crypt_file_name | tee $crypt_file_name.sha1
-shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256
-md5sum $crypt_file_name | tee $crypt_file_name.md5
-
-# signing crypt linux artifact with gpg
-cat << EOF >> gpg_signing_commands.sh
-gpgloader # loading gpg keys.
-gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name
-EOF
-
-podman run \
- --env-file=signing-envfile \
- --rm \
- -v $(pwd):$(pwd) -w $(pwd) \
- ${garasign_gpg_image} \
- /bin/bash -c "$(cat ./gpg_signing_commands.sh)"
diff --git a/evergreen/garasign_gpg_sign.sh b/evergreen/garasign_gpg_sign.sh
deleted file mode 100644
index ea83f39d04b..00000000000
--- a/evergreen/garasign_gpg_sign.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60}" >> "signing-envfile"
-echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60}" >> "signing-envfile"
-
-set -o errexit
-set -o verbose
-
-long_ext=${ext}
-if [ "$long_ext" == "tgz" ]; then
- long_ext="tar.gz"
-fi
-
-mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext}
-mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true
-mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true
-mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true
-mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true
-
-# generating checksums
-function gen_checksums() {
- if [ -e $1 ]; then
- shasum -a 1 $1 | tee $1.sha1
- shasum -a 256 $1 | tee $1.sha256
- md5sum $1 | tee $1.md5
- else
- echo "$1 does not exist. Skipping checksum generation"
- fi
-}
-
-gen_checksums mongodb-$push_name-$push_arch-$suffix.$ext
-gen_checksums mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
-gen_checksums mongodb-src-$src_suffix.$long_ext
-gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
-
-# signing linux artifacts with gpg
-cat << 'EOF' > gpg_signing_commands.sh
-gpgloader # loading gpg keys.
-function sign(){
- if [ -e $1 ]
- then
- gpg --yes -v --armor -o $1.sig --detach-sign $1
- else
- echo "$1 does not exist. Skipping signing"
- fi
-}
-
-EOF
-
-cat << EOF >> gpg_signing_commands.sh
-sign mongodb-$push_name-$push_arch-$suffix.$ext
-sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext
-sign mongodb-src-$src_suffix.$long_ext
-sign mongodb-cryptd-$push_name-$push_arch-$suffix.$ext
-EOF
-
-podman run \
- --env-file=signing-envfile \
- --rm \
- -v $(pwd):$(pwd) -w $(pwd) \
- ${garasign_gpg_image} \
- /bin/bash -c "$(cat ./gpg_signing_commands.sh)"
diff --git a/evergreen/garasign_jsign_sign.sh b/evergreen/garasign_jsign_sign.sh
deleted file mode 100644
index 7a3737ecd7d..00000000000
--- a/evergreen/garasign_jsign_sign.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >> "signing-envfile"
-echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >> "signing-envfile"
-
-set -o errexit
-set -o verbose
-
-msi_filename=mongodb-${push_name}-${push_arch}-${suffix}.msi
-/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename $msi_filename || true
-
-# signing windows artifacts with jsign
-cat << 'EOF' > jsign_signing_commands.sh
-function sign(){
- if [ -e $1 ]
- then
- jsign -a mongo-authenticode-2021 --replace --tsaurl http://timestamp.digicert.com -d SHA-256 $1
- else
- echo "$1 does not exist. Skipping signing"
- fi
-}
-EOF
-cat << EOF >> jsign_signing_commands.sh
-sign $msi_filename
-EOF
-
-podman run \
- --env-file=signing-envfile \
- --rm \
- -v $(pwd):$(pwd) -w $(pwd) \
- ${garasign_jsign_image} \
- /bin/bash -c "$(cat ./jsign_signing_commands.sh)"
-
-# generating checksums
-if [ -e $msi_filename ]; then
- shasum -a 1 $msi_filename | tee $msi_filename.sha1
- shasum -a 256 $msi_filename | tee $msi_filename.sha256
- md5sum $msi_filename | tee $msi_filename.md5
-else
- echo "$msi_filename does not exist. Skipping checksum generation"
-fi
diff --git a/evergreen/gen_feature_flags.sh b/evergreen/gen_feature_flags.sh
index b06f614c243..4e6dc0e9b4c 100755
--- a/evergreen/gen_feature_flags.sh
+++ b/evergreen/gen_feature_flags.sh
@@ -6,4 +6,4 @@ cd src
set -o errexit
set -o verbose
activate_venv
-$python buildscripts/idl/gen_all_feature_flag_list.py
+$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src
diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh
index 142614ce821..d1866e961a5 100644
--- a/evergreen/generate_buildid_debug_symbols_mapping.sh
+++ b/evergreen/generate_buildid_debug_symbols_mapping.sh
@@ -6,16 +6,10 @@ cd src
set -o errexit
set -o verbose
-is_san_variant_arg=""
-if [[ -n "${san_options}" ]]; then
- is_san_variant_arg="--is-san-variant"
-fi
-
activate_venv
$python buildscripts/debugsymb_mapper.py \
--version "${version_id}" \
--client-id "${symbolizer_client_id}" \
--client-secret "${symbolizer_client_secret}" \
- --variant "${build_variant}" \
- $is_san_variant_arg
+ --variant "${build_variant}"
diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh
index 32785c0338b..db38741fbbd 100644
--- a/evergreen/generate_version.sh
+++ b/evergreen/generate_version.sh
@@ -6,12 +6,13 @@ cd src
set -o errexit
set -o verbose
-setup_mongo_task_generator
+curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.3.6/mongo-task-generator --output mongo-task-generator
+chmod +x mongo-task-generator
+
activate_venv
PATH=$PATH:$HOME:/ ./mongo-task-generator \
--expansion-file ../expansions.yml \
--evg-auth-file ./.evergreen.yml \
--evg-project-file ${evergreen_config_file_path} \
--generate-sub-tasks-config etc/generate_subtasks_config.yml \
- --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \
$@
diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh
deleted file mode 100644
index b92b9fcfd16..00000000000
--- a/evergreen/generate_version_burn_in.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-set -o errexit
-set -o verbose
-
-setup_mongo_task_generator
-activate_venv
-$python buildscripts/burn_in_tests.py generate-test-membership-map-file-for-ci
-PATH=$PATH:$HOME:/ ./mongo-task-generator \
- --expansion-file ../expansions.yml \
- --evg-auth-file ./.evergreen.yml \
- --evg-project-file ${evergreen_config_file_path} \
- --generate-sub-tasks-config etc/generate_subtasks_config.yml \
- --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \
- --burn-in \
- $@
diff --git a/evergreen/jepsen_docker/list-append.sh b/evergreen/jepsen_docker/list-append.sh
index 66192de57cf..3eb8a8b67fa 100644
--- a/evergreen/jepsen_docker/list-append.sh
+++ b/evergreen/jepsen_docker/list-append.sh
@@ -10,22 +10,7 @@ cd jepsen/docker
# actually run the tests
start_time=$(date +%s)
-sudo docker exec jepsen-control bash --login -c "\
-cd /jepsen/mongodb && \
-lein run test-all -w list-append \
--n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 \
--r 1000 \
---concurrency 3n \
---time-limit 120 \
---max-writes-per-key 128 \
---read-concern majority \
---write-concern majority \
---txn-read-concern snapshot \
---txn-write-concern majority \
---nemesis-interval 1 \
---nemesis partition \
---test-count 30" \
- | tee jepsen_${task_name}_${execution}.log
+sudo docker exec jepsen-control bash --login -c "cd /jepsen/mongodb && lein run test-all -w list-append -n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 -r 1000 --concurrency 3n --time-limit 240 --max-writes-per-key 128 --read-concern majority --write-concern majority --txn-read-concern snapshot --txn-write-concern majority --nemesis-interval 1 --nemesis partition --test-count 30" | tee jepsen_${task_name}_${execution}.log
end_time=$(date +%s)
elapsed_secs=$((end_time - start_time))
@@ -33,17 +18,8 @@ elapsed_secs=$((end_time - start_time))
cd ../../
mkdir -p src/jepsen-mongodb
sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store
-cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb/
+cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb
sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir
-cd src
-activate_venv
-$python buildscripts/jepsen_report.py --start_time=$start_time --end_time=$end_time --elapsed=$elapsed_secs --emit_status_files --store ./jepsen-mongodb jepsen-mongodb/jepsen_${task_name}_${execution}.log
-exit_code=$?
-
-if [ -f "jepsen_system_fail.txt" ]; then
- mv jepsen_system_fail.txt jepsen-mongodb/jepsen_system_failure_${task_name}_${execution}
- exit 0
-fi
-
-exit $exit_code
+cd src/jepsen-mongodb
+. ../evergreen/jepsen_report.sh
diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh
index 2512f03a6a1..cc76e20e541 100644
--- a/evergreen/jepsen_docker/setup.sh
+++ b/evergreen/jepsen_docker/setup.sh
@@ -1,15 +1,43 @@
set -euo pipefail
-# Clone our internal fork of jepsen-io/jepsen to get the core
-# functionality with a few tweaks meant for evergreen integration.
-git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepsen
+if [ ! $(which docker) ]; then
+ sudo apt-get update
+ sudo apt-get install -yq \
+ apt-transport-https \
+ ca-certificates \
+ curl \
+ gnupg \
+ lsb-release
-# Copy our mongodb source for jepsen to run into the docker area to be
-# copied into the image during the build process.
-cp -rf src/dist-test jepsen/docker/node
+ if [ ! -f "/usr/share/keyrings/docker-archive-keyring.gpg" ]; then
+ curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+ fi
+
+ set +e
+ if ! grep "https://download.docker.com/linux/debian" "/etc/apt/sources.list.d/docker.list"; then
+ echo \
+ "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \
+ $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+ fi
+ set -e
+
+ sudo apt-get update
+ sudo apt-get install -yq docker-ce docker-ce-cli containerd.io
+ sudo docker run hello-world
+fi
-# Clone our internal tests to run
-git clone --branch=v0.2.0 git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
+if [ ! $(which docker-compose) ]; then
+ sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose
+ sudo chmod +x /usr/bin/docker-compose
+fi
+
+sudo chmod 777 /var/run/docker.sock
+
+git clone --branch=evergreen-master git@github.com:10gen/jepsen.git jepsen
+cp -rf src/dist-test jepsen/docker/node
+# place the mongodb jepsen test adjacent to the control node's Dockerfile.
+# it'll get copied into the image during the build process
+git clone --branch=no-download-master git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb
-# Kill any running containers
+# kill any running containers
sudo docker container kill $(docker ps -q) || true
diff --git a/evergreen/jepsen_test_run.sh b/evergreen/jepsen_test_run.sh
index b5f79426079..2cf8b0ed387 100644
--- a/evergreen/jepsen_test_run.sh
+++ b/evergreen/jepsen_test_run.sh
@@ -16,11 +16,6 @@ mkdir -p $TMPDIR
export _JAVA_OPTIONS=-Djava.io.tmpdir=$TMPDIR
start_time=$(date +%s)
-# XXX Do not run this test in a loop or with Jepsen's repeat flag
-# the report generator (jepsen_report.sh) does not support parsing more than one
-# test at a time. If you need to run these tests with a repeat flag, you
-# will need to adopt buildscripts/jepsen_report.py to support the log output
-# of older-style Jepsen tests
lein run test --test ${jepsen_test_name} \
--mongodb-dir ../ \
--working-dir ${workdir}/src/jepsen-workdir \
diff --git a/evergreen/jstestfuzz_minimize.sh b/evergreen/jstestfuzz_minimize.sh
index e1a3f54a1a4..13ddbaf5709 100644
--- a/evergreen/jstestfuzz_minimize.sh
+++ b/evergreen/jstestfuzz_minimize.sh
@@ -6,6 +6,8 @@ cd src/jstestfuzz
set -o errexit
set -o verbose
+add_nodejs_to_path
+
if [ -f "../minimizer-outputs.json" ]; then
- eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json"
+ eval npm run ${npm_command} -- -j "../minimizer-outputs.json"
fi
diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh
index 9e6f94ee65b..d79ed69ac9b 100644
--- a/evergreen/jstestfuzz_run.sh
+++ b/evergreen/jstestfuzz_run.sh
@@ -1,10 +1,13 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
-set -evo pipefail
-
cd src/jstestfuzz
+set -o errexit
+set -o verbose
+
+add_nodejs_to_path
+
in_patch_build_flag=""
if [[ "${is_patch}" = "true" ]]; then
case "${npm_command}" in
@@ -14,4 +17,4 @@ if [[ "${is_patch}" = "true" ]]; then
esac
fi
-./src/scripts/npm_run.sh ${npm_command} -- ${jstestfuzz_vars} ${in_patch_build_flag} --branch ${branch_name}
+eval npm run "${npm_command}" -- "${jstestfuzz_vars}" "${in_patch_build_flag}" --branch "${branch_name}"
diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh
index 83623ca6f16..f0d2b7a4c29 100644
--- a/evergreen/jstestfuzz_setup.sh
+++ b/evergreen/jstestfuzz_setup.sh
@@ -6,12 +6,11 @@ cd src
set -o errexit
set -o verbose
-for i in {1..5}; do
- git clone git@github.com:10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5
- echo "Failed to clone git@github.com:10gen/jstestfuzz.git, retrying..."
-done
+add_nodejs_to_path
-if [ $RET -ne 0 ]; then
- echo "Failed to clone git@github.com:10gen/jstestfuzz.git"
- exit $RET
-fi
+git clone git@github.com:10gen/jstestfuzz.git
+
+pushd jstestfuzz
+npm install
+npm run prepare
+popd
diff --git a/evergreen/kitchen_run.sh b/evergreen/kitchen_run.sh
new file mode 100644
index 00000000000..321e8196e7e
--- /dev/null
+++ b/evergreen/kitchen_run.sh
@@ -0,0 +1,40 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src/buildscripts/package_test
+
+set -o errexit
+
+export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz"
+export KITCHEN_SECURITY_GROUP="${kitchen_security_group}"
+export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}"
+export KITCHEN_SUBNET="${kitchen_subnet}"
+export KITCHEN_VPC="${kitchen_vpc}"
+
+if [ ${packager_distro} == "suse12" ]; then
+ export KITCHEN_YAML="kitchen.legacy.yml"
+fi
+
+if [[ "${packager_arch}" == "aarch64" || "${packager_arch}" == "arm64" ]]; then
+ kitchen_packager_distro="${packager_distro}-arm64"
+else
+ kitchen_packager_distro="${packager_distro}-x86-64"
+fi
+
+activate_venv
+# set expiration tag 2 hours in the future, since no test should take this long
+export KITCHEN_EXPIRE="$($python -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')"
+
+for i in {1..3}; do
+ if ! kitchen verify $kitchen_packager_distro; then
+ verified="false"
+ kitchen destroy $kitchen_packager_distro || true
+ sleep 30
+ else
+ verified="true"
+ break
+ fi
+done
+
+kitchen destroy $kitchen_packager_distro || true
+test "$verified" = "true"
diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh
index 9c2159824e1..16ae53b923c 100644
--- a/evergreen/lint_fuzzer_sanity_all.sh
+++ b/evergreen/lint_fuzzer_sanity_all.sh
@@ -1,14 +1,12 @@
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
-cd src/jstestfuzz
+cd src
set -eo pipefail
set -o verbose
-# Run first with help which will do the install
-# Then we can run it in parallel
-./src/scripts/npm_run.sh --help
+add_nodejs_to_path
+
# Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
-# Skip javascript files in third_party directory
-find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles --
+find "$PWD/jstests" "$PWD/src/mongo/db/modules/enterprise" -name "*.js" -print | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles --
diff --git a/evergreen/lint_fuzzer_sanity_patch.sh b/evergreen/lint_fuzzer_sanity_patch.sh
index af1afd63da7..11196aff1ea 100644
--- a/evergreen/lint_fuzzer_sanity_patch.sh
+++ b/evergreen/lint_fuzzer_sanity_patch.sh
@@ -6,16 +6,15 @@ cd src
set -eo pipefail
set -o verbose
-mkdir -p jstestfuzzinput jstestfuzzoutput
+add_nodejs_to_path
-# We need to be the jstestfuzz repo for node to install/run
-cd jstestfuzz
+mkdir -p jstestfuzzinput jstestfuzzoutput
-indir="$(pwd)/../jstestfuzzinput"
-outdir="$(pwd)/../jstestfuzzoutput"
+indir="$(pwd)/jstestfuzzinput"
+outdir="$(pwd)/jstestfuzzoutput"
# Grep all the js files from modified_and_created_patch_files.txt and put them into $indir.
-(grep -v "\.tpl\.js$" ../modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true)
+(grep -v "\.tpl\.js$" modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true)
# Count the number of files in $indir.
if [[ "$(ls -A $indir)" ]]; then
@@ -26,8 +25,8 @@ if [[ "$(ls -A $indir)" ]]; then
num_files=50
fi
- ./src/scripts/npm_run.sh jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50
+ npm run --prefix jstestfuzz jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50
# Run parse-jsfiles on 50 files at a time with 32 processes in parallel.
- ls -1 -d $outdir/* | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles --
+ ls -1 -d $outdir/* | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles --
fi
diff --git a/evergreen/macos_notary.py b/evergreen/macos_notary.py
deleted file mode 100644
index ef799a8e5db..00000000000
--- a/evergreen/macos_notary.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import os
-import platform
-import shutil
-import urllib.request
-import subprocess
-import zipfile
-import stat
-import sys
-
-if platform.system().lower() != 'darwin':
- print("Not a macos system, skipping macos signing.")
- sys.exit(0)
-
-if len(sys.argv) < 2:
- print("Must provide at least 1 archive to sign.")
- sys.exit(1)
-
-supported_archs = {
- 'arm64': 'arm64',
- 'x86_64': 'amd64'
-}
-arch = platform.uname().machine.lower()
-
-if arch not in supported_archs:
- print(f"Unsupported platform uname arch: {arch}, must be {supported_archs.keys()}")
- sys.exit(1)
-
-macnotary_name = f'darwin_{supported_archs[arch]}'
-
-if os.environ['project'] == "mongodb-mongo-master-nightly":
- signing_type = 'notarizeAndSign'
-else:
- signing_type = 'sign'
-
-macnotary_url = f'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/{macnotary_name}.zip'
-print(f'Fetching macnotary tool from: {macnotary_url}')
-local_filename, headers = urllib.request.urlretrieve(macnotary_url, f'{macnotary_name}.zip')
-with zipfile.ZipFile(f'{macnotary_name}.zip') as zipf:
- zipf.extractall()
-
-st = os.stat(f'{macnotary_name}/macnotary')
-os.chmod(f'{macnotary_name}/macnotary', st.st_mode | stat.S_IEXEC)
-
-failed = False
-archives = sys.argv[1:]
-
-for archive in archives:
- archive_base, archive_ext = os.path.splitext(archive)
- unsigned_archive = f'{archive_base}_unsigned{archive_ext}'
- shutil.move(archive, unsigned_archive)
-
- signing_cmd = [
- f'./{macnotary_name}/macnotary',
- '-f', f'{unsigned_archive}',
- '-m', f'{signing_type}',
- '-u', 'https://dev.macos-notary.build.10gen.cc/api',
- '-k', 'server',
- '--entitlements', 'etc/macos_entitlements.xml',
- '--verify',
- '-b', 'server.mongodb.com',
- '-i', f'{os.environ["task_id"]}',
- '-c', f'{os.environ["project"]}',
- '-o', f'{archive}'
- ]
-
- signing_env = os.environ.copy()
- signing_env['MACOS_NOTARY_SECRET'] = os.environ["macos_notarization_secret"]
- print(' '.join(signing_cmd))
- p = subprocess.Popen(signing_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=signing_env)
-
- print(f"Signing tool completed with exitcode: {p.returncode}")
- for line in iter(p.stdout.readline, b''):
- print(f'macnotary: {line.decode("utf-8").strip()}')
-
- # TODO: BUILD-14595 remove timeout when codesign doesn't frequently hang on macos hosts
- timeout = 3600
- timed_out = False
- try:
- p.wait(timeout=timeout)
- except subprocess.TimeoutExpired:
- print(f"ERROR: failed to finish signing in timeout period of {timeout} seconds. This most likely is related to hung codesign, see issues underlying BUILD-14595.")
- timed_out = True
- pass
-
- if timed_out:
- shutil.move(unsigned_archive, archive)
- elif p.returncode != 0:
- failed = True
- shutil.move(unsigned_archive, archive)
- else:
- os.unlink(unsigned_archive)
-
-if failed:
- exit(1)
-
diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh
index f20257297ff..1e932a0c143 100644
--- a/evergreen/multiversion_setup.sh
+++ b/evergreen/multiversion_setup.sh
@@ -7,7 +7,8 @@ set -o errexit
set -o verbose
activate_venv
-setup_db_contrib_tool
+
+setup_db_contrib_tool_venv
export PIPX_HOME="${workdir}/pipx"
export PIPX_BIN_DIR="${workdir}/pipx/bin"
@@ -93,30 +94,5 @@ db-contrib-tool setup-repro-env \
--fallbackToMaster \
--resmokeCmd "python buildscripts/resmoke.py" \
--debug \
- $last_continuous_arg 4.4 5.0.28
-
-# Certain build variants define additional multiversion_*_last_lts expansions in order to
-# be able to fetch a complete set of versions.
-
-if [ ! -z "${multiversion_edition_last_lts}" ]; then
- edition="${multiversion_edition_last_lts}"
-fi
-
-if [ ! -z "${multiversion_platform_last_lts}" ]; then
- platform="${multiversion_platform_last_lts}"
-fi
-
-if [ ! -z "${multiversion_architecture_last_lts}" ]; then
- architecture="${multiversion_architecture_last_lts}"
-fi
-
-db-contrib-tool setup-repro-env \
- --installDir /data/install \
- --linkDir /data/multiversion \
- --edition $edition \
- --platform $platform \
- --architecture $architecture \
- --fallbackToMaster \
- --resmokeCmd "python buildscripts/resmoke.py" \
- --debug \
- $last_lts_arg
+ $last_lts_arg \
+ $last_continuous_arg 4.4 5.0
diff --git a/evergreen/ninja_compile.sh b/evergreen/ninja_compile.sh
index 823944277be..f3b51ae5e1d 100644
--- a/evergreen/ninja_compile.sh
+++ b/evergreen/ninja_compile.sh
@@ -5,7 +5,6 @@ cd src
set -o errexit
set -o verbose
-
activate_venv
python -m pip install ninja
if [ "Windows_NT" = "$OS" ]; then
@@ -14,8 +13,8 @@ if [ "Windows_NT" = "$OS" ]; then
for i in "${compile_env[@]}"; do
echo "set $i" >> msvc.bat
done
- echo "ninja -f ${ninja_file} install-core" >> msvc.bat
+ echo "ninja install-core" >> msvc.bat
cmd /C msvc.bat
else
- eval ${compile_env} ninja -f ${ninja_file} install-core
+ eval ${compile_env} ninja install-core
fi
diff --git a/evergreen/notary_client_crypt_run.sh b/evergreen/notary_client_crypt_run.sh
new file mode 100644
index 00000000000..2d9b6b3d689
--- /dev/null
+++ b/evergreen/notary_client_crypt_run.sh
@@ -0,0 +1,20 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+. ./notary_env.sh
+
+set -o errexit
+set -o verbose
+
+ext="${ext:-tgz}"
+
+mv "mongo_crypt_shared_v1.$ext" mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}
+
+/usr/local/bin/notary-client.py \
+ --key-name "server-6.0" \
+ --auth-token-file ${workdir}/src/signing_auth_token \
+ --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" \
+ --notary-url http://notary-service.build.10gen.cc:5000 \
+ mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext}
diff --git a/evergreen/notary_client_run.sh b/evergreen/notary_client_run.sh
new file mode 100644
index 00000000000..82526660d96
--- /dev/null
+++ b/evergreen/notary_client_run.sh
@@ -0,0 +1,23 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+. ./notary_env.sh
+
+set -o errexit
+set -o verbose
+
+long_ext=${ext}
+if [ "$long_ext" == "tgz" ]; then
+ long_ext="tar.gz"
+fi
+
+mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext}
+mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true
+mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true
+mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true
+mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true
+/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-${push_name}-${push_arch}-${suffix}.msi || true
+
+/usr/local/bin/notary-client.py --key-name "server-6.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} mongodb-${push_name}-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${long_ext} mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext}
diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
index eb671d4b5ae..3844afaead5 100755
--- a/evergreen/prelude.sh
+++ b/evergreen/prelude.sh
@@ -11,7 +11,6 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
. "$evergreen_dir/prelude_python.sh"
. "$evergreen_dir/prelude_venv.sh"
. "$evergreen_dir/prelude_db_contrib_tool.sh"
-. "$evergreen_dir/prelude_mongo_task_generator.sh"
expansions_yaml="$evergreen_dir/../../expansions.yml"
expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
@@ -32,6 +31,21 @@ unset expansions_default_yaml
unset script
unset evergreen_dir
+function add_nodejs_to_path {
+ # Add node and npm binaries to PATH
+ if [ "Windows_NT" = "$OS" ]; then
+ # An "npm" directory might not have been created in %APPDATA% by the Windows installer.
+ # Work around the issue by specifying a different %APPDATA% path.
+ # See: https://github.com/nodejs/node-v0.x-archive/issues/8141
+ export APPDATA=${workdir}/npm-app-data
+ export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location
+ # TODO: this is to work around BUILD-8652
+ cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')"
+ else
+ export PATH="$PATH:/opt/node/bin"
+ fi
+}
+
function posix_workdir {
if [ "Windows_NT" = "$OS" ]; then
echo $(cygpath -u "${workdir}")
diff --git a/evergreen/prelude_db_contrib_tool.sh b/evergreen/prelude_db_contrib_tool.sh
index 44f80987c42..59c03d7684a 100644
--- a/evergreen/prelude_db_contrib_tool.sh
+++ b/evergreen/prelude_db_contrib_tool.sh
@@ -1,12 +1,11 @@
-function setup_db_contrib_tool {
+function setup_db_contrib_tool_venv {
- mkdir -p ${workdir}/pipx
+ mkdir ${workdir}/pipx
export PIPX_HOME="${workdir}/pipx"
export PIPX_BIN_DIR="${workdir}/pipx/bin"
export PATH="$PATH:$PIPX_BIN_DIR"
- export PIP_CACHE_DIR=${workdir}/pip_cache
python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1
python -m pip --disable-pip-version-check install "pipx" || exit 1
- pipx install "db-contrib-tool==0.8.3" --pip-args="--no-cache-dir" || exit 1
+ pipx install "db-contrib-tool==0.4.5" || exit 1
}
diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh
deleted file mode 100644
index 81e0e6ccce3..00000000000
--- a/evergreen/prelude_mongo_task_generator.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-function setup_mongo_task_generator {
- if [ ! -f mongo-task-generator ]; then
- curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.10/mongo-task-generator --output mongo-task-generator
- chmod +x mongo-task-generator
- fi
-}
-
-## Comment above and uncomment below to test unreleased mongo-task-generator changes that are
-## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git`
-## repo
-#function setup_mongo_task_generator {
-# if [ ! -f mongo-task-generator ]; then
-#
-# curl https://sh.rustup.rs -sSf | sh -s -- -y
-# source "$HOME/.cargo/env"
-# git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator
-# pushd unreleased-mongo-task-generator
-# git checkout <branch-name>
-# cargo build --release --locked
-# generator_path="$(pwd)/target/release/mongo-task-generator"
-# popd
-# cp "$generator_path" mongo-task-generator
-# fi
-#}
diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh
index a0851406f57..6b97a8a9cfb 100644
--- a/evergreen/prelude_venv.sh
+++ b/evergreen/prelude_venv.sh
@@ -22,26 +22,9 @@ function activate_venv {
if [ "Windows_NT" = "$OS" ]; then
export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
- elif [ "$(uname)" = "Darwin" ]; then
- #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting
- #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path.
- python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
- export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src"
else
- python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
- site_packages="${workdir}/venv/lib/python${python_version}/site-packages"
- python -c "import sys; print(sys.path)"
-
- # Check if site_packages is already in sys.path
- in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)")
- if [ "$in_sys_path" = "False" ]; then
- export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src"
- else
- export PYTHONPATH="$PYTHONPATH:${workdir}/src"
- fi
- python -c "import sys; print(sys.path)"
+ export PYTHONPATH="$PYTHONPATH:${workdir}/src"
fi
- export PIP_CACHE_DIR=${workdir}/pip_cache
- echo "python set to $(which $python) and python version: $($python --version)"
+ echo "python set to $(which $python)"
}
diff --git a/evergreen/record_mongodb_server_version.sh b/evergreen/record_mongodb_server_version.sh
deleted file mode 100755
index e7ad5785657..00000000000
--- a/evergreen/record_mongodb_server_version.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-cd src
-
-set -o errexit
-set -o verbose
-
-"$1" --version > "$2"
diff --git a/evergreen/resmoke_tests_execute.sh b/evergreen/resmoke_tests_execute.sh
index 911a85414fd..ce1ecc7b582 100644
--- a/evergreen/resmoke_tests_execute.sh
+++ b/evergreen/resmoke_tests_execute.sh
@@ -14,19 +14,15 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
# activate the virtualenv if it has been set up
activate_venv
- # Install db-contrib-tool to symbolize crashes during resmoke suite runs
- # This is not supported on Windows and MacOS, so doing it only on Linux
- if [ "$(uname)" == "Linux" ]; then
- setup_db_contrib_tool
- fi
if [[ -f "patch_test_tags.tgz" ]]; then
tags_build_variant="${build_variant}"
if [[ "${build_variant}" =~ .*"-query-patch-only" ]]; then
- # Use the RHEL 8 all feature flags variant for the SBE variant. The original SBE variant is
- # not a required builder and therefore not captured in patch test failure history.
- tags_build_variant="enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required"
+ # Use the RHEL 8 all feature flags variant for the classic engine variant. The original
+ # classic engine variant is not a required builder and therefore not captured in patch
+ # test failure history.
+ tags_build_variant="enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required"
fi
$python buildscripts/testmatrix/getdisplaytaskname.py "${task_name}" "${build_variant}" > display_task_name.txt
@@ -104,18 +100,10 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
set -o errexit
# Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant.
- if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then
+ if [[ "${build_variant}" = "enterprise-rhel-80-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then
extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\""
fi
- # Even though all feature flags may be enabled on a variant, often times we do not want to run
- # feature flag tests because they will most likely fail. For example, during multiversion testing,
- # all feature flags may be enabled on the latest version, but running feature flag specific tests on
- # older versions (last-lts/last-continuous) will likely fail because those features most likely do not exist.
- if [[ ${run_no_feature_flag_tests} == "true" ]]; then
- extra_args="$extra_args --runNoFeatureFlagTests"
- fi
-
path_value="$PATH:/data/multiversion"
# Set the suite name to be the task name by default; unless overridden with the `suite` expansion.
@@ -142,7 +130,7 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
$extra_args \
${test_flags} \
--suites=${suite_name} \
- --log=${resmoke_logger} \
+ --log=buildlogger \
--staggerJobs=on \
--installDir=${install_dir} \
--buildId=${build_id} \
@@ -155,7 +143,6 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
--taskName=${task_name} \
--variantName=${build_variant} \
--versionId=${version_id} \
- --taskWorkDir='${workdir}' \
--reportFile=report.json \
--perfReportFile=perf.json \
--cedarReportFile=cedar_report.json
diff --git a/evergreen/resmoke_tests_runtime_validate.sh b/evergreen/resmoke_tests_runtime_validate.sh
index aee2111ff92..1ccfc71c225 100644
--- a/evergreen/resmoke_tests_runtime_validate.sh
+++ b/evergreen/resmoke_tests_runtime_validate.sh
@@ -9,6 +9,7 @@ set -o errexit
activate_venv
$python buildscripts/resmoke_tests_runtime_validate.py \
--resmoke-report-file ./report.json \
+ --evg-api-config ./.evergreen.yml \
--project-id ${project_id} \
--build-variant ${build_variant} \
--task-name ${task_name}
diff --git a/evergreen/run_jstestfuzz/clone_repos.sh b/evergreen/run_jstestfuzz/clone_repos.sh
index 74605e058ef..b64e91fd4d3 100755
--- a/evergreen/run_jstestfuzz/clone_repos.sh
+++ b/evergreen/run_jstestfuzz/clone_repos.sh
@@ -2,14 +2,5 @@ set -o errexit
set -o verbose
cd src
-for i in {1..5}; do
- git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1
- echo "Failed to clone git@github.com:10gen/QA.git, retrying..."
-done
-
-if [ $RET -ne 0 ]; then
- echo "Failed to clone git@github.com:10gen/QA.git"
- exit $RET
-fi
-
-cp -r src/mongo/db/modules/enterprise jstests/enterprise_tests
+git clone --depth 1 git@github.com:10gen/mongo-enterprise-modules.git jstests/enterprise_tests
+git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests
diff --git a/evergreen/run_python_script.sh b/evergreen/run_python_script.sh
deleted file mode 100644
index 35181dec3c1..00000000000
--- a/evergreen/run_python_script.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-unset workdir
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-set -o errexit
-set -o verbose
-
-cd src
-
-activate_venv
-$python $@
diff --git a/evergreen/sasl_windows_cyrussasl_setup.sh b/evergreen/sasl_windows_cyrussasl_setup.sh
deleted file mode 100644
index 149bb563325..00000000000
--- a/evergreen/sasl_windows_cyrussasl_setup.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-readonly k_cyrussasl_plugin_filename="cyrus_sasl_windows_test_plugin.dll"
-readonly k_cyrussasl_plugin_dir="/cygdrive/c/CMU/bin/sasl2"
-
-plugin_path="$(find . -name "*${k_cyrussasl_plugin_filename}")"
-
-if [[ -z "$plugin_path" ]]; then
- echo >&2 "Could not find ${k_cyrussasl_plugin_filename} from path '$(pwd)' !"
- exit 1
-fi
-
-echo "Configuring CyrusSASL plugin .dll from '$plugin_path'"
-
-mkdir -p "$k_cyrussasl_plugin_dir"
-
-cp "$plugin_path" "$k_cyrussasl_plugin_dir"
diff --git a/evergreen/sasl_windows_cyrussasl_teardown.sh b/evergreen/sasl_windows_cyrussasl_teardown.sh
deleted file mode 100644
index 60e65055643..00000000000
--- a/evergreen/sasl_windows_cyrussasl_teardown.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
-. "$DIR/prelude.sh"
-
-if [ "${task_name}" != "sasl_windows_cyrussasl" ]; then
- exit 0
-fi
-
-echo "Cleaning up Windows CyrusSASL Test Artifacts"
-
-readonly k_cyrussasl_default_dir_root="/cygdrive/c/CMU"
-
-if [[ ! -d "$k_cyrussasl_default_dir_root" ]]; then
- echo "Could not find $k_cyrussasl_default_dir_root to cleanup..."
- exit 0
-fi
-
-rm -rf "$k_cyrussasl_default_dir_root"
-echo "Deleted $k_cyrussasl_default_dir_root from host"
diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh
index a6e2be4539e..af3ff0bad6d 100755
--- a/evergreen/scons_compile.sh
+++ b/evergreen/scons_compile.sh
@@ -8,6 +8,15 @@ set -o verbose
rm -rf ${install_directory}
+# Use hardlinks to reduce the disk space impact of installing
+# all of the binaries and associated debug info.
+
+# The expansion here is a workaround to let us set a different install-action
+# for tasks that don't support the one we set here. A better plan would be
+# to support install-action for Ninja builds directly.
+# TODO: https://jira.mongodb.org/browse/SERVER-48203
+extra_args="--install-action=${task_install_action}"
+
# By default, limit link jobs to one quarter of our overall -j
# concurrency unless locally overridden. We do this because in
# static link environments, the memory consumption of each
@@ -49,8 +58,6 @@ else
extra_args="$extra_args --release"
fi
-extra_args="$extra_args ENABLE_OOM_RETRY=1"
-
if [ "${generating_for_ninja}" = "true" ] && [ "Windows_NT" = "$OS" ]; then
vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
diff --git a/evergreen/selinux_run_test.sh b/evergreen/selinux_run_test.sh
index 15fe8be87b6..318d73adbe2 100755
--- a/evergreen/selinux_run_test.sh
+++ b/evergreen/selinux_run_test.sh
@@ -22,6 +22,8 @@ if [ "$bypass_prelude" != "yes" ]; then
src="src"
fi
+set -o xtrace
+
if [ "$hostname" == "" ]; then
hostname="$(tr -d '"[]{}' < "$workdir"/$src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')"
fi
@@ -50,12 +52,10 @@ connection_attempts=50
# Check for remote connectivity
set +o errexit
ssh_options="-i $ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no"
-while ! ssh -q $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do
- if [ "$attempts" -ge "$connection_attempts" ]; then
- printf "SSH connection attempt failed after %d attempts.\n" "$attempts"
- exit 1
- fi
+while ! ssh $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do
+ if [ "$attempts" -ge "$connection_attempts" ]; then exit 1; fi
((attempts++))
+ printf "SSH connection attempt %d/%d failed. Retrying...\n" "$attempts" "$connection_attempts"
sleep 10
done
diff --git a/evergreen/selinux_test_executor.sh b/evergreen/selinux_test_executor.sh
index fc1e04cf2be..a0616d314d6 100755
--- a/evergreen/selinux_test_executor.sh
+++ b/evergreen/selinux_test_executor.sh
@@ -1,177 +1,109 @@
#!/bin/bash
-set +o errexit
-readonly k_log_path="/var/log/mongodb/mongod.log"
-readonly k_mongo="$(pwd)/dist-test/bin/mongo"
-readonly k_test_path="$1"
-return_code=1
+set -o errexit
+set -o xtrace
-export PATH="$(dirname "$k_mongo"):$PATH"
+mongo="$(pwd)/dist-test/bin/mongo"
+export PATH="$(dirname "$mongo"):$PATH"
+if [ ! -f "$mongo" ]; then
+ echo "Mongo shell at $mongo is missing"
+ exit 1
+fi
-function print_err() {
+function print() {
echo "$@" >&2
}
function monitor_log() {
- sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f $k_log_path)
-}
-
-function output_ausearch() {
- local cmd_parameters="AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR"
-
- echo ""
- echo "====== SELinux errors (ausearch -m $cmd_parameters): ======"
- sudo --non-interactive ausearch -m $cmd_parameters -ts $1
-}
-
-function output_journalctl() {
- echo ""
- echo "============================== journalctl ========================================="
- sudo --non-interactive journalctl --no-pager --catalog --since="$1" | grep -i mongo
+ sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f /var/log/mongodb/mongod.log)
}
-function fail_and_exit_err() {
-
- echo ""
- echo "==================================================================================="
- echo "++++++++ Test failed, outputting last 5 seconds of additional log info ++++++++++++"
- echo "==================================================================================="
- output_ausearch "$(date --utc --date='5 seconds ago' '+%x %H:%M:%S')"
- output_journalctl "$(date --utc --date='5 seconds ago' +'%Y-%m-%d %H:%M:%S')"
-
- echo ""
- echo "==== FAIL: $1 ===="
+TEST_PATH="$1"
+if [ ! -f "$TEST_PATH" ]; then
+ print "No test supplied or test file not found. Run:"
+ print " $(basename "${BASH_SOURCE[0]}") <path>"
exit 1
-}
-
-function create_mongo_config() {
- echo "Writing /etc/mongod.conf for $k_test_path:"
- "$k_mongo" --nodb --norc --quiet --eval='
- assert(load("'"$k_test_path"'"));
- const test = new TestDefinition();
- print(JSON.stringify(test.config, null, 2));
-
- ' | sudo --non-interactive tee /etc/mongod.conf
-}
-
-function start_mongod() {
- # Start mongod and if it won't come up, fail and exit
-
- sudo --non-interactive systemctl start mongod \
- && sudo --non-interactive systemctl status mongod || (
- fail_and_exit_err "systemd failed to start mongod server!"
- )
-}
-
-function wait_for_mongod_to_accept_connections() {
- # Once the mongod process starts via systemd, it can still take a couple of seconds
- # to set up and accept connections... we will wait for log id 23016 to show up
- # indicating that the server is ready to accept incoming connections before starting the tests
-
- local server_ready=0
- local wait_seconds=2
- local wait_retries_max=30
- local wait_retries=0
-
- while [[ $wait_retries -le $wait_retries_max ]]; do
- local server_status="$(grep 23016 $k_log_path || echo "")"
-
- if [ "$server_status" != "" ]; then
- server_ready=1
- break
- fi
-
- sleep $wait_seconds
- ((wait_retries++))
- done
-
- if [ ! $server_ready ]; then
- fail_and_exit_err "failed to connect to mongod server after waiting for $(($wait_seconds * $wait_retries)) seconds!"
- fi
-}
-
-function clear_mongo_config() {
- # stop mongod, zero mongo log, clean up database, set all booleans to off
- sudo --non-interactive bash -c '
- systemctl stop mongod
-
- rm -f '"$k_log_path"'
- touch '"$k_log_path"'
- chown mongod '"$k_log_path"'
-
- rm -rf /var/lib/mongo/*
-
- rm -rf /etc/sysconfig/mongod /etc/mongod
-
- setsebool mongod_can_connect_ldap off
- setsebool mongod_can_use_kerberos off
- '
-}
-
-function exit_with_code() {
- exit $return_code
-}
-
-function setup_test_definition() {
- "$k_mongo" --nodb --norc --quiet --eval='
- assert(load("'"$k_test_path"'"));
- (() => {
- const test = new TestDefinition();
- print("Running setup() for '"$k_test_path"'");
- test.setup();
- })();
- '
-}
-
-function run_test() {
- "$k_mongo" --norc --gssapiServiceName=mockservice --eval='
- assert(load("'"$k_test_path"'"));
- print("Running test '"$k_test_path"'");
+fi
- (() => {
- const test = new TestDefinition();
+# test file is even good before going on
+if ! "$mongo" --nodb --norc --quiet "$TEST_PATH"; then
+ print "File $TEST_PATH has syntax errors"
+ exit 1
+fi
- try {
- test.run();
- } finally {
- test.teardown();
- }
- })();
- ' || fail_and_exit_err "Test failed"
+# stop mongod, zero mongo log, clean up database, set all booleans to off
+sudo --non-interactive bash -c '
+ systemctl stop mongod
- echo "SUCCESS: $k_test_path"
-}
+ rm -f /var/log/mongodb/mongod.log
+ touch /var/log/mongodb/mongod.log
+ chown mongod /var/log/mongodb/mongod.log
-if [ ! -f "$k_mongo" ]; then
- print_err "Mongo shell at $k_mongo is missing"
- exit 1
-fi
+ rm -rf /var/lib/mongo/*
-if [ ! -f "$k_test_path" ]; then
- print_err "No test supplied or test file not found. Run:"
- print_err "$(basename "${BASH_SOURCE[0]}") <path>"
- exit 1
-fi
+ rm -rf /etc/sysconfig/mongod /etc/mongod
-# Ensure file containing tests is valid before executing
-if ! "$k_mongo" --nodb --norc --quiet "$k_test_path"; then
- print_err "File $k_test_path has syntax errors"
- exit 1
-fi
+ setsebool mongod_can_connect_snmp off
+ setsebool mongod_can_connect_ldap off
+ setsebool mongod_can_use_kerberos off
+'
-echo "STARTING TEST: $k_test_path"
+# create mongo config
+"$mongo" --nodb --norc --quiet --eval='
+ assert(load("'"$TEST_PATH"'"));
+ const test = new TestDefinition();
+ print(typeof(test.config) === "string" ? test.config : JSON.stringify(test.config, null, 2));
+' | sudo --non-interactive tee /etc/mongod.conf
-clear_mongo_config
-create_mongo_config
-setup_test_definition
+# setup
+"$mongo" --nodb --norc --quiet --eval='
+ assert(load("'"$TEST_PATH"'"));
+ const test = new TestDefinition();
+ jsTest.log("Running setup()");
+ test.setup();
+'
# start log monitor, also kill it on exit
monitor_log &
-monitor_pid="$!"
-trap "sudo --non-interactive pkill -P $monitor_pid; exit_with_code" SIGINT SIGTERM ERR EXIT
-
-start_mongod
-wait_for_mongod_to_accept_connections
-run_test
+MONITORPID="$!"
+trap "sudo --non-interactive pkill -P $MONITORPID" SIGINT SIGTERM ERR EXIT
+
+# start mongod and if it won't come up, log SELinux errors
+ts="$(date --utc --date='1 seconds ago' '+%x %H:%M:%S')"
+tsj="$(date --utc --date='1 seconds ago' +'%Y-%m-%d %H:%M:%S')"
+sudo --non-interactive systemctl start mongod \
+ && sudo --non-interactive systemctl status mongod || (
+ set +o errexit
+ echo "=== SELinux errors:"
+ sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts
+ echo "=== journalctl --unit=mongod:"
+ sudo --non-interactive journalctl --no-pager --since="$tsj" --unit=mongod --unit=systemd --catalog
+ echo "=== /var/log/mongodb/mongod.log:"
+ sudo --non-interactive cat /var/log/mongodb/mongod.log
+ echo "==== FAIL: mongod service was not started successfully"
+ exit 1
+)
+
+# run test and teardown
+"$mongo" --norc --gssapiServiceName=mockservice --eval='
+ assert(load("'"$TEST_PATH"'"));
+ // name is such to prevent collisions
+ const test_812de7ce = new TestDefinition();
+ try {
+ jsTest.log("Running test");
+ test_812de7ce.run();
+ } finally {
+ test_812de7ce.teardown();
+ }
+' || (
+ echo "==== FAIL: test returned result: $?"
+ echo "=== SELinux errors:"
+ set +o errexit
+ sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts
+ echo "=== /var/log/mongodb/mongod.log:"
+ sudo --non-interactive cat /var/log/mongodb/mongod.log
+ exit 1
+)
-return_code=0
+set +o xtrace
+echo "SUCCESS: $TEST_PATH"