diff options
| author | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
|---|---|---|
| committer | Lucas de Castro Borges <lucas@gnuabordo.com.br> | 2025-02-11 15:07:35 -0300 |
| commit | 4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch) | |
| tree | 1682a647d4463397c119183369ae6f750d5fdcff /evergreen | |
| parent | aa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff) | |
| parent | 8f0827553e09872941945a093b647a4211a9db7f (diff) | |
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'evergreen')
53 files changed, 425 insertions, 754 deletions
diff --git a/evergreen/blackduck_hub.sh b/evergreen/blackduck_hub.sh new file mode 100755 index 00000000000..f7edf2d486e --- /dev/null +++ b/evergreen/blackduck_hub.sh @@ -0,0 +1,17 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +# activate the virtualenv if it has been set up +activate_venv + +additional_args= +if [ "$branch_name" != "master" ]; then + additional_args="--vulnerabilities_only" +fi + +python buildscripts/blackduck_hub.py -v scan_and_report --build_logger=mci.buildlogger --build_logger_task_id=${task_id} --report_file=report.json $additional_args diff --git a/evergreen/blackduck_setup.sh b/evergreen/blackduck_setup.sh new file mode 100755 index 00000000000..8ed396833a3 --- /dev/null +++ b/evergreen/blackduck_setup.sh @@ -0,0 +1,15 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +# Create the Evergreen API credentials +cat > .restconfig.json << END_OF_CREDS +{ +"baseurl": "${blackduck_url}", +"username": "${blackduck_username}", +"password": "${blackduck_password}", +"debug": false, +"insecure" : false +} +END_OF_CREDS diff --git a/evergreen/burn_in_tests.sh b/evergreen/burn_in_tests.sh new file mode 100755 index 00000000000..5f6124a5f47 --- /dev/null +++ b/evergreen/burn_in_tests.sh @@ -0,0 +1,20 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +activate_venv + +# Capture a list of new and modified tests. The expansion macro burn_in_tests_build_variant +# is used to for finding the associated tasks from a different build variant than the +# burn_in_tests_gen task executes on. +build_variant_opts="--build-variant=${build_variant}" +if [ -n "${burn_in_tests_build_variant}" ]; then + build_variant_opts="--build-variant=${burn_in_tests_build_variant} --run-build-variant=${build_variant}" +fi +burn_in_args="$burn_in_args --repeat-tests-min=2 --repeat-tests-max=1000 --repeat-tests-secs=600" +# Evergreen executable is in $HOME. +PATH="$PATH:$HOME" eval $python buildscripts/evergreen_burn_in_tests.py --project=${project} $build_variant_opts --distro=${distro_id} --generate-tasks-file=burn_in_tests_gen.json --task_id ${task_id} $burn_in_args --verbose --install-dir "${install_dir}" diff --git a/evergreen/burn_in_tests_generate.sh b/evergreen/burn_in_tests_generate.sh new file mode 100644 index 00000000000..6e2499d5e39 --- /dev/null +++ b/evergreen/burn_in_tests_generate.sh @@ -0,0 +1,12 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +activate_venv + +# Evergreen executable is in $HOME. +PATH=$PATH:$HOME $python buildscripts/burn_in_tags.py --expansion-file ../expansions.yml --install-dir "${install_dir}" diff --git a/evergreen/cleanup_environment.sh b/evergreen/cleanup_environment.sh index 4efc9f8d346..ec442defb40 100755 --- a/evergreen/cleanup_environment.sh +++ b/evergreen/cleanup_environment.sh @@ -1,13 +1,4 @@ set -o verbose -rm -rf \ - /data/db/* \ - mongo-diskstats* \ - mongo-*.tgz \ - ~/.aws \ - ~/.boto \ - venv \ - /data/install \ - /data/multiversion - +rm -rf /data/db/* mongo-diskstats* mongo-*.tgz ~/.aws ~/.boto venv exit 0 diff --git a/evergreen/commit_message_validate.sh b/evergreen/commit_message_validate.sh index 6a66a2113cc..45602695a95 100644 --- a/evergreen/commit_message_validate.sh +++ b/evergreen/commit_message_validate.sh @@ -1,25 +1,14 @@ -# Note: This script is called "commit_message_validate", but it does not actually -# evaluate the commit message. Rather, it leverages the fact that a GitHub merge queue -# triggered branch will have a single commit message that is the name of the PR. -# This script ensures the PR name matches a well-known regex (see below for details). DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" + cd src + set -o verbose set -o errexit if [ "${is_commit_queue}" = "true" ]; then - # Fetch the gitlog from the last merge commit to HEAD, which is guaranteed - # to be the commit message of the PR. - gitlog=$(git log --pretty=format:'%s' ${branch_name}...HEAD 2>&1) - # In case there are any special characters that could cause issues (like "). To - # do this, we will write it out to a file, then read that file into a variable. - cat > commit_message.txt << END_OF_COMMIT_MSG -${gitlog} -END_OF_COMMIT_MSG - commit_message_content=$(cat commit_message.txt) activate_venv - $python buildscripts/validate_commit_message.py "$commit_message_content" -else - echo "Not a commit queue PR, skipping commit message validation" - exit 0 + $python -m pip --disable-pip-version-check install --upgrade cryptography==36.0.2 || exit 1 + $python buildscripts/validate_commit_message.py \ + --evg-config-file ./.evergreen.yml \ + ${version_id} fi diff --git a/evergreen/compiled_binaries_get.sh b/evergreen/compiled_binaries_get.sh index 42372bb93dd..b2d726452a9 100755 --- a/evergreen/compiled_binaries_get.sh +++ b/evergreen/compiled_binaries_get.sh @@ -6,9 +6,8 @@ cd src set -o errexit set -o verbose -# activate_venv will make sure we are using python 3 -activate_venv -setup_db_contrib_tool +setup_db_contrib_tool_venv +activate_db_contrib_tool_venv rm -rf /data/install /data/multiversion @@ -49,5 +48,5 @@ db-contrib-tool setup-repro-env \ --architecture $architecture \ $version -dist_test_dir=$(find /data/install -type d -iname "dist-test") -mv "$dist_test_dir" "$(pwd)" +version_dir=$(find /data/install -type d -iname "*$version*") +mv $version_dir/dist-test $(pwd) diff --git a/evergreen/do_jepsen_setup/install_jepsen.sh b/evergreen/do_jepsen_setup/install_jepsen.sh index e92b684935e..7af559692e4 100755 --- a/evergreen/do_jepsen_setup/install_jepsen.sh +++ b/evergreen/do_jepsen_setup/install_jepsen.sh @@ -1,7 +1,10 @@ set -o errexit cd src -git clone --branch=v0.2.0-jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb +git clone --branch=jepsen-mongodb-master --depth=1 git@github.com:10gen/jepsen.git jepsen-mongodb cd jepsen-mongodb +branch=$(git symbolic-ref --short HEAD) +commit=$(git show -s --pretty=format:"%h - %an, %ar: %s") +echo "Git branch: $branch, commit: $commit" lein install diff --git a/evergreen/external_auth_aws_setup.sh b/evergreen/external_auth_aws_setup.sh index 233da94c338..c59b025028c 100755 --- a/evergreen/external_auth_aws_setup.sh +++ b/evergreen/external_auth_aws_setup.sh @@ -17,6 +17,10 @@ cat << EOF > aws_e2e_setup.json "iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}", "iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}", - "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}" + "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}", + + "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", + "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", + "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" } EOF diff --git a/evergreen/failed_unittests_gather.sh b/evergreen/failed_unittests_gather.sh index 83b18ab0f0e..9c61c14fef0 100644 --- a/evergreen/failed_unittests_gather.sh +++ b/evergreen/failed_unittests_gather.sh @@ -6,11 +6,7 @@ cd src set -eou pipefail # Only run on unit test tasks so we don't target mongod binaries from cores. -if [ "${task_name}" != "run_unittests" ] && [ "${task_name}" != "run_dbtest" ] \ - && [ "${task_name}" != "run_unittests_with_recording" ] \ - && [[ ${task_name} != integration_tests* ]] \ - && [[ "${task_name}" != compile_and_run_unittests_* ]]; then - echo "Not gathering failed unittests binaries as this is not a unittest task: ${task_name}" +if [ "${task_name}" != "run_unittests" ] && [ "${task_name}" != "run_dbtest" ] && [ "${task_name}" != "run_unittests_with_recording" ]; then exit 0 fi diff --git a/evergreen/feature_flag_tags_check.sh b/evergreen/feature_flag_tags_check.sh index 3d2cf818142..5b046843a76 100755 --- a/evergreen/feature_flag_tags_check.sh +++ b/evergreen/feature_flag_tags_check.sh @@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise" diff_file_name="with_base_upstream.diff" # get the list of feature flags from the patched version -$python buildscripts/idl/gen_all_feature_flag_list.py +$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src mv all_feature_flags.txt patch_all_feature_flags.txt # get the list of feature flags from the base commit @@ -21,7 +21,15 @@ if [ -s "$diff_file_name" ]; then git apply -R "$diff_file_name" fi -$python buildscripts/idl/gen_all_feature_flag_list.py +# This script has to be run on an Evergreen variant or local repo with the enterprise module. +pushd "$enterprise_path" +git --no-pager diff "$(git merge-base origin/${branch_name} HEAD)" --output="$diff_file_name" --binary +if [ -s "$diff_file_name" ]; then + git apply -R "$diff_file_name" +fi +popd + +$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src mv all_feature_flags.txt base_all_feature_flags.txt # print out the list of tests that previously had feature flag tag, that was diff --git a/evergreen/functions/added_and_modified_patch_files_get.sh b/evergreen/functions/added_and_modified_patch_files_get.sh index 7bc89909742..608b53787ef 100755 --- a/evergreen/functions/added_and_modified_patch_files_get.sh +++ b/evergreen/functions/added_and_modified_patch_files_get.sh @@ -7,3 +7,8 @@ set -o verbose set -o errexit git diff --name-only origin/${branch_name}... --line-prefix="${workdir}/src/" --diff-filter=d >> modified_and_created_patch_files.txt +if [ -d src/mongo/db/modules/enterprise ]; then + pushd src/mongo/db/modules/enterprise + git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >> ~1/modified_and_created_patch_files.txt + popd +fi diff --git a/evergreen/functions/binary_version_check.sh b/evergreen/functions/binary_version_check.sh new file mode 100755 index 00000000000..ac01374a07e --- /dev/null +++ b/evergreen/functions/binary_version_check.sh @@ -0,0 +1,16 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/../prelude.sh" + +cd src + +set -o errexit +mongo_binary=dist-test/bin/mongo${exe} +activate_venv +bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]') +# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();" +mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]') +# The versions must match +if [ "$bin_ver" != "$mongo_ver" ]; then + echo "The mongo version is $mongo_ver, expected version is $bin_ver" + exit 1 +fi diff --git a/evergreen/functions/compile_expansions_generate.sh b/evergreen/functions/compile_expansions_generate.sh index fe06d0fdf7d..dc9642f5cd4 100644 --- a/evergreen/functions/compile_expansions_generate.sh +++ b/evergreen/functions/compile_expansions_generate.sh @@ -5,6 +5,14 @@ cd src set -o errexit set -o verbose +# We get the raw version string (r1.2.3-45-gabcdef) from git +MONGO_VERSION=$(git describe --abbrev=7) +# If this is a patch build, we add the patch version id to the version string so we know +# this build was a patch, and which evergreen task it came from +if [ "${is_patch}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" +fi +echo "MONGO_VERSION = ${MONGO_VERSION}" activate_venv # shared scons cache testing # if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true @@ -35,12 +43,12 @@ if [ ! -z ${scons_cache_scope} ]; then set -o errexit fi echo "Shared Cache with setting: ${scons_cache_scope}" - SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml + MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml # Legacy Expansion generation else echo "Using legacy expansion generation" # Proceed with regular expansions generated # This script converts the generated version string into a sanitized version string for # use by scons and uploading artifacts as well as information about for the scons cache. - SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml + MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml fi diff --git a/evergreen/functions/credentials_setup.sh b/evergreen/functions/credentials_setup.sh index 55a9df5282e..15be8ea1e66 100755 --- a/evergreen/functions/credentials_setup.sh +++ b/evergreen/functions/credentials_setup.sh @@ -6,7 +6,7 @@ cd src cat > mci.buildlogger << END_OF_CREDS slavename='${slave}' passwd='${passwd}' -builder='${build_variant}_${project}' +builder='MCI_${build_variant}' build_num=${builder_num} build_phase='${task_name}_${execution}' END_OF_CREDS diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh index 49dda4dd4a9..79d393bb964 100644 --- a/evergreen/functions/task_timeout_determine.sh +++ b/evergreen/functions/task_timeout_determine.sh @@ -25,20 +25,13 @@ if [[ -n "${burn_in_bypass}" ]]; then build_variant_for_timeout=${burn_in_bypass} fi -if [[ -n "${alias}" ]]; then - evg_alias=${alias} -else - evg_alias="evg-alias-absent" -fi - activate_venv PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_factor \ --install-dir "${install_dir}" \ --task-name ${task_name} \ --suite-name ${suite_name} \ - --project ${project} \ --build-variant $build_variant_for_timeout \ - --evg-alias $evg_alias \ + --evg-alias '${alias}' \ --timeout ${timeout_secs} \ --exec-timeout ${exec_timeout_secs} \ --evg-api-config ./.evergreen.yml \ diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh index c6244686084..3d935d87594 100644 --- a/evergreen/functions/venv_setup.sh +++ b/evergreen/functions/venv_setup.sh @@ -74,22 +74,10 @@ activate_venv echo "Upgrading pip to 21.0.1" python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1 -count=0 -while :; do - if ! python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log; then - count=$((count + 1)) - if [[ $count < 5 ]]; then - rand=$((1 + $RANDOM % 3)) - # delay = 20+[1-3] -> 80+[4-12] -> 180+[9-27] -> 320+[16-48] - sleep $((count * count * (20 + rand))) - continue - fi - echo "Pip install error" - cat install.log || true - exit 1 - else - break - fi -done - +python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log +if [ $? != 0 ]; then + echo "Pip install error" + cat install.log + exit 1 +fi python -m pip freeze > pip-requirements.txt diff --git a/evergreen/functions/version_expansions_generate.sh b/evergreen/functions/version_expansions_generate.sh deleted file mode 100755 index 75c89b94420..00000000000 --- a/evergreen/functions/version_expansions_generate.sh +++ /dev/null @@ -1,24 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/../prelude.sh" - -cd src - -set -o errexit -set -o verbose -# We get the raw version string (r1.2.3-45-gabcdef) from git -MONGO_VERSION=$(git describe --abbrev=7) - -# If the project is sys-perf (or related), add the string -sys-perf to the version -if [[ "${project}" == sys-perf* ]]; then - MONGO_VERSION="$MONGO_VERSION-sys-perf" -fi - -# If this is a patch build, we add the patch version id to the version string so we know -# this build was a patch, and which evergreen task it came from -if [ "${is_patch}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" -fi -echo "MONGO_VERSION = ${MONGO_VERSION}" - -activate_venv -MONGO_VERSION=${MONGO_VERSION} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_version_expansions.py --out version_expansions.yml diff --git a/evergreen/garasign_gpg_crypt_sign.sh b/evergreen/garasign_gpg_crypt_sign.sh deleted file mode 100644 index 9ae7d568ac9..00000000000 --- a/evergreen/garasign_gpg_crypt_sign.sh +++ /dev/null @@ -1,33 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60}" >> "signing-envfile" - -set -o errexit -set -o verbose - -ext="${ext:-tgz}" - -crypt_file_name=mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} -mv "mongo_crypt_shared_v1.$ext" $crypt_file_name - -# generating checksums -shasum -a 1 $crypt_file_name | tee $crypt_file_name.sha1 -shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256 -md5sum $crypt_file_name | tee $crypt_file_name.md5 - -# signing crypt linux artifact with gpg -cat << EOF >> gpg_signing_commands.sh -gpgloader # loading gpg keys. -gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name -EOF - -podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_gpg_image} \ - /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_gpg_sign.sh b/evergreen/garasign_gpg_sign.sh deleted file mode 100644 index ea83f39d04b..00000000000 --- a/evergreen/garasign_gpg_sign.sh +++ /dev/null @@ -1,65 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -echo "GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60}" >> "signing-envfile" - -set -o errexit -set -o verbose - -long_ext=${ext} -if [ "$long_ext" == "tgz" ]; then - long_ext="tar.gz" -fi - -mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext} -mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true -mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true -mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true -mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true - -# generating checksums -function gen_checksums() { - if [ -e $1 ]; then - shasum -a 1 $1 | tee $1.sha1 - shasum -a 256 $1 | tee $1.sha256 - md5sum $1 | tee $1.md5 - else - echo "$1 does not exist. Skipping checksum generation" - fi -} - -gen_checksums mongodb-$push_name-$push_arch-$suffix.$ext -gen_checksums mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext -gen_checksums mongodb-src-$src_suffix.$long_ext -gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext - -# signing linux artifacts with gpg -cat << 'EOF' > gpg_signing_commands.sh -gpgloader # loading gpg keys. -function sign(){ - if [ -e $1 ] - then - gpg --yes -v --armor -o $1.sig --detach-sign $1 - else - echo "$1 does not exist. Skipping signing" - fi -} - -EOF - -cat << EOF >> gpg_signing_commands.sh -sign mongodb-$push_name-$push_arch-$suffix.$ext -sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext -sign mongodb-src-$src_suffix.$long_ext -sign mongodb-cryptd-$push_name-$push_arch-$suffix.$ext -EOF - -podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_gpg_image} \ - /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_jsign_sign.sh b/evergreen/garasign_jsign_sign.sh deleted file mode 100644 index 7a3737ecd7d..00000000000 --- a/evergreen/garasign_jsign_sign.sh +++ /dev/null @@ -1,44 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -echo "GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username}" >> "signing-envfile" -echo "GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password}" >> "signing-envfile" - -set -o errexit -set -o verbose - -msi_filename=mongodb-${push_name}-${push_arch}-${suffix}.msi -/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename $msi_filename || true - -# signing windows artifacts with jsign -cat << 'EOF' > jsign_signing_commands.sh -function sign(){ - if [ -e $1 ] - then - jsign -a mongo-authenticode-2021 --replace --tsaurl http://timestamp.digicert.com -d SHA-256 $1 - else - echo "$1 does not exist. Skipping signing" - fi -} -EOF -cat << EOF >> jsign_signing_commands.sh -sign $msi_filename -EOF - -podman run \ - --env-file=signing-envfile \ - --rm \ - -v $(pwd):$(pwd) -w $(pwd) \ - ${garasign_jsign_image} \ - /bin/bash -c "$(cat ./jsign_signing_commands.sh)" - -# generating checksums -if [ -e $msi_filename ]; then - shasum -a 1 $msi_filename | tee $msi_filename.sha1 - shasum -a 256 $msi_filename | tee $msi_filename.sha256 - md5sum $msi_filename | tee $msi_filename.md5 -else - echo "$msi_filename does not exist. Skipping checksum generation" -fi diff --git a/evergreen/gen_feature_flags.sh b/evergreen/gen_feature_flags.sh index b06f614c243..4e6dc0e9b4c 100755 --- a/evergreen/gen_feature_flags.sh +++ b/evergreen/gen_feature_flags.sh @@ -6,4 +6,4 @@ cd src set -o errexit set -o verbose activate_venv -$python buildscripts/idl/gen_all_feature_flag_list.py +$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh index 142614ce821..d1866e961a5 100644 --- a/evergreen/generate_buildid_debug_symbols_mapping.sh +++ b/evergreen/generate_buildid_debug_symbols_mapping.sh @@ -6,16 +6,10 @@ cd src set -o errexit set -o verbose -is_san_variant_arg="" -if [[ -n "${san_options}" ]]; then - is_san_variant_arg="--is-san-variant" -fi - activate_venv $python buildscripts/debugsymb_mapper.py \ --version "${version_id}" \ --client-id "${symbolizer_client_id}" \ --client-secret "${symbolizer_client_secret}" \ - --variant "${build_variant}" \ - $is_san_variant_arg + --variant "${build_variant}" diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh index 32785c0338b..db38741fbbd 100644 --- a/evergreen/generate_version.sh +++ b/evergreen/generate_version.sh @@ -6,12 +6,13 @@ cd src set -o errexit set -o verbose -setup_mongo_task_generator +curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.3.6/mongo-task-generator --output mongo-task-generator +chmod +x mongo-task-generator + activate_venv PATH=$PATH:$HOME:/ ./mongo-task-generator \ --expansion-file ../expansions.yml \ --evg-auth-file ./.evergreen.yml \ --evg-project-file ${evergreen_config_file_path} \ --generate-sub-tasks-config etc/generate_subtasks_config.yml \ - --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \ $@ diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh deleted file mode 100644 index b92b9fcfd16..00000000000 --- a/evergreen/generate_version_burn_in.sh +++ /dev/null @@ -1,19 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -setup_mongo_task_generator -activate_venv -$python buildscripts/burn_in_tests.py generate-test-membership-map-file-for-ci -PATH=$PATH:$HOME:/ ./mongo-task-generator \ - --expansion-file ../expansions.yml \ - --evg-auth-file ./.evergreen.yml \ - --evg-project-file ${evergreen_config_file_path} \ - --generate-sub-tasks-config etc/generate_subtasks_config.yml \ - --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \ - --burn-in \ - $@ diff --git a/evergreen/jepsen_docker/list-append.sh b/evergreen/jepsen_docker/list-append.sh index 66192de57cf..3eb8a8b67fa 100644 --- a/evergreen/jepsen_docker/list-append.sh +++ b/evergreen/jepsen_docker/list-append.sh @@ -10,22 +10,7 @@ cd jepsen/docker # actually run the tests start_time=$(date +%s) -sudo docker exec jepsen-control bash --login -c "\ -cd /jepsen/mongodb && \ -lein run test-all -w list-append \ --n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 \ --r 1000 \ ---concurrency 3n \ ---time-limit 120 \ ---max-writes-per-key 128 \ ---read-concern majority \ ---write-concern majority \ ---txn-read-concern snapshot \ ---txn-write-concern majority \ ---nemesis-interval 1 \ ---nemesis partition \ ---test-count 30" \ - | tee jepsen_${task_name}_${execution}.log +sudo docker exec jepsen-control bash --login -c "cd /jepsen/mongodb && lein run test-all -w list-append -n n1 -n n2 -n n3 -n n4 -n n5 -n n6 -n n7 -n n8 -n n9 -r 1000 --concurrency 3n --time-limit 240 --max-writes-per-key 128 --read-concern majority --write-concern majority --txn-read-concern snapshot --txn-write-concern majority --nemesis-interval 1 --nemesis partition --test-count 30" | tee jepsen_${task_name}_${execution}.log end_time=$(date +%s) elapsed_secs=$((end_time - start_time)) @@ -33,17 +18,8 @@ elapsed_secs=$((end_time - start_time)) cd ../../ mkdir -p src/jepsen-mongodb sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store -cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb/ +cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir -cd src -activate_venv -$python buildscripts/jepsen_report.py --start_time=$start_time --end_time=$end_time --elapsed=$elapsed_secs --emit_status_files --store ./jepsen-mongodb jepsen-mongodb/jepsen_${task_name}_${execution}.log -exit_code=$? - -if [ -f "jepsen_system_fail.txt" ]; then - mv jepsen_system_fail.txt jepsen-mongodb/jepsen_system_failure_${task_name}_${execution} - exit 0 -fi - -exit $exit_code +cd src/jepsen-mongodb +. ../evergreen/jepsen_report.sh diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh index 2512f03a6a1..cc76e20e541 100644 --- a/evergreen/jepsen_docker/setup.sh +++ b/evergreen/jepsen_docker/setup.sh @@ -1,15 +1,43 @@ set -euo pipefail -# Clone our internal fork of jepsen-io/jepsen to get the core -# functionality with a few tweaks meant for evergreen integration. -git clone --branch=v0.2.0-evergreen-master git@github.com:10gen/jepsen.git jepsen +if [ ! $(which docker) ]; then + sudo apt-get update + sudo apt-get install -yq \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg \ + lsb-release -# Copy our mongodb source for jepsen to run into the docker area to be -# copied into the image during the build process. -cp -rf src/dist-test jepsen/docker/node + if [ ! -f "/usr/share/keyrings/docker-archive-keyring.gpg" ]; then + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + fi + + set +e + if ! grep "https://download.docker.com/linux/debian" "/etc/apt/sources.list.d/docker.list"; then + echo \ + "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + fi + set -e + + sudo apt-get update + sudo apt-get install -yq docker-ce docker-ce-cli containerd.io + sudo docker run hello-world +fi -# Clone our internal tests to run -git clone --branch=v0.2.0 git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb +if [ ! $(which docker-compose) ]; then + sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose + sudo chmod +x /usr/bin/docker-compose +fi + +sudo chmod 777 /var/run/docker.sock + +git clone --branch=evergreen-master git@github.com:10gen/jepsen.git jepsen +cp -rf src/dist-test jepsen/docker/node +# place the mongodb jepsen test adjacent to the control node's Dockerfile. +# it'll get copied into the image during the build process +git clone --branch=no-download-master git@github.com:10gen/jepsen-io-mongodb.git jepsen/docker/control/mongodb -# Kill any running containers +# kill any running containers sudo docker container kill $(docker ps -q) || true diff --git a/evergreen/jepsen_test_run.sh b/evergreen/jepsen_test_run.sh index b5f79426079..2cf8b0ed387 100644 --- a/evergreen/jepsen_test_run.sh +++ b/evergreen/jepsen_test_run.sh @@ -16,11 +16,6 @@ mkdir -p $TMPDIR export _JAVA_OPTIONS=-Djava.io.tmpdir=$TMPDIR start_time=$(date +%s) -# XXX Do not run this test in a loop or with Jepsen's repeat flag -# the report generator (jepsen_report.sh) does not support parsing more than one -# test at a time. If you need to run these tests with a repeat flag, you -# will need to adopt buildscripts/jepsen_report.py to support the log output -# of older-style Jepsen tests lein run test --test ${jepsen_test_name} \ --mongodb-dir ../ \ --working-dir ${workdir}/src/jepsen-workdir \ diff --git a/evergreen/jstestfuzz_minimize.sh b/evergreen/jstestfuzz_minimize.sh index e1a3f54a1a4..13ddbaf5709 100644 --- a/evergreen/jstestfuzz_minimize.sh +++ b/evergreen/jstestfuzz_minimize.sh @@ -6,6 +6,8 @@ cd src/jstestfuzz set -o errexit set -o verbose +add_nodejs_to_path + if [ -f "../minimizer-outputs.json" ]; then - eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json" + eval npm run ${npm_command} -- -j "../minimizer-outputs.json" fi diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh index 9e6f94ee65b..d79ed69ac9b 100644 --- a/evergreen/jstestfuzz_run.sh +++ b/evergreen/jstestfuzz_run.sh @@ -1,10 +1,13 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -set -evo pipefail - cd src/jstestfuzz +set -o errexit +set -o verbose + +add_nodejs_to_path + in_patch_build_flag="" if [[ "${is_patch}" = "true" ]]; then case "${npm_command}" in @@ -14,4 +17,4 @@ if [[ "${is_patch}" = "true" ]]; then esac fi -./src/scripts/npm_run.sh ${npm_command} -- ${jstestfuzz_vars} ${in_patch_build_flag} --branch ${branch_name} +eval npm run "${npm_command}" -- "${jstestfuzz_vars}" "${in_patch_build_flag}" --branch "${branch_name}" diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index 83623ca6f16..f0d2b7a4c29 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -6,12 +6,11 @@ cd src set -o errexit set -o verbose -for i in {1..5}; do - git clone git@github.com:10gen/jstestfuzz.git && RET=0 && break || RET=$? && sleep 5 - echo "Failed to clone git@github.com:10gen/jstestfuzz.git, retrying..." -done +add_nodejs_to_path -if [ $RET -ne 0 ]; then - echo "Failed to clone git@github.com:10gen/jstestfuzz.git" - exit $RET -fi +git clone git@github.com:10gen/jstestfuzz.git + +pushd jstestfuzz +npm install +npm run prepare +popd diff --git a/evergreen/kitchen_run.sh b/evergreen/kitchen_run.sh new file mode 100644 index 00000000000..321e8196e7e --- /dev/null +++ b/evergreen/kitchen_run.sh @@ -0,0 +1,40 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src/buildscripts/package_test + +set -o errexit + +export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz" +export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" +export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" +export KITCHEN_SUBNET="${kitchen_subnet}" +export KITCHEN_VPC="${kitchen_vpc}" + +if [ ${packager_distro} == "suse12" ]; then + export KITCHEN_YAML="kitchen.legacy.yml" +fi + +if [[ "${packager_arch}" == "aarch64" || "${packager_arch}" == "arm64" ]]; then + kitchen_packager_distro="${packager_distro}-arm64" +else + kitchen_packager_distro="${packager_distro}-x86-64" +fi + +activate_venv +# set expiration tag 2 hours in the future, since no test should take this long +export KITCHEN_EXPIRE="$($python -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')" + +for i in {1..3}; do + if ! kitchen verify $kitchen_packager_distro; then + verified="false" + kitchen destroy $kitchen_packager_distro || true + sleep 30 + else + verified="true" + break + fi +done + +kitchen destroy $kitchen_packager_distro || true +test "$verified" = "true" diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh index 9c2159824e1..16ae53b923c 100644 --- a/evergreen/lint_fuzzer_sanity_all.sh +++ b/evergreen/lint_fuzzer_sanity_all.sh @@ -1,14 +1,12 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src/jstestfuzz +cd src set -eo pipefail set -o verbose -# Run first with help which will do the install -# Then we can run it in parallel -./src/scripts/npm_run.sh --help +add_nodejs_to_path + # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. -# Skip javascript files in third_party directory -find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- +find "$PWD/jstests" "$PWD/src/mongo/db/modules/enterprise" -name "*.js" -print | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- diff --git a/evergreen/lint_fuzzer_sanity_patch.sh b/evergreen/lint_fuzzer_sanity_patch.sh index af1afd63da7..11196aff1ea 100644 --- a/evergreen/lint_fuzzer_sanity_patch.sh +++ b/evergreen/lint_fuzzer_sanity_patch.sh @@ -6,16 +6,15 @@ cd src set -eo pipefail set -o verbose -mkdir -p jstestfuzzinput jstestfuzzoutput +add_nodejs_to_path -# We need to be the jstestfuzz repo for node to install/run -cd jstestfuzz +mkdir -p jstestfuzzinput jstestfuzzoutput -indir="$(pwd)/../jstestfuzzinput" -outdir="$(pwd)/../jstestfuzzoutput" +indir="$(pwd)/jstestfuzzinput" +outdir="$(pwd)/jstestfuzzoutput" # Grep all the js files from modified_and_created_patch_files.txt and put them into $indir. -(grep -v "\.tpl\.js$" ../modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true) +(grep -v "\.tpl\.js$" modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true) # Count the number of files in $indir. if [[ "$(ls -A $indir)" ]]; then @@ -26,8 +25,8 @@ if [[ "$(ls -A $indir)" ]]; then num_files=50 fi - ./src/scripts/npm_run.sh jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50 + npm run --prefix jstestfuzz jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50 # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. - ls -1 -d $outdir/* | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- + ls -1 -d $outdir/* | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- fi diff --git a/evergreen/macos_notary.py b/evergreen/macos_notary.py deleted file mode 100644 index ef799a8e5db..00000000000 --- a/evergreen/macos_notary.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import platform -import shutil -import urllib.request -import subprocess -import zipfile -import stat -import sys - -if platform.system().lower() != 'darwin': - print("Not a macos system, skipping macos signing.") - sys.exit(0) - -if len(sys.argv) < 2: - print("Must provide at least 1 archive to sign.") - sys.exit(1) - -supported_archs = { - 'arm64': 'arm64', - 'x86_64': 'amd64' -} -arch = platform.uname().machine.lower() - -if arch not in supported_archs: - print(f"Unsupported platform uname arch: {arch}, must be {supported_archs.keys()}") - sys.exit(1) - -macnotary_name = f'darwin_{supported_archs[arch]}' - -if os.environ['project'] == "mongodb-mongo-master-nightly": - signing_type = 'notarizeAndSign' -else: - signing_type = 'sign' - -macnotary_url = f'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/{macnotary_name}.zip' -print(f'Fetching macnotary tool from: {macnotary_url}') -local_filename, headers = urllib.request.urlretrieve(macnotary_url, f'{macnotary_name}.zip') -with zipfile.ZipFile(f'{macnotary_name}.zip') as zipf: - zipf.extractall() - -st = os.stat(f'{macnotary_name}/macnotary') -os.chmod(f'{macnotary_name}/macnotary', st.st_mode | stat.S_IEXEC) - -failed = False -archives = sys.argv[1:] - -for archive in archives: - archive_base, archive_ext = os.path.splitext(archive) - unsigned_archive = f'{archive_base}_unsigned{archive_ext}' - shutil.move(archive, unsigned_archive) - - signing_cmd = [ - f'./{macnotary_name}/macnotary', - '-f', f'{unsigned_archive}', - '-m', f'{signing_type}', - '-u', 'https://dev.macos-notary.build.10gen.cc/api', - '-k', 'server', - '--entitlements', 'etc/macos_entitlements.xml', - '--verify', - '-b', 'server.mongodb.com', - '-i', f'{os.environ["task_id"]}', - '-c', f'{os.environ["project"]}', - '-o', f'{archive}' - ] - - signing_env = os.environ.copy() - signing_env['MACOS_NOTARY_SECRET'] = os.environ["macos_notarization_secret"] - print(' '.join(signing_cmd)) - p = subprocess.Popen(signing_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=signing_env) - - print(f"Signing tool completed with exitcode: {p.returncode}") - for line in iter(p.stdout.readline, b''): - print(f'macnotary: {line.decode("utf-8").strip()}') - - # TODO: BUILD-14595 remove timeout when codesign doesn't frequently hang on macos hosts - timeout = 3600 - timed_out = False - try: - p.wait(timeout=timeout) - except subprocess.TimeoutExpired: - print(f"ERROR: failed to finish signing in timeout period of {timeout} seconds. This most likely is related to hung codesign, see issues underlying BUILD-14595.") - timed_out = True - pass - - if timed_out: - shutil.move(unsigned_archive, archive) - elif p.returncode != 0: - failed = True - shutil.move(unsigned_archive, archive) - else: - os.unlink(unsigned_archive) - -if failed: - exit(1) - diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh index f20257297ff..1e932a0c143 100644 --- a/evergreen/multiversion_setup.sh +++ b/evergreen/multiversion_setup.sh @@ -7,7 +7,8 @@ set -o errexit set -o verbose activate_venv -setup_db_contrib_tool + +setup_db_contrib_tool_venv export PIPX_HOME="${workdir}/pipx" export PIPX_BIN_DIR="${workdir}/pipx/bin" @@ -93,30 +94,5 @@ db-contrib-tool setup-repro-env \ --fallbackToMaster \ --resmokeCmd "python buildscripts/resmoke.py" \ --debug \ - $last_continuous_arg 4.4 5.0.28 - -# Certain build variants define additional multiversion_*_last_lts expansions in order to -# be able to fetch a complete set of versions. - -if [ ! -z "${multiversion_edition_last_lts}" ]; then - edition="${multiversion_edition_last_lts}" -fi - -if [ ! -z "${multiversion_platform_last_lts}" ]; then - platform="${multiversion_platform_last_lts}" -fi - -if [ ! -z "${multiversion_architecture_last_lts}" ]; then - architecture="${multiversion_architecture_last_lts}" -fi - -db-contrib-tool setup-repro-env \ - --installDir /data/install \ - --linkDir /data/multiversion \ - --edition $edition \ - --platform $platform \ - --architecture $architecture \ - --fallbackToMaster \ - --resmokeCmd "python buildscripts/resmoke.py" \ - --debug \ - $last_lts_arg + $last_lts_arg \ + $last_continuous_arg 4.4 5.0 diff --git a/evergreen/ninja_compile.sh b/evergreen/ninja_compile.sh index 823944277be..f3b51ae5e1d 100644 --- a/evergreen/ninja_compile.sh +++ b/evergreen/ninja_compile.sh @@ -5,7 +5,6 @@ cd src set -o errexit set -o verbose - activate_venv python -m pip install ninja if [ "Windows_NT" = "$OS" ]; then @@ -14,8 +13,8 @@ if [ "Windows_NT" = "$OS" ]; then for i in "${compile_env[@]}"; do echo "set $i" >> msvc.bat done - echo "ninja -f ${ninja_file} install-core" >> msvc.bat + echo "ninja install-core" >> msvc.bat cmd /C msvc.bat else - eval ${compile_env} ninja -f ${ninja_file} install-core + eval ${compile_env} ninja install-core fi diff --git a/evergreen/notary_client_crypt_run.sh b/evergreen/notary_client_crypt_run.sh new file mode 100644 index 00000000000..2d9b6b3d689 --- /dev/null +++ b/evergreen/notary_client_crypt_run.sh @@ -0,0 +1,20 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +. ./notary_env.sh + +set -o errexit +set -o verbose + +ext="${ext:-tgz}" + +mv "mongo_crypt_shared_v1.$ext" mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} + +/usr/local/bin/notary-client.py \ + --key-name "server-6.0" \ + --auth-token-file ${workdir}/src/signing_auth_token \ + --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" \ + --notary-url http://notary-service.build.10gen.cc:5000 \ + mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} diff --git a/evergreen/notary_client_run.sh b/evergreen/notary_client_run.sh new file mode 100644 index 00000000000..82526660d96 --- /dev/null +++ b/evergreen/notary_client_run.sh @@ -0,0 +1,23 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +. ./notary_env.sh + +set -o errexit +set -o verbose + +long_ext=${ext} +if [ "$long_ext" == "tgz" ]; then + long_ext="tar.gz" +fi + +mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext} +mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true +mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true +mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true +mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true +/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-${push_name}-${push_arch}-${suffix}.msi || true + +/usr/local/bin/notary-client.py --key-name "server-6.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} mongodb-${push_name}-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${long_ext} mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh index eb671d4b5ae..3844afaead5 100755 --- a/evergreen/prelude.sh +++ b/evergreen/prelude.sh @@ -11,7 +11,6 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$evergreen_dir/prelude_python.sh" . "$evergreen_dir/prelude_venv.sh" . "$evergreen_dir/prelude_db_contrib_tool.sh" -. "$evergreen_dir/prelude_mongo_task_generator.sh" expansions_yaml="$evergreen_dir/../../expansions.yml" expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" @@ -32,6 +31,21 @@ unset expansions_default_yaml unset script unset evergreen_dir +function add_nodejs_to_path { + # Add node and npm binaries to PATH + if [ "Windows_NT" = "$OS" ]; then + # An "npm" directory might not have been created in %APPDATA% by the Windows installer. + # Work around the issue by specifying a different %APPDATA% path. + # See: https://github.com/nodejs/node-v0.x-archive/issues/8141 + export APPDATA=${workdir}/npm-app-data + export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location + # TODO: this is to work around BUILD-8652 + cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')" + else + export PATH="$PATH:/opt/node/bin" + fi +} + function posix_workdir { if [ "Windows_NT" = "$OS" ]; then echo $(cygpath -u "${workdir}") diff --git a/evergreen/prelude_db_contrib_tool.sh b/evergreen/prelude_db_contrib_tool.sh index 44f80987c42..59c03d7684a 100644 --- a/evergreen/prelude_db_contrib_tool.sh +++ b/evergreen/prelude_db_contrib_tool.sh @@ -1,12 +1,11 @@ -function setup_db_contrib_tool { +function setup_db_contrib_tool_venv { - mkdir -p ${workdir}/pipx + mkdir ${workdir}/pipx export PIPX_HOME="${workdir}/pipx" export PIPX_BIN_DIR="${workdir}/pipx/bin" export PATH="$PATH:$PIPX_BIN_DIR" - export PIP_CACHE_DIR=${workdir}/pip_cache python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1 python -m pip --disable-pip-version-check install "pipx" || exit 1 - pipx install "db-contrib-tool==0.8.3" --pip-args="--no-cache-dir" || exit 1 + pipx install "db-contrib-tool==0.4.5" || exit 1 } diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh deleted file mode 100644 index 81e0e6ccce3..00000000000 --- a/evergreen/prelude_mongo_task_generator.sh +++ /dev/null @@ -1,24 +0,0 @@ -function setup_mongo_task_generator { - if [ ! -f mongo-task-generator ]; then - curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.10/mongo-task-generator --output mongo-task-generator - chmod +x mongo-task-generator - fi -} - -## Comment above and uncomment below to test unreleased mongo-task-generator changes that are -## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git` -## repo -#function setup_mongo_task_generator { -# if [ ! -f mongo-task-generator ]; then -# -# curl https://sh.rustup.rs -sSf | sh -s -- -y -# source "$HOME/.cargo/env" -# git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator -# pushd unreleased-mongo-task-generator -# git checkout <branch-name> -# cargo build --release --locked -# generator_path="$(pwd)/target/release/mongo-task-generator" -# popd -# cp "$generator_path" mongo-task-generator -# fi -#} diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh index a0851406f57..6b97a8a9cfb 100644 --- a/evergreen/prelude_venv.sh +++ b/evergreen/prelude_venv.sh @@ -22,26 +22,9 @@ function activate_venv { if [ "Windows_NT" = "$OS" ]; then export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" - elif [ "$(uname)" = "Darwin" ]; then - #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting - #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path. - python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src" else - python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') - site_packages="${workdir}/venv/lib/python${python_version}/site-packages" - python -c "import sys; print(sys.path)" - - # Check if site_packages is already in sys.path - in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)") - if [ "$in_sys_path" = "False" ]; then - export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src" - else - export PYTHONPATH="$PYTHONPATH:${workdir}/src" - fi - python -c "import sys; print(sys.path)" + export PYTHONPATH="$PYTHONPATH:${workdir}/src" fi - export PIP_CACHE_DIR=${workdir}/pip_cache - echo "python set to $(which $python) and python version: $($python --version)" + echo "python set to $(which $python)" } diff --git a/evergreen/record_mongodb_server_version.sh b/evergreen/record_mongodb_server_version.sh deleted file mode 100755 index e7ad5785657..00000000000 --- a/evergreen/record_mongodb_server_version.sh +++ /dev/null @@ -1,9 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -set -o errexit -set -o verbose - -"$1" --version > "$2" diff --git a/evergreen/resmoke_tests_execute.sh b/evergreen/resmoke_tests_execute.sh index 911a85414fd..ce1ecc7b582 100644 --- a/evergreen/resmoke_tests_execute.sh +++ b/evergreen/resmoke_tests_execute.sh @@ -14,19 +14,15 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then # activate the virtualenv if it has been set up activate_venv - # Install db-contrib-tool to symbolize crashes during resmoke suite runs - # This is not supported on Windows and MacOS, so doing it only on Linux - if [ "$(uname)" == "Linux" ]; then - setup_db_contrib_tool - fi if [[ -f "patch_test_tags.tgz" ]]; then tags_build_variant="${build_variant}" if [[ "${build_variant}" =~ .*"-query-patch-only" ]]; then - # Use the RHEL 8 all feature flags variant for the SBE variant. The original SBE variant is - # not a required builder and therefore not captured in patch test failure history. - tags_build_variant="enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required" + # Use the RHEL 8 all feature flags variant for the classic engine variant. The original + # classic engine variant is not a required builder and therefore not captured in patch + # test failure history. + tags_build_variant="enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required" fi $python buildscripts/testmatrix/getdisplaytaskname.py "${task_name}" "${build_variant}" > display_task_name.txt @@ -104,18 +100,10 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then set -o errexit # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant. - if [[ "${build_variant}" = "enterprise-rhel-8-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then + if [[ "${build_variant}" = "enterprise-rhel-80-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\"" fi - # Even though all feature flags may be enabled on a variant, often times we do not want to run - # feature flag tests because they will most likely fail. For example, during multiversion testing, - # all feature flags may be enabled on the latest version, but running feature flag specific tests on - # older versions (last-lts/last-continuous) will likely fail because those features most likely do not exist. - if [[ ${run_no_feature_flag_tests} == "true" ]]; then - extra_args="$extra_args --runNoFeatureFlagTests" - fi - path_value="$PATH:/data/multiversion" # Set the suite name to be the task name by default; unless overridden with the `suite` expansion. @@ -142,7 +130,7 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then $extra_args \ ${test_flags} \ --suites=${suite_name} \ - --log=${resmoke_logger} \ + --log=buildlogger \ --staggerJobs=on \ --installDir=${install_dir} \ --buildId=${build_id} \ @@ -155,7 +143,6 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then --taskName=${task_name} \ --variantName=${build_variant} \ --versionId=${version_id} \ - --taskWorkDir='${workdir}' \ --reportFile=report.json \ --perfReportFile=perf.json \ --cedarReportFile=cedar_report.json diff --git a/evergreen/resmoke_tests_runtime_validate.sh b/evergreen/resmoke_tests_runtime_validate.sh index aee2111ff92..1ccfc71c225 100644 --- a/evergreen/resmoke_tests_runtime_validate.sh +++ b/evergreen/resmoke_tests_runtime_validate.sh @@ -9,6 +9,7 @@ set -o errexit activate_venv $python buildscripts/resmoke_tests_runtime_validate.py \ --resmoke-report-file ./report.json \ + --evg-api-config ./.evergreen.yml \ --project-id ${project_id} \ --build-variant ${build_variant} \ --task-name ${task_name} diff --git a/evergreen/run_jstestfuzz/clone_repos.sh b/evergreen/run_jstestfuzz/clone_repos.sh index 74605e058ef..b64e91fd4d3 100755 --- a/evergreen/run_jstestfuzz/clone_repos.sh +++ b/evergreen/run_jstestfuzz/clone_repos.sh @@ -2,14 +2,5 @@ set -o errexit set -o verbose cd src -for i in {1..5}; do - git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests && RET=0 && break || RET=$? && sleep 1 - echo "Failed to clone git@github.com:10gen/QA.git, retrying..." -done - -if [ $RET -ne 0 ]; then - echo "Failed to clone git@github.com:10gen/QA.git" - exit $RET -fi - -cp -r src/mongo/db/modules/enterprise jstests/enterprise_tests +git clone --depth 1 git@github.com:10gen/mongo-enterprise-modules.git jstests/enterprise_tests +git clone --depth 1 git@github.com:10gen/QA.git jstests/qa_tests diff --git a/evergreen/run_python_script.sh b/evergreen/run_python_script.sh deleted file mode 100644 index 35181dec3c1..00000000000 --- a/evergreen/run_python_script.sh +++ /dev/null @@ -1,11 +0,0 @@ -unset workdir -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -set -o errexit -set -o verbose - -cd src - -activate_venv -$python $@ diff --git a/evergreen/sasl_windows_cyrussasl_setup.sh b/evergreen/sasl_windows_cyrussasl_setup.sh deleted file mode 100644 index 149bb563325..00000000000 --- a/evergreen/sasl_windows_cyrussasl_setup.sh +++ /dev/null @@ -1,18 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -readonly k_cyrussasl_plugin_filename="cyrus_sasl_windows_test_plugin.dll" -readonly k_cyrussasl_plugin_dir="/cygdrive/c/CMU/bin/sasl2" - -plugin_path="$(find . -name "*${k_cyrussasl_plugin_filename}")" - -if [[ -z "$plugin_path" ]]; then - echo >&2 "Could not find ${k_cyrussasl_plugin_filename} from path '$(pwd)' !" - exit 1 -fi - -echo "Configuring CyrusSASL plugin .dll from '$plugin_path'" - -mkdir -p "$k_cyrussasl_plugin_dir" - -cp "$plugin_path" "$k_cyrussasl_plugin_dir" diff --git a/evergreen/sasl_windows_cyrussasl_teardown.sh b/evergreen/sasl_windows_cyrussasl_teardown.sh deleted file mode 100644 index 60e65055643..00000000000 --- a/evergreen/sasl_windows_cyrussasl_teardown.sh +++ /dev/null @@ -1,18 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -if [ "${task_name}" != "sasl_windows_cyrussasl" ]; then - exit 0 -fi - -echo "Cleaning up Windows CyrusSASL Test Artifacts" - -readonly k_cyrussasl_default_dir_root="/cygdrive/c/CMU" - -if [[ ! -d "$k_cyrussasl_default_dir_root" ]]; then - echo "Could not find $k_cyrussasl_default_dir_root to cleanup..." - exit 0 -fi - -rm -rf "$k_cyrussasl_default_dir_root" -echo "Deleted $k_cyrussasl_default_dir_root from host" diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh index a6e2be4539e..af3ff0bad6d 100755 --- a/evergreen/scons_compile.sh +++ b/evergreen/scons_compile.sh @@ -8,6 +8,15 @@ set -o verbose rm -rf ${install_directory} +# Use hardlinks to reduce the disk space impact of installing +# all of the binaries and associated debug info. + +# The expansion here is a workaround to let us set a different install-action +# for tasks that don't support the one we set here. A better plan would be +# to support install-action for Ninja builds directly. +# TODO: https://jira.mongodb.org/browse/SERVER-48203 +extra_args="--install-action=${task_install_action}" + # By default, limit link jobs to one quarter of our overall -j # concurrency unless locally overridden. We do this because in # static link environments, the memory consumption of each @@ -49,8 +58,6 @@ else extra_args="$extra_args --release" fi -extra_args="$extra_args ENABLE_OOM_RETRY=1" - if [ "${generating_for_ninja}" = "true" ] && [ "Windows_NT" = "$OS" ]; then vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH diff --git a/evergreen/selinux_run_test.sh b/evergreen/selinux_run_test.sh index 15fe8be87b6..318d73adbe2 100755 --- a/evergreen/selinux_run_test.sh +++ b/evergreen/selinux_run_test.sh @@ -22,6 +22,8 @@ if [ "$bypass_prelude" != "yes" ]; then src="src" fi +set -o xtrace + if [ "$hostname" == "" ]; then hostname="$(tr -d '"[]{}' < "$workdir"/$src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" fi @@ -50,12 +52,10 @@ connection_attempts=50 # Check for remote connectivity set +o errexit ssh_options="-i $ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" -while ! ssh -q $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do - if [ "$attempts" -ge "$connection_attempts" ]; then - printf "SSH connection attempt failed after %d attempts.\n" "$attempts" - exit 1 - fi +while ! ssh $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do + if [ "$attempts" -ge "$connection_attempts" ]; then exit 1; fi ((attempts++)) + printf "SSH connection attempt %d/%d failed. Retrying...\n" "$attempts" "$connection_attempts" sleep 10 done diff --git a/evergreen/selinux_test_executor.sh b/evergreen/selinux_test_executor.sh index fc1e04cf2be..a0616d314d6 100755 --- a/evergreen/selinux_test_executor.sh +++ b/evergreen/selinux_test_executor.sh @@ -1,177 +1,109 @@ #!/bin/bash -set +o errexit -readonly k_log_path="/var/log/mongodb/mongod.log" -readonly k_mongo="$(pwd)/dist-test/bin/mongo" -readonly k_test_path="$1" -return_code=1 +set -o errexit +set -o xtrace -export PATH="$(dirname "$k_mongo"):$PATH" +mongo="$(pwd)/dist-test/bin/mongo" +export PATH="$(dirname "$mongo"):$PATH" +if [ ! -f "$mongo" ]; then + echo "Mongo shell at $mongo is missing" + exit 1 +fi -function print_err() { +function print() { echo "$@" >&2 } function monitor_log() { - sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f $k_log_path) -} - -function output_ausearch() { - local cmd_parameters="AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR" - - echo "" - echo "====== SELinux errors (ausearch -m $cmd_parameters): ======" - sudo --non-interactive ausearch -m $cmd_parameters -ts $1 -} - -function output_journalctl() { - echo "" - echo "============================== journalctl =========================================" - sudo --non-interactive journalctl --no-pager --catalog --since="$1" | grep -i mongo + sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f /var/log/mongodb/mongod.log) } -function fail_and_exit_err() { - - echo "" - echo "===================================================================================" - echo "++++++++ Test failed, outputting last 5 seconds of additional log info ++++++++++++" - echo "===================================================================================" - output_ausearch "$(date --utc --date='5 seconds ago' '+%x %H:%M:%S')" - output_journalctl "$(date --utc --date='5 seconds ago' +'%Y-%m-%d %H:%M:%S')" - - echo "" - echo "==== FAIL: $1 ====" +TEST_PATH="$1" +if [ ! -f "$TEST_PATH" ]; then + print "No test supplied or test file not found. Run:" + print " $(basename "${BASH_SOURCE[0]}") <path>" exit 1 -} - -function create_mongo_config() { - echo "Writing /etc/mongod.conf for $k_test_path:" - "$k_mongo" --nodb --norc --quiet --eval=' - assert(load("'"$k_test_path"'")); - const test = new TestDefinition(); - print(JSON.stringify(test.config, null, 2)); - - ' | sudo --non-interactive tee /etc/mongod.conf -} - -function start_mongod() { - # Start mongod and if it won't come up, fail and exit - - sudo --non-interactive systemctl start mongod \ - && sudo --non-interactive systemctl status mongod || ( - fail_and_exit_err "systemd failed to start mongod server!" - ) -} - -function wait_for_mongod_to_accept_connections() { - # Once the mongod process starts via systemd, it can still take a couple of seconds - # to set up and accept connections... we will wait for log id 23016 to show up - # indicating that the server is ready to accept incoming connections before starting the tests - - local server_ready=0 - local wait_seconds=2 - local wait_retries_max=30 - local wait_retries=0 - - while [[ $wait_retries -le $wait_retries_max ]]; do - local server_status="$(grep 23016 $k_log_path || echo "")" - - if [ "$server_status" != "" ]; then - server_ready=1 - break - fi - - sleep $wait_seconds - ((wait_retries++)) - done - - if [ ! $server_ready ]; then - fail_and_exit_err "failed to connect to mongod server after waiting for $(($wait_seconds * $wait_retries)) seconds!" - fi -} - -function clear_mongo_config() { - # stop mongod, zero mongo log, clean up database, set all booleans to off - sudo --non-interactive bash -c ' - systemctl stop mongod - - rm -f '"$k_log_path"' - touch '"$k_log_path"' - chown mongod '"$k_log_path"' - - rm -rf /var/lib/mongo/* - - rm -rf /etc/sysconfig/mongod /etc/mongod - - setsebool mongod_can_connect_ldap off - setsebool mongod_can_use_kerberos off - ' -} - -function exit_with_code() { - exit $return_code -} - -function setup_test_definition() { - "$k_mongo" --nodb --norc --quiet --eval=' - assert(load("'"$k_test_path"'")); - (() => { - const test = new TestDefinition(); - print("Running setup() for '"$k_test_path"'"); - test.setup(); - })(); - ' -} - -function run_test() { - "$k_mongo" --norc --gssapiServiceName=mockservice --eval=' - assert(load("'"$k_test_path"'")); - print("Running test '"$k_test_path"'"); +fi - (() => { - const test = new TestDefinition(); +# test file is even good before going on +if ! "$mongo" --nodb --norc --quiet "$TEST_PATH"; then + print "File $TEST_PATH has syntax errors" + exit 1 +fi - try { - test.run(); - } finally { - test.teardown(); - } - })(); - ' || fail_and_exit_err "Test failed" +# stop mongod, zero mongo log, clean up database, set all booleans to off +sudo --non-interactive bash -c ' + systemctl stop mongod - echo "SUCCESS: $k_test_path" -} + rm -f /var/log/mongodb/mongod.log + touch /var/log/mongodb/mongod.log + chown mongod /var/log/mongodb/mongod.log -if [ ! -f "$k_mongo" ]; then - print_err "Mongo shell at $k_mongo is missing" - exit 1 -fi + rm -rf /var/lib/mongo/* -if [ ! -f "$k_test_path" ]; then - print_err "No test supplied or test file not found. Run:" - print_err "$(basename "${BASH_SOURCE[0]}") <path>" - exit 1 -fi + rm -rf /etc/sysconfig/mongod /etc/mongod -# Ensure file containing tests is valid before executing -if ! "$k_mongo" --nodb --norc --quiet "$k_test_path"; then - print_err "File $k_test_path has syntax errors" - exit 1 -fi + setsebool mongod_can_connect_snmp off + setsebool mongod_can_connect_ldap off + setsebool mongod_can_use_kerberos off +' -echo "STARTING TEST: $k_test_path" +# create mongo config +"$mongo" --nodb --norc --quiet --eval=' + assert(load("'"$TEST_PATH"'")); + const test = new TestDefinition(); + print(typeof(test.config) === "string" ? test.config : JSON.stringify(test.config, null, 2)); +' | sudo --non-interactive tee /etc/mongod.conf -clear_mongo_config -create_mongo_config -setup_test_definition +# setup +"$mongo" --nodb --norc --quiet --eval=' + assert(load("'"$TEST_PATH"'")); + const test = new TestDefinition(); + jsTest.log("Running setup()"); + test.setup(); +' # start log monitor, also kill it on exit monitor_log & -monitor_pid="$!" -trap "sudo --non-interactive pkill -P $monitor_pid; exit_with_code" SIGINT SIGTERM ERR EXIT - -start_mongod -wait_for_mongod_to_accept_connections -run_test +MONITORPID="$!" +trap "sudo --non-interactive pkill -P $MONITORPID" SIGINT SIGTERM ERR EXIT + +# start mongod and if it won't come up, log SELinux errors +ts="$(date --utc --date='1 seconds ago' '+%x %H:%M:%S')" +tsj="$(date --utc --date='1 seconds ago' +'%Y-%m-%d %H:%M:%S')" +sudo --non-interactive systemctl start mongod \ + && sudo --non-interactive systemctl status mongod || ( + set +o errexit + echo "=== SELinux errors:" + sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts + echo "=== journalctl --unit=mongod:" + sudo --non-interactive journalctl --no-pager --since="$tsj" --unit=mongod --unit=systemd --catalog + echo "=== /var/log/mongodb/mongod.log:" + sudo --non-interactive cat /var/log/mongodb/mongod.log + echo "==== FAIL: mongod service was not started successfully" + exit 1 +) + +# run test and teardown +"$mongo" --norc --gssapiServiceName=mockservice --eval=' + assert(load("'"$TEST_PATH"'")); + // name is such to prevent collisions + const test_812de7ce = new TestDefinition(); + try { + jsTest.log("Running test"); + test_812de7ce.run(); + } finally { + test_812de7ce.teardown(); + } +' || ( + echo "==== FAIL: test returned result: $?" + echo "=== SELinux errors:" + set +o errexit + sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts + echo "=== /var/log/mongodb/mongod.log:" + sudo --non-interactive cat /var/log/mongodb/mongod.log + exit 1 +) -return_code=0 +set +o xtrace +echo "SUCCESS: $TEST_PATH" |
