diff options
Diffstat (limited to 'evergreen')
41 files changed, 576 insertions, 314 deletions
diff --git a/evergreen/blackduck_hub.sh b/evergreen/blackduck_hub.sh index f7edf2d486e..ce3d81fc79c 100755 --- a/evergreen/blackduck_hub.sh +++ b/evergreen/blackduck_hub.sh @@ -14,4 +14,14 @@ if [ "$branch_name" != "master" ]; then additional_args="--vulnerabilities_only" fi +# BlackDuck crashes on this gzip file because it is not well-formed +# invalid compressed data--format violated +rm ./src/third_party/zstandard/zstd/tests/gzip/hufts-segv.gz + +# Remove package.json since it only exists for vscode +# MongoDB server does not use Node.JS code so we strip this file to not confuse BlackDuck Detect +# Otherwise we need to run npm install to install everything in package.json or disable the NPM +# scanner. +rm package.json + python buildscripts/blackduck_hub.py -v scan_and_report --build_logger=mci.buildlogger --build_logger_task_id=${task_id} --report_file=report.json $additional_args diff --git a/evergreen/blackduck_setup.sh b/evergreen/blackduck_setup.sh index 8ed396833a3..d2fc4cfaa87 100755 --- a/evergreen/blackduck_setup.sh +++ b/evergreen/blackduck_setup.sh @@ -9,6 +9,7 @@ cat > .restconfig.json << END_OF_CREDS "baseurl": "${blackduck_url}", "username": "${blackduck_username}", "password": "${blackduck_password}", +"token": "${blackduck_token}", "debug": false, "insecure" : false } diff --git a/evergreen/compiled_binaries_get.sh b/evergreen/compiled_binaries_get.sh index b2d726452a9..67ed74a8db9 100755 --- a/evergreen/compiled_binaries_get.sh +++ b/evergreen/compiled_binaries_get.sh @@ -6,8 +6,9 @@ cd src set -o errexit set -o verbose -setup_db_contrib_tool_venv -activate_db_contrib_tool_venv +# activate_venv will make sure we are using python 3 +activate_venv +setup_db_contrib_tool rm -rf /data/install /data/multiversion diff --git a/evergreen/external_auth_aws_setup.sh b/evergreen/external_auth_aws_setup.sh index c59b025028c..233da94c338 100755 --- a/evergreen/external_auth_aws_setup.sh +++ b/evergreen/external_auth_aws_setup.sh @@ -17,10 +17,6 @@ cat << EOF > aws_e2e_setup.json "iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}", "iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}", - "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}", - - "iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}", - "iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}", - "iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}" + "iam_auth_assume_role_name" : "${iam_auth_assume_role_name}" } EOF diff --git a/evergreen/feature_flag_tags_check.sh b/evergreen/feature_flag_tags_check.sh index 5b046843a76..825591d5b41 100755 --- a/evergreen/feature_flag_tags_check.sh +++ b/evergreen/feature_flag_tags_check.sh @@ -12,7 +12,7 @@ enterprise_path="src/mongo/db/modules/enterprise" diff_file_name="with_base_upstream.diff" # get the list of feature flags from the patched version -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src +$python buildscripts/idl/gen_all_feature_flag_list.py mv all_feature_flags.txt patch_all_feature_flags.txt # get the list of feature flags from the base commit @@ -29,7 +29,7 @@ if [ -s "$diff_file_name" ]; then fi popd -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir "$enterprise_path"/src +$python buildscripts/idl/gen_all_feature_flag_list.py mv all_feature_flags.txt base_all_feature_flags.txt # print out the list of tests that previously had feature flag tag, that was diff --git a/evergreen/functions/binary_version_check.sh b/evergreen/functions/binary_version_check.sh deleted file mode 100755 index ac01374a07e..00000000000 --- a/evergreen/functions/binary_version_check.sh +++ /dev/null @@ -1,16 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/../prelude.sh" - -cd src - -set -o errexit -mongo_binary=dist-test/bin/mongo${exe} -activate_venv -bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]') -# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();" -mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]') -# The versions must match -if [ "$bin_ver" != "$mongo_ver" ]; then - echo "The mongo version is $mongo_ver, expected version is $bin_ver" - exit 1 -fi diff --git a/evergreen/functions/compile_expansions_generate.sh b/evergreen/functions/compile_expansions_generate.sh index dc9642f5cd4..fe06d0fdf7d 100644 --- a/evergreen/functions/compile_expansions_generate.sh +++ b/evergreen/functions/compile_expansions_generate.sh @@ -5,14 +5,6 @@ cd src set -o errexit set -o verbose -# We get the raw version string (r1.2.3-45-gabcdef) from git -MONGO_VERSION=$(git describe --abbrev=7) -# If this is a patch build, we add the patch version id to the version string so we know -# this build was a patch, and which evergreen task it came from -if [ "${is_patch}" = "true" ]; then - MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" -fi -echo "MONGO_VERSION = ${MONGO_VERSION}" activate_venv # shared scons cache testing # if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true @@ -43,12 +35,12 @@ if [ ! -z ${scons_cache_scope} ]; then set -o errexit fi echo "Shared Cache with setting: ${scons_cache_scope}" - MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml + SCONS_CACHE_MODE=${scons_cache_mode} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml # Legacy Expansion generation else echo "Using legacy expansion generation" # Proceed with regular expansions generated # This script converts the generated version string into a sanitized version string for # use by scons and uploading artifacts as well as information about for the scons cache. - MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml + SCONS_CACHE_MODE=${scons_cache_mode} USE_SCONS_CACHE=${use_scons_cache} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_compile_expansions.py --out compile_expansions.yml fi diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh index 79d393bb964..49dda4dd4a9 100644 --- a/evergreen/functions/task_timeout_determine.sh +++ b/evergreen/functions/task_timeout_determine.sh @@ -25,13 +25,20 @@ if [[ -n "${burn_in_bypass}" ]]; then build_variant_for_timeout=${burn_in_bypass} fi +if [[ -n "${alias}" ]]; then + evg_alias=${alias} +else + evg_alias="evg-alias-absent" +fi + activate_venv PATH=$PATH:$HOME:/ $python buildscripts/evergreen_task_timeout.py $timeout_factor \ --install-dir "${install_dir}" \ --task-name ${task_name} \ --suite-name ${suite_name} \ + --project ${project} \ --build-variant $build_variant_for_timeout \ - --evg-alias '${alias}' \ + --evg-alias $evg_alias \ --timeout ${timeout_secs} \ --exec-timeout ${exec_timeout_secs} \ --evg-api-config ./.evergreen.yml \ diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh index 3d935d87594..9b53989aa0e 100644 --- a/evergreen/functions/venv_setup.sh +++ b/evergreen/functions/venv_setup.sh @@ -74,10 +74,9 @@ activate_venv echo "Upgrading pip to 21.0.1" python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1 -python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log -if [ $? != 0 ]; then +if ! python -m pip --disable-pip-version-check install -r "$toolchain_txt" -q --log install.log; then echo "Pip install error" - cat install.log + cat install.log || true exit 1 fi python -m pip freeze > pip-requirements.txt diff --git a/evergreen/functions/version_expansions_generate.sh b/evergreen/functions/version_expansions_generate.sh new file mode 100755 index 00000000000..c0e577a59e6 --- /dev/null +++ b/evergreen/functions/version_expansions_generate.sh @@ -0,0 +1,18 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/../prelude.sh" + +cd src + +set -o errexit +set -o verbose +# We get the raw version string (r1.2.3-45-gabcdef) from git +MONGO_VERSION=$(git describe --abbrev=7) +# If this is a patch build, we add the patch version id to the version string so we know +# this build was a patch, and which evergreen task it came from +if [ "${is_patch}" = "true" ]; then + MONGO_VERSION="$MONGO_VERSION-patch-${version_id}" +fi +echo "MONGO_VERSION = ${MONGO_VERSION}" + +activate_venv +MONGO_VERSION=${MONGO_VERSION} IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue} $python buildscripts/generate_version_expansions.py --out version_expansions.yml diff --git a/evergreen/garasign_gpg_crypt_sign.sh b/evergreen/garasign_gpg_crypt_sign.sh new file mode 100644 index 00000000000..233a598ce56 --- /dev/null +++ b/evergreen/garasign_gpg_crypt_sign.sh @@ -0,0 +1,31 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +ext="${ext:-tgz}" + +crypt_file_name=mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} +mv "mongo_crypt_shared_v1.$ext" $crypt_file_name + +# generating checksums +shasum -a 1 $crypt_file_name | tee $crypt_file_name.sha1 +shasum -a 256 $crypt_file_name | tee $crypt_file_name.sha256 +md5sum $crypt_file_name | tee $crypt_file_name.md5 + +# signing crypt linux artifact with gpg +cat << EOF >> gpg_signing_commands.sh +gpgloader # loading gpg keys. +gpg --yes -v --armor -o $crypt_file_name.sig --detach-sign $crypt_file_name +EOF + +podman run \ + -e GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60} \ + -e GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60} \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_gpg_image} \ + /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_gpg_sign.sh b/evergreen/garasign_gpg_sign.sh new file mode 100644 index 00000000000..e060130afe4 --- /dev/null +++ b/evergreen/garasign_gpg_sign.sh @@ -0,0 +1,63 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +set -o errexit +set -o verbose + +cd src + +long_ext=${ext} +if [ "$long_ext" == "tgz" ]; then + long_ext="tar.gz" +fi + +mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext} +mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true +mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true +mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true +mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true + +# generating checksums +function gen_checksums() { + if [ -e $1 ]; then + shasum -a 1 $1 | tee $1.sha1 + shasum -a 256 $1 | tee $1.sha256 + md5sum $1 | tee $1.md5 + else + echo "$1 does not exist. Skipping checksum generation" + fi +} + +gen_checksums mongodb-$push_name-$push_arch-$suffix.$ext +gen_checksums mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext +gen_checksums mongodb-src-$src_suffix.$long_ext +gen_checksums mongodb-cryptd-$push_name-$push_arch-$suffix.$ext + +# signing linux artifacts with gpg +cat << 'EOF' > gpg_signing_commands.sh +gpgloader # loading gpg keys. +function sign(){ + if [ -e $1 ] + then + gpg --yes -v --armor -o $1.sig --detach-sign $1 + else + echo "$1 does not exist. Skipping signing" + fi +} + +EOF + +cat << EOF >> gpg_signing_commands.sh +sign mongodb-$push_name-$push_arch-$suffix.$ext +sign mongodb-$push_name-$push_arch-debugsymbols-$suffix.$ext +sign mongodb-src-$src_suffix.$long_ext +sign mongodb-cryptd-$push_name-$push_arch-$suffix.$ext +EOF + +podman run \ + -e GRS_CONFIG_USER1_USERNAME=${garasign_gpg_username_60} \ + -e GRS_CONFIG_USER1_PASSWORD=${garasign_gpg_password_60} \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_gpg_image} \ + /bin/bash -c "$(cat ./gpg_signing_commands.sh)" diff --git a/evergreen/garasign_jsign_sign.sh b/evergreen/garasign_jsign_sign.sh new file mode 100644 index 00000000000..474a5176318 --- /dev/null +++ b/evergreen/garasign_jsign_sign.sh @@ -0,0 +1,42 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +set -o errexit +set -o verbose + +cd src + +msi_filename=mongodb-${push_name}-${push_arch}-${suffix}.msi +/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename $msi_filename || true + +# signing windows artifacts with jsign +cat << 'EOF' > jsign_signing_commands.sh +function sign(){ + if [ -e $1 ] + then + jsign -a mongo-authenticode-2021 --replace --tsaurl http://timestamp.digicert.com -d SHA-256 $1 + else + echo "$1 does not exist. Skipping signing" + fi +} +EOF +cat << EOF >> jsign_signing_commands.sh +sign $msi_filename +EOF + +podman run \ + -e GRS_CONFIG_USER1_USERNAME=${garasign_jsign_username} \ + -e GRS_CONFIG_USER1_PASSWORD=${garasign_jsign_password} \ + --rm \ + -v $(pwd):$(pwd) -w $(pwd) \ + ${garasign_jsign_image} \ + /bin/bash -c "$(cat ./jsign_signing_commands.sh)" + +# generating checksums +if [ -e $msi_filename ]; then + shasum -a 1 $msi_filename | tee $msi_filename.sha1 + shasum -a 256 $msi_filename | tee $msi_filename.sha256 + md5sum $msi_filename | tee $msi_filename.md5 +else + echo "$msi_filename does not exist. Skipping checksum generation" +fi diff --git a/evergreen/gen_feature_flags.sh b/evergreen/gen_feature_flags.sh index 4e6dc0e9b4c..b06f614c243 100755 --- a/evergreen/gen_feature_flags.sh +++ b/evergreen/gen_feature_flags.sh @@ -6,4 +6,4 @@ cd src set -o errexit set -o verbose activate_venv -$python buildscripts/idl/gen_all_feature_flag_list.py --import-dir src --import-dir src/mongo/db/modules/enterprise/src +$python buildscripts/idl/gen_all_feature_flag_list.py diff --git a/evergreen/generate_buildid_debug_symbols_mapping.sh b/evergreen/generate_buildid_debug_symbols_mapping.sh index d1866e961a5..142614ce821 100644 --- a/evergreen/generate_buildid_debug_symbols_mapping.sh +++ b/evergreen/generate_buildid_debug_symbols_mapping.sh @@ -6,10 +6,16 @@ cd src set -o errexit set -o verbose +is_san_variant_arg="" +if [[ -n "${san_options}" ]]; then + is_san_variant_arg="--is-san-variant" +fi + activate_venv $python buildscripts/debugsymb_mapper.py \ --version "${version_id}" \ --client-id "${symbolizer_client_id}" \ --client-secret "${symbolizer_client_secret}" \ - --variant "${build_variant}" + --variant "${build_variant}" \ + $is_san_variant_arg diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh index db38741fbbd..32785c0338b 100644 --- a/evergreen/generate_version.sh +++ b/evergreen/generate_version.sh @@ -6,13 +6,12 @@ cd src set -o errexit set -o verbose -curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.3.6/mongo-task-generator --output mongo-task-generator -chmod +x mongo-task-generator - +setup_mongo_task_generator activate_venv PATH=$PATH:$HOME:/ ./mongo-task-generator \ --expansion-file ../expansions.yml \ --evg-auth-file ./.evergreen.yml \ --evg-project-file ${evergreen_config_file_path} \ --generate-sub-tasks-config etc/generate_subtasks_config.yml \ + --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \ $@ diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh new file mode 100644 index 00000000000..b92b9fcfd16 --- /dev/null +++ b/evergreen/generate_version_burn_in.sh @@ -0,0 +1,19 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +setup_mongo_task_generator +activate_venv +$python buildscripts/burn_in_tests.py generate-test-membership-map-file-for-ci +PATH=$PATH:$HOME:/ ./mongo-task-generator \ + --expansion-file ../expansions.yml \ + --evg-auth-file ./.evergreen.yml \ + --evg-project-file ${evergreen_config_file_path} \ + --generate-sub-tasks-config etc/generate_subtasks_config.yml \ + --s3-test-stats-endpoint https://mongo-test-stats.s3.amazonaws.com \ + --burn-in \ + $@ diff --git a/evergreen/jepsen_docker/list-append.sh b/evergreen/jepsen_docker/list-append.sh index 3eb8a8b67fa..2204a6a2e9b 100644 --- a/evergreen/jepsen_docker/list-append.sh +++ b/evergreen/jepsen_docker/list-append.sh @@ -18,8 +18,17 @@ elapsed_secs=$((end_time - start_time)) cd ../../ mkdir -p src/jepsen-mongodb sudo docker cp jepsen-control:/jepsen/mongodb/store src/jepsen-mongodb/store -cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb +cp jepsen/docker/jepsen_${task_name}_${execution}.log src/jepsen-mongodb/ sudo docker cp jepsen-control:/jepsen/mongodb src/jepsen-workdir -cd src/jepsen-mongodb -. ../evergreen/jepsen_report.sh +cd src +activate_venv +$python buildscripts/jepsen_report.py --start_time=$start_time --end_time=$end_time --elapsed=$elapsed_secs --emit_status_files --store ./jepsen-mongodb jepsen-mongodb/jepsen_${task_name}_${execution}.log +exit_code=$? + +if [ -f "jepsen_system_fail.txt" ]; then + mv jepsen_system_fail.txt jepsen-mongodb/jepsen_system_failure_${task_name}_${execution} + exit 0 +fi + +exit $exit_code diff --git a/evergreen/jepsen_docker/setup.sh b/evergreen/jepsen_docker/setup.sh index cc76e20e541..38c73cecbec 100644 --- a/evergreen/jepsen_docker/setup.sh +++ b/evergreen/jepsen_docker/setup.sh @@ -1,38 +1,5 @@ set -euo pipefail -if [ ! $(which docker) ]; then - sudo apt-get update - sudo apt-get install -yq \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg \ - lsb-release - - if [ ! -f "/usr/share/keyrings/docker-archive-keyring.gpg" ]; then - curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - fi - - set +e - if ! grep "https://download.docker.com/linux/debian" "/etc/apt/sources.list.d/docker.list"; then - echo \ - "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - fi - set -e - - sudo apt-get update - sudo apt-get install -yq docker-ce docker-ce-cli containerd.io - sudo docker run hello-world -fi - -if [ ! $(which docker-compose) ]; then - sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/bin/docker-compose - sudo chmod +x /usr/bin/docker-compose -fi - -sudo chmod 777 /var/run/docker.sock - git clone --branch=evergreen-master git@github.com:10gen/jepsen.git jepsen cp -rf src/dist-test jepsen/docker/node # place the mongodb jepsen test adjacent to the control node's Dockerfile. diff --git a/evergreen/jepsen_test_run.sh b/evergreen/jepsen_test_run.sh index 2cf8b0ed387..b5f79426079 100644 --- a/evergreen/jepsen_test_run.sh +++ b/evergreen/jepsen_test_run.sh @@ -16,6 +16,11 @@ mkdir -p $TMPDIR export _JAVA_OPTIONS=-Djava.io.tmpdir=$TMPDIR start_time=$(date +%s) +# XXX Do not run this test in a loop or with Jepsen's repeat flag +# the report generator (jepsen_report.sh) does not support parsing more than one +# test at a time. If you need to run these tests with a repeat flag, you +# will need to adopt buildscripts/jepsen_report.py to support the log output +# of older-style Jepsen tests lein run test --test ${jepsen_test_name} \ --mongodb-dir ../ \ --working-dir ${workdir}/src/jepsen-workdir \ diff --git a/evergreen/jstestfuzz_minimize.sh b/evergreen/jstestfuzz_minimize.sh index 13ddbaf5709..e1a3f54a1a4 100644 --- a/evergreen/jstestfuzz_minimize.sh +++ b/evergreen/jstestfuzz_minimize.sh @@ -6,8 +6,6 @@ cd src/jstestfuzz set -o errexit set -o verbose -add_nodejs_to_path - if [ -f "../minimizer-outputs.json" ]; then - eval npm run ${npm_command} -- -j "../minimizer-outputs.json" + eval ./src/scripts/npm_run.sh ${npm_command} -- -j "../minimizer-outputs.json" fi diff --git a/evergreen/jstestfuzz_run.sh b/evergreen/jstestfuzz_run.sh index d79ed69ac9b..9e6f94ee65b 100644 --- a/evergreen/jstestfuzz_run.sh +++ b/evergreen/jstestfuzz_run.sh @@ -1,12 +1,9 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src/jstestfuzz - -set -o errexit -set -o verbose +set -evo pipefail -add_nodejs_to_path +cd src/jstestfuzz in_patch_build_flag="" if [[ "${is_patch}" = "true" ]]; then @@ -17,4 +14,4 @@ if [[ "${is_patch}" = "true" ]]; then esac fi -eval npm run "${npm_command}" -- "${jstestfuzz_vars}" "${in_patch_build_flag}" --branch "${branch_name}" +./src/scripts/npm_run.sh ${npm_command} -- ${jstestfuzz_vars} ${in_patch_build_flag} --branch ${branch_name} diff --git a/evergreen/jstestfuzz_setup.sh b/evergreen/jstestfuzz_setup.sh index f0d2b7a4c29..2d7e2a53cdc 100644 --- a/evergreen/jstestfuzz_setup.sh +++ b/evergreen/jstestfuzz_setup.sh @@ -6,11 +6,4 @@ cd src set -o errexit set -o verbose -add_nodejs_to_path - git clone git@github.com:10gen/jstestfuzz.git - -pushd jstestfuzz -npm install -npm run prepare -popd diff --git a/evergreen/kitchen_run.sh b/evergreen/kitchen_run.sh deleted file mode 100644 index 321e8196e7e..00000000000 --- a/evergreen/kitchen_run.sh +++ /dev/null @@ -1,40 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src/buildscripts/package_test - -set -o errexit - -export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz" -export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" -export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" -export KITCHEN_SUBNET="${kitchen_subnet}" -export KITCHEN_VPC="${kitchen_vpc}" - -if [ ${packager_distro} == "suse12" ]; then - export KITCHEN_YAML="kitchen.legacy.yml" -fi - -if [[ "${packager_arch}" == "aarch64" || "${packager_arch}" == "arm64" ]]; then - kitchen_packager_distro="${packager_distro}-arm64" -else - kitchen_packager_distro="${packager_distro}-x86-64" -fi - -activate_venv -# set expiration tag 2 hours in the future, since no test should take this long -export KITCHEN_EXPIRE="$($python -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')" - -for i in {1..3}; do - if ! kitchen verify $kitchen_packager_distro; then - verified="false" - kitchen destroy $kitchen_packager_distro || true - sleep 30 - else - verified="true" - break - fi -done - -kitchen destroy $kitchen_packager_distro || true -test "$verified" = "true" diff --git a/evergreen/lint_fuzzer_sanity_all.sh b/evergreen/lint_fuzzer_sanity_all.sh index 16ae53b923c..9c2159824e1 100644 --- a/evergreen/lint_fuzzer_sanity_all.sh +++ b/evergreen/lint_fuzzer_sanity_all.sh @@ -1,12 +1,14 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$DIR/prelude.sh" -cd src +cd src/jstestfuzz set -eo pipefail set -o verbose -add_nodejs_to_path - +# Run first with help which will do the install +# Then we can run it in parallel +./src/scripts/npm_run.sh --help # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. -find "$PWD/jstests" "$PWD/src/mongo/db/modules/enterprise" -name "*.js" -print | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- +# Skip javascript files in third_party directory +find "$PWD/../jstests" "$PWD/../src/mongo/db/modules/enterprise" -path "$PWD/../jstests/third_party" -prune -o -name "*.js" -print | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- diff --git a/evergreen/lint_fuzzer_sanity_patch.sh b/evergreen/lint_fuzzer_sanity_patch.sh index 11196aff1ea..af1afd63da7 100644 --- a/evergreen/lint_fuzzer_sanity_patch.sh +++ b/evergreen/lint_fuzzer_sanity_patch.sh @@ -6,15 +6,16 @@ cd src set -eo pipefail set -o verbose -add_nodejs_to_path - mkdir -p jstestfuzzinput jstestfuzzoutput -indir="$(pwd)/jstestfuzzinput" -outdir="$(pwd)/jstestfuzzoutput" +# We need to be the jstestfuzz repo for node to install/run +cd jstestfuzz + +indir="$(pwd)/../jstestfuzzinput" +outdir="$(pwd)/../jstestfuzzoutput" # Grep all the js files from modified_and_created_patch_files.txt and put them into $indir. -(grep -v "\.tpl\.js$" modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true) +(grep -v "\.tpl\.js$" ../modified_and_created_patch_files.txt | grep ".*jstests/.*\.js$" | xargs -I {} cp {} $indir || true) # Count the number of files in $indir. if [[ "$(ls -A $indir)" ]]; then @@ -25,8 +26,8 @@ if [[ "$(ls -A $indir)" ]]; then num_files=50 fi - npm run --prefix jstestfuzz jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50 + ./src/scripts/npm_run.sh jstestfuzz -- --jsTestsDir $indir --out $outdir --numSourceFiles $num_files --numGeneratedFiles 50 # Run parse-jsfiles on 50 files at a time with 32 processes in parallel. - ls -1 -d $outdir/* | xargs -P 32 -L 50 npm run --prefix jstestfuzz parse-jsfiles -- + ls -1 -d $outdir/* | xargs -P 32 -L 50 ./src/scripts/npm_run.sh parse-jsfiles -- fi diff --git a/evergreen/macos_notary.py b/evergreen/macos_notary.py new file mode 100644 index 00000000000..ef799a8e5db --- /dev/null +++ b/evergreen/macos_notary.py @@ -0,0 +1,95 @@ +import os +import platform +import shutil +import urllib.request +import subprocess +import zipfile +import stat +import sys + +if platform.system().lower() != 'darwin': + print("Not a macos system, skipping macos signing.") + sys.exit(0) + +if len(sys.argv) < 2: + print("Must provide at least 1 archive to sign.") + sys.exit(1) + +supported_archs = { + 'arm64': 'arm64', + 'x86_64': 'amd64' +} +arch = platform.uname().machine.lower() + +if arch not in supported_archs: + print(f"Unsupported platform uname arch: {arch}, must be {supported_archs.keys()}") + sys.exit(1) + +macnotary_name = f'darwin_{supported_archs[arch]}' + +if os.environ['project'] == "mongodb-mongo-master-nightly": + signing_type = 'notarizeAndSign' +else: + signing_type = 'sign' + +macnotary_url = f'https://macos-notary-1628249594.s3.amazonaws.com/releases/client/latest/{macnotary_name}.zip' +print(f'Fetching macnotary tool from: {macnotary_url}') +local_filename, headers = urllib.request.urlretrieve(macnotary_url, f'{macnotary_name}.zip') +with zipfile.ZipFile(f'{macnotary_name}.zip') as zipf: + zipf.extractall() + +st = os.stat(f'{macnotary_name}/macnotary') +os.chmod(f'{macnotary_name}/macnotary', st.st_mode | stat.S_IEXEC) + +failed = False +archives = sys.argv[1:] + +for archive in archives: + archive_base, archive_ext = os.path.splitext(archive) + unsigned_archive = f'{archive_base}_unsigned{archive_ext}' + shutil.move(archive, unsigned_archive) + + signing_cmd = [ + f'./{macnotary_name}/macnotary', + '-f', f'{unsigned_archive}', + '-m', f'{signing_type}', + '-u', 'https://dev.macos-notary.build.10gen.cc/api', + '-k', 'server', + '--entitlements', 'etc/macos_entitlements.xml', + '--verify', + '-b', 'server.mongodb.com', + '-i', f'{os.environ["task_id"]}', + '-c', f'{os.environ["project"]}', + '-o', f'{archive}' + ] + + signing_env = os.environ.copy() + signing_env['MACOS_NOTARY_SECRET'] = os.environ["macos_notarization_secret"] + print(' '.join(signing_cmd)) + p = subprocess.Popen(signing_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=signing_env) + + print(f"Signing tool completed with exitcode: {p.returncode}") + for line in iter(p.stdout.readline, b''): + print(f'macnotary: {line.decode("utf-8").strip()}') + + # TODO: BUILD-14595 remove timeout when codesign doesn't frequently hang on macos hosts + timeout = 3600 + timed_out = False + try: + p.wait(timeout=timeout) + except subprocess.TimeoutExpired: + print(f"ERROR: failed to finish signing in timeout period of {timeout} seconds. This most likely is related to hung codesign, see issues underlying BUILD-14595.") + timed_out = True + pass + + if timed_out: + shutil.move(unsigned_archive, archive) + elif p.returncode != 0: + failed = True + shutil.move(unsigned_archive, archive) + else: + os.unlink(unsigned_archive) + +if failed: + exit(1) + diff --git a/evergreen/multiversion_setup.sh b/evergreen/multiversion_setup.sh index 1e932a0c143..6eb9b0edb3f 100644 --- a/evergreen/multiversion_setup.sh +++ b/evergreen/multiversion_setup.sh @@ -7,8 +7,7 @@ set -o errexit set -o verbose activate_venv - -setup_db_contrib_tool_venv +setup_db_contrib_tool export PIPX_HOME="${workdir}/pipx" export PIPX_BIN_DIR="${workdir}/pipx/bin" diff --git a/evergreen/ninja_compile.sh b/evergreen/ninja_compile.sh index f3b51ae5e1d..823944277be 100644 --- a/evergreen/ninja_compile.sh +++ b/evergreen/ninja_compile.sh @@ -5,6 +5,7 @@ cd src set -o errexit set -o verbose + activate_venv python -m pip install ninja if [ "Windows_NT" = "$OS" ]; then @@ -13,8 +14,8 @@ if [ "Windows_NT" = "$OS" ]; then for i in "${compile_env[@]}"; do echo "set $i" >> msvc.bat done - echo "ninja install-core" >> msvc.bat + echo "ninja -f ${ninja_file} install-core" >> msvc.bat cmd /C msvc.bat else - eval ${compile_env} ninja install-core + eval ${compile_env} ninja -f ${ninja_file} install-core fi diff --git a/evergreen/notary_client_crypt_run.sh b/evergreen/notary_client_crypt_run.sh deleted file mode 100644 index 2d9b6b3d689..00000000000 --- a/evergreen/notary_client_crypt_run.sh +++ /dev/null @@ -1,20 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -. ./notary_env.sh - -set -o errexit -set -o verbose - -ext="${ext:-tgz}" - -mv "mongo_crypt_shared_v1.$ext" mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} - -/usr/local/bin/notary-client.py \ - --key-name "server-6.0" \ - --auth-token-file ${workdir}/src/signing_auth_token \ - --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" \ - --notary-url http://notary-service.build.10gen.cc:5000 \ - mongo_crypt_shared_v1-${push_name}-${push_arch}-${suffix}.${ext} diff --git a/evergreen/notary_client_run.sh b/evergreen/notary_client_run.sh deleted file mode 100644 index 82526660d96..00000000000 --- a/evergreen/notary_client_run.sh +++ /dev/null @@ -1,23 +0,0 @@ -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" -. "$DIR/prelude.sh" - -cd src - -. ./notary_env.sh - -set -o errexit -set -o verbose - -long_ext=${ext} -if [ "$long_ext" == "tgz" ]; then - long_ext="tar.gz" -fi - -mv mongo-binaries.tgz mongodb-${push_name}-${push_arch}-${suffix}.${ext} -mv mongo-cryptd.tgz mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} || true -mv mh.tgz mh-${push_name}-${push_arch}-${suffix}.${ext} || true -mv mongo-debugsymbols.tgz mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} || true -mv distsrc.${ext} mongodb-src-${src_suffix}.${long_ext} || true -/usr/bin/find build/ -type f | grep msi$ | xargs -I original_filename cp original_filename mongodb-${push_name}-${push_arch}-${suffix}.msi || true - -/usr/local/bin/notary-client.py --key-name "server-6.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext} mongodb-${push_name}-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${long_ext} mongodb-cryptd-${push_name}-${push_arch}-${suffix}.${ext} diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh index 3844afaead5..eb671d4b5ae 100755 --- a/evergreen/prelude.sh +++ b/evergreen/prelude.sh @@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$evergreen_dir/prelude_python.sh" . "$evergreen_dir/prelude_venv.sh" . "$evergreen_dir/prelude_db_contrib_tool.sh" +. "$evergreen_dir/prelude_mongo_task_generator.sh" expansions_yaml="$evergreen_dir/../../expansions.yml" expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" @@ -31,21 +32,6 @@ unset expansions_default_yaml unset script unset evergreen_dir -function add_nodejs_to_path { - # Add node and npm binaries to PATH - if [ "Windows_NT" = "$OS" ]; then - # An "npm" directory might not have been created in %APPDATA% by the Windows installer. - # Work around the issue by specifying a different %APPDATA% path. - # See: https://github.com/nodejs/node-v0.x-archive/issues/8141 - export APPDATA=${workdir}/npm-app-data - export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location - # TODO: this is to work around BUILD-8652 - cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')" - else - export PATH="$PATH:/opt/node/bin" - fi -} - function posix_workdir { if [ "Windows_NT" = "$OS" ]; then echo $(cygpath -u "${workdir}") diff --git a/evergreen/prelude_db_contrib_tool.sh b/evergreen/prelude_db_contrib_tool.sh index 59c03d7684a..388400a8bd2 100644 --- a/evergreen/prelude_db_contrib_tool.sh +++ b/evergreen/prelude_db_contrib_tool.sh @@ -1,11 +1,11 @@ -function setup_db_contrib_tool_venv { +function setup_db_contrib_tool { - mkdir ${workdir}/pipx + mkdir -p ${workdir}/pipx export PIPX_HOME="${workdir}/pipx" export PIPX_BIN_DIR="${workdir}/pipx/bin" export PATH="$PATH:$PIPX_BIN_DIR" python -m pip --disable-pip-version-check install "pip==21.0.1" "wheel==0.37.0" || exit 1 python -m pip --disable-pip-version-check install "pipx" || exit 1 - pipx install "db-contrib-tool==0.4.5" || exit 1 + pipx install "db-contrib-tool==0.6.2" || exit 1 } diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh new file mode 100644 index 00000000000..d0260ceca63 --- /dev/null +++ b/evergreen/prelude_mongo_task_generator.sh @@ -0,0 +1,24 @@ +function setup_mongo_task_generator { + if [ ! -f mongo-task-generator ]; then + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.7.9/mongo-task-generator --output mongo-task-generator + chmod +x mongo-task-generator + fi +} + +## Comment above and uncomment below to test unreleased mongo-task-generator changes that are +## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git` +## repo +#function setup_mongo_task_generator { +# if [ ! -f mongo-task-generator ]; then +# +# curl https://sh.rustup.rs -sSf | sh -s -- -y +# source "$HOME/.cargo/env" +# git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator +# pushd unreleased-mongo-task-generator +# git checkout <branch-name> +# cargo build --release --locked +# generator_path="$(pwd)/target/release/mongo-task-generator" +# popd +# cp "$generator_path" mongo-task-generator +# fi +#} diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh index 6b97a8a9cfb..a4e9318375e 100644 --- a/evergreen/prelude_venv.sh +++ b/evergreen/prelude_venv.sh @@ -22,8 +22,24 @@ function activate_venv { if [ "Windows_NT" = "$OS" ]; then export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" + elif [ "$(uname)" = "Darwin" ]; then + #SERVER-75626 After activating the virtual environment under the mocos host. the PYTHONPATH setting + #is incorrect, and the site-packages directory of the virtual environment cannot be found in the sys.path. + python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + export PYTHONPATH="${workdir}/venv/lib/python${python_version}/site-packages:${PYTHONPATH}:${workdir}/src" else - export PYTHONPATH="$PYTHONPATH:${workdir}/src" + python_version=$($python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + site_packages="${workdir}/venv/lib/python${python_version}/site-packages" + python -c "import sys; print(sys.path)" + + # Check if site_packages is already in sys.path + in_sys_path=$($python -c "import sys; print('$site_packages' in sys.path)") + if [ "$in_sys_path" = "False" ]; then + export PYTHONPATH="${site_packages}:${PYTHONPATH}:${workdir}/src" + else + export PYTHONPATH="$PYTHONPATH:${workdir}/src" + fi + python -c "import sys; print(sys.path)" fi echo "python set to $(which $python)" diff --git a/evergreen/resmoke_tests_execute.sh b/evergreen/resmoke_tests_execute.sh index ce1ecc7b582..08b9f23042b 100644 --- a/evergreen/resmoke_tests_execute.sh +++ b/evergreen/resmoke_tests_execute.sh @@ -14,6 +14,11 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then # activate the virtualenv if it has been set up activate_venv + # Install db-contrib-tool to symbolize crashes during resmoke suite runs + # This is not supported on Windows and MacOS, so doing it only on Linux + if [ "$(uname)" == "Linux" ]; then + setup_db_contrib_tool + fi if [[ -f "patch_test_tags.tgz" ]]; then tags_build_variant="${build_variant}" @@ -104,6 +109,14 @@ if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\"" fi + # Even though all feature flags may be enabled on a variant, often times we do not want to run + # feature flag tests because they will most likely fail. For example, during multiversion testing, + # all feature flags may be enabled on the latest version, but running feature flag specific tests on + # older versions (last-lts/last-continuous) will likely fail because those features most likely do not exist. + if [[ ${run_no_feature_flag_tests} == "true" ]]; then + extra_args="$extra_args --runNoFeatureFlagTests" + fi + path_value="$PATH:/data/multiversion" # Set the suite name to be the task name by default; unless overridden with the `suite` expansion. diff --git a/evergreen/resmoke_tests_runtime_validate.sh b/evergreen/resmoke_tests_runtime_validate.sh index 1ccfc71c225..aee2111ff92 100644 --- a/evergreen/resmoke_tests_runtime_validate.sh +++ b/evergreen/resmoke_tests_runtime_validate.sh @@ -9,7 +9,6 @@ set -o errexit activate_venv $python buildscripts/resmoke_tests_runtime_validate.py \ --resmoke-report-file ./report.json \ - --evg-api-config ./.evergreen.yml \ --project-id ${project_id} \ --build-variant ${build_variant} \ --task-name ${task_name} diff --git a/evergreen/run_python_script.sh b/evergreen/run_python_script.sh new file mode 100644 index 00000000000..35181dec3c1 --- /dev/null +++ b/evergreen/run_python_script.sh @@ -0,0 +1,11 @@ +unset workdir +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +set -o errexit +set -o verbose + +cd src + +activate_venv +$python $@ diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh index af3ff0bad6d..a6e2be4539e 100755 --- a/evergreen/scons_compile.sh +++ b/evergreen/scons_compile.sh @@ -8,15 +8,6 @@ set -o verbose rm -rf ${install_directory} -# Use hardlinks to reduce the disk space impact of installing -# all of the binaries and associated debug info. - -# The expansion here is a workaround to let us set a different install-action -# for tasks that don't support the one we set here. A better plan would be -# to support install-action for Ninja builds directly. -# TODO: https://jira.mongodb.org/browse/SERVER-48203 -extra_args="--install-action=${task_install_action}" - # By default, limit link jobs to one quarter of our overall -j # concurrency unless locally overridden. We do this because in # static link environments, the memory consumption of each @@ -58,6 +49,8 @@ else extra_args="$extra_args --release" fi +extra_args="$extra_args ENABLE_OOM_RETRY=1" + if [ "${generating_for_ninja}" = "true" ] && [ "Windows_NT" = "$OS" ]; then vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH diff --git a/evergreen/selinux_run_test.sh b/evergreen/selinux_run_test.sh index 318d73adbe2..15fe8be87b6 100755 --- a/evergreen/selinux_run_test.sh +++ b/evergreen/selinux_run_test.sh @@ -22,8 +22,6 @@ if [ "$bypass_prelude" != "yes" ]; then src="src" fi -set -o xtrace - if [ "$hostname" == "" ]; then hostname="$(tr -d '"[]{}' < "$workdir"/$src/hosts.yml | cut -d , -f 1 | awk -F : '{print $2}')" fi @@ -52,10 +50,12 @@ connection_attempts=50 # Check for remote connectivity set +o errexit ssh_options="-i $ssh_key -o IdentitiesOnly=yes -o StrictHostKeyChecking=no" -while ! ssh $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do - if [ "$attempts" -ge "$connection_attempts" ]; then exit 1; fi +while ! ssh -q $ssh_options -o ConnectTimeout=10 "$host" echo "I am working"; do + if [ "$attempts" -ge "$connection_attempts" ]; then + printf "SSH connection attempt failed after %d attempts.\n" "$attempts" + exit 1 + fi ((attempts++)) - printf "SSH connection attempt %d/%d failed. Retrying...\n" "$attempts" "$connection_attempts" sleep 10 done diff --git a/evergreen/selinux_test_executor.sh b/evergreen/selinux_test_executor.sh index a0616d314d6..fc1e04cf2be 100755 --- a/evergreen/selinux_test_executor.sh +++ b/evergreen/selinux_test_executor.sh @@ -1,109 +1,177 @@ #!/bin/bash +set +o errexit -set -o errexit -set -o xtrace +readonly k_log_path="/var/log/mongodb/mongod.log" +readonly k_mongo="$(pwd)/dist-test/bin/mongo" +readonly k_test_path="$1" +return_code=1 -mongo="$(pwd)/dist-test/bin/mongo" -export PATH="$(dirname "$mongo"):$PATH" -if [ ! -f "$mongo" ]; then - echo "Mongo shell at $mongo is missing" - exit 1 -fi +export PATH="$(dirname "$k_mongo"):$PATH" -function print() { +function print_err() { echo "$@" >&2 } function monitor_log() { - sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f /var/log/mongodb/mongod.log) + sed "s!^!mongod| $(date '+%F %H-%M-%S') !" <(sudo --non-interactive tail -f $k_log_path) } -TEST_PATH="$1" -if [ ! -f "$TEST_PATH" ]; then - print "No test supplied or test file not found. Run:" - print " $(basename "${BASH_SOURCE[0]}") <path>" - exit 1 -fi +function output_ausearch() { + local cmd_parameters="AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR" + + echo "" + echo "====== SELinux errors (ausearch -m $cmd_parameters): ======" + sudo --non-interactive ausearch -m $cmd_parameters -ts $1 +} -# test file is even good before going on -if ! "$mongo" --nodb --norc --quiet "$TEST_PATH"; then - print "File $TEST_PATH has syntax errors" +function output_journalctl() { + echo "" + echo "============================== journalctl =========================================" + sudo --non-interactive journalctl --no-pager --catalog --since="$1" | grep -i mongo +} + +function fail_and_exit_err() { + + echo "" + echo "===================================================================================" + echo "++++++++ Test failed, outputting last 5 seconds of additional log info ++++++++++++" + echo "===================================================================================" + output_ausearch "$(date --utc --date='5 seconds ago' '+%x %H:%M:%S')" + output_journalctl "$(date --utc --date='5 seconds ago' +'%Y-%m-%d %H:%M:%S')" + + echo "" + echo "==== FAIL: $1 ====" exit 1 -fi +} -# stop mongod, zero mongo log, clean up database, set all booleans to off -sudo --non-interactive bash -c ' - systemctl stop mongod +function create_mongo_config() { + echo "Writing /etc/mongod.conf for $k_test_path:" + "$k_mongo" --nodb --norc --quiet --eval=' + assert(load("'"$k_test_path"'")); + const test = new TestDefinition(); + print(JSON.stringify(test.config, null, 2)); - rm -f /var/log/mongodb/mongod.log - touch /var/log/mongodb/mongod.log - chown mongod /var/log/mongodb/mongod.log + ' | sudo --non-interactive tee /etc/mongod.conf +} - rm -rf /var/lib/mongo/* +function start_mongod() { + # Start mongod and if it won't come up, fail and exit - rm -rf /etc/sysconfig/mongod /etc/mongod + sudo --non-interactive systemctl start mongod \ + && sudo --non-interactive systemctl status mongod || ( + fail_and_exit_err "systemd failed to start mongod server!" + ) +} - setsebool mongod_can_connect_snmp off - setsebool mongod_can_connect_ldap off - setsebool mongod_can_use_kerberos off -' +function wait_for_mongod_to_accept_connections() { + # Once the mongod process starts via systemd, it can still take a couple of seconds + # to set up and accept connections... we will wait for log id 23016 to show up + # indicating that the server is ready to accept incoming connections before starting the tests -# create mongo config -"$mongo" --nodb --norc --quiet --eval=' - assert(load("'"$TEST_PATH"'")); - const test = new TestDefinition(); - print(typeof(test.config) === "string" ? test.config : JSON.stringify(test.config, null, 2)); -' | sudo --non-interactive tee /etc/mongod.conf + local server_ready=0 + local wait_seconds=2 + local wait_retries_max=30 + local wait_retries=0 -# setup -"$mongo" --nodb --norc --quiet --eval=' - assert(load("'"$TEST_PATH"'")); - const test = new TestDefinition(); - jsTest.log("Running setup()"); - test.setup(); -' + while [[ $wait_retries -le $wait_retries_max ]]; do + local server_status="$(grep 23016 $k_log_path || echo "")" -# start log monitor, also kill it on exit -monitor_log & -MONITORPID="$!" -trap "sudo --non-interactive pkill -P $MONITORPID" SIGINT SIGTERM ERR EXIT - -# start mongod and if it won't come up, log SELinux errors -ts="$(date --utc --date='1 seconds ago' '+%x %H:%M:%S')" -tsj="$(date --utc --date='1 seconds ago' +'%Y-%m-%d %H:%M:%S')" -sudo --non-interactive systemctl start mongod \ - && sudo --non-interactive systemctl status mongod || ( - set +o errexit - echo "=== SELinux errors:" - sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts - echo "=== journalctl --unit=mongod:" - sudo --non-interactive journalctl --no-pager --since="$tsj" --unit=mongod --unit=systemd --catalog - echo "=== /var/log/mongodb/mongod.log:" - sudo --non-interactive cat /var/log/mongodb/mongod.log - echo "==== FAIL: mongod service was not started successfully" + if [ "$server_status" != "" ]; then + server_ready=1 + break + fi + + sleep $wait_seconds + ((wait_retries++)) + done + + if [ ! $server_ready ]; then + fail_and_exit_err "failed to connect to mongod server after waiting for $(($wait_seconds * $wait_retries)) seconds!" + fi +} + +function clear_mongo_config() { + # stop mongod, zero mongo log, clean up database, set all booleans to off + sudo --non-interactive bash -c ' + systemctl stop mongod + + rm -f '"$k_log_path"' + touch '"$k_log_path"' + chown mongod '"$k_log_path"' + + rm -rf /var/lib/mongo/* + + rm -rf /etc/sysconfig/mongod /etc/mongod + + setsebool mongod_can_connect_ldap off + setsebool mongod_can_use_kerberos off + ' +} + +function exit_with_code() { + exit $return_code +} + +function setup_test_definition() { + "$k_mongo" --nodb --norc --quiet --eval=' + assert(load("'"$k_test_path"'")); + (() => { + const test = new TestDefinition(); + print("Running setup() for '"$k_test_path"'"); + test.setup(); + })(); + ' +} + +function run_test() { + "$k_mongo" --norc --gssapiServiceName=mockservice --eval=' + assert(load("'"$k_test_path"'")); + print("Running test '"$k_test_path"'"); + + (() => { + const test = new TestDefinition(); + + try { + test.run(); + } finally { + test.teardown(); + } + })(); + ' || fail_and_exit_err "Test failed" + + echo "SUCCESS: $k_test_path" +} + +if [ ! -f "$k_mongo" ]; then + print_err "Mongo shell at $k_mongo is missing" exit 1 -) - -# run test and teardown -"$mongo" --norc --gssapiServiceName=mockservice --eval=' - assert(load("'"$TEST_PATH"'")); - // name is such to prevent collisions - const test_812de7ce = new TestDefinition(); - try { - jsTest.log("Running test"); - test_812de7ce.run(); - } finally { - test_812de7ce.teardown(); - } -' || ( - echo "==== FAIL: test returned result: $?" - echo "=== SELinux errors:" - set +o errexit - sudo --non-interactive ausearch -m AVC,USER_AVC,SELINUX_ERR,USER_SELINUX_ERR -ts $ts - echo "=== /var/log/mongodb/mongod.log:" - sudo --non-interactive cat /var/log/mongodb/mongod.log +fi + +if [ ! -f "$k_test_path" ]; then + print_err "No test supplied or test file not found. Run:" + print_err "$(basename "${BASH_SOURCE[0]}") <path>" + exit 1 +fi + +# Ensure file containing tests is valid before executing +if ! "$k_mongo" --nodb --norc --quiet "$k_test_path"; then + print_err "File $k_test_path has syntax errors" exit 1 -) +fi + +echo "STARTING TEST: $k_test_path" + +clear_mongo_config +create_mongo_config +setup_test_definition + +# start log monitor, also kill it on exit +monitor_log & +monitor_pid="$!" +trap "sudo --non-interactive pkill -P $monitor_pid; exit_with_code" SIGINT SIGTERM ERR EXIT + +start_mongod +wait_for_mongod_to_accept_connections +run_test -set +o xtrace -echo "SUCCESS: $TEST_PATH" +return_code=0 |