diff options
Diffstat (limited to 'buildscripts/resmokelib')
36 files changed, 528 insertions, 1113 deletions
diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py index 3dc6a0a522d..2cb85721e9e 100644 --- a/buildscripts/resmokelib/cli.py +++ b/buildscripts/resmokelib/cli.py @@ -18,6 +18,6 @@ def main(argv): usage="Resmoke is MongoDB's correctness testing orchestrator.\n" "For more information, see the help message for each subcommand.\n" "For example: resmoke.py run -h\n" - "Note: bisect, setup-multiversion and symbolize subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" + "Note: bisect and setup-multiversion subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" ) subcommand.execute() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index f3ab984756c..c1cc1048585 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -55,7 +55,7 @@ DEFAULTS = { "archive_limit_tests": 10, "base_port": 20000, "backup_on_restart_dir": None, - "buildlogger_url": "https://logkeeper2.build.10gen.cc", + "buildlogger_url": "https://logkeeper.mongodb.org", "continue_on_failure": False, "dbpath_prefix": None, "dbtest_executable": None, @@ -88,11 +88,11 @@ DEFAULTS = { "repeat_tests_min": None, "repeat_tests_secs": None, "replay_file": None, + "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, - "run_no_feature_flag_tests": False, + "run_all_feature_flags_no_tests": False, "additional_feature_flags": None, - "additional_feature_flags_file": None, "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, @@ -101,7 +101,6 @@ DEFAULTS = { "stagger_jobs": None, "majority_read_concern": "on", "storage_engine": "wiredTiger", - "enable_enterprise_tests": "on", "storage_engine_cache_size_gb": None, "suite_files": "with_server", "tag_files": [], @@ -132,7 +131,6 @@ DEFAULTS = { "task_doc": None, "variant_name": None, "version_id": None, - "work_dir": None, # WiredTiger options. "wt_coll_config": None, @@ -168,6 +166,7 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "num_repeat_tests_max", "num_repeat_tests_min", "time_repeat_tests_secs", + "report_failure_status", ]) @@ -234,6 +233,7 @@ class SuiteOptions(_SuiteOptions): REPEAT_TESTS_MAX, REPEAT_TESTS_MIN, REPEAT_TESTS_SECS, + REPORT_FAILURE_STATUS, ]))) options = self._asdict() @@ -295,9 +295,6 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None -# if set, enables enterprise jstest to automatically be included -ENABLE_ENTERPRISE_TESTS = None - # URL to connect to the Evergreen service. EVERGREEN_URL = None @@ -340,12 +337,6 @@ EVERGREEN_VARIANT_NAME = None # the commit hash. EVERGREEN_VERSION_ID = None -# The Evergreen task's working directory. -EVERGREEN_WORK_DIR = None - -# Path to evergreen project configuration yaml file -EVERGREEN_PROJECT_CONFIG_PATH = None - # If set, then any jstests that have any of the specified tags will be excluded from the suite(s). EXCLUDE_WITH_ANY_TAGS = None @@ -384,11 +375,8 @@ INSTALL_DIR = None # Whether to run tests for feature flags. RUN_ALL_FEATURE_FLAG_TESTS = None -# Whether to run the tests with enabled feature flags -RUN_NO_FEATURE_FLAG_TESTS = None - -# the path to a file containing feature flags -ADDITIONAL_FEATURE_FLAGS_FILE = None +# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true. +RUN_ALL_FEATURE_FLAGS = None # List of enabled feature flags. ENABLED_FEATURE_FLAGS = [] @@ -445,6 +433,9 @@ REPEAT_TESTS_MIN = None # If set, then each test is repeated the specified time (seconds) inside the suites. REPEAT_TESTS_SECS = None +# Controls if the test failure status should be reported as failed or be silently ignored. +REPORT_FAILURE_STATUS = None + # If set, then resmoke.py will write out a report file with the status of each test that ran. REPORT_FILE = None @@ -565,17 +556,12 @@ DEFAULT_BENCHMARK_TEST_LIST = "build/benchmarks.txt" DEFAULT_UNIT_TEST_LIST = "build/unittests.txt" DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt" DEFAULT_LIBFUZZER_TEST_LIST = "build/libfuzzer_tests.txt" -DEFAULT_PRETTY_PRINTER_TEST_LIST = "build/pretty_printer_tests.txt" -SPLIT_UNITTESTS_LISTS = [ - f"build/{test_group}_quarter_unittests.txt" - for test_group in ['first', 'second', 'third', 'fourth'] -] + # External files or executables, used as suite selectors, that are created during the build and # therefore might not be available when creating a test membership map. EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE, - DEFAULT_LIBFUZZER_TEST_LIST, DEFAULT_PRETTY_PRINTER_TEST_LIST, - *SPLIT_UNITTESTS_LISTS) + DEFAULT_LIBFUZZER_TEST_LIST) # Where to look for logging and suite configuration files CONFIG_DIR = None @@ -590,12 +576,3 @@ SHORTEN_LOGGER_NAME_CONFIG: dict = {} # that get loaded for multiversion tests can behave correctly on master and on v5.0; the latter # case runs 5.0 and 4.4 binaries and has this value set to True. Can be removed after 6.0. USE_LEGACY_MULTIVERSION = True - -# Expansions file location -# in CI, the expansions file is located in the ${workdir}, one dir up -# from src, the checkout directory -EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else "expansions.yml" - -# Symbolizer secrets -SYMBOLIZER_CLIENT_SECRET = None -SYMBOLIZER_CLIENT_ID = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index ed659d2f79b..db1319b8343 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -1,6 +1,5 @@ """Configure the command line input for the resmoke 'run' subcommand.""" -import argparse import collections import configparser import datetime @@ -16,21 +15,18 @@ import shlex import pymongo.uri_parser -from buildscripts.idl import gen_all_feature_flag_list from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils from buildscripts.resmokelib import mongod_fuzzer_configs from buildscripts.resmokelib.suitesconfig import SuiteFinder -from buildscripts.util.read_config import read_config_file def validate_and_update_config(parser, args): """Validate inputs and update config module.""" _validate_options(parser, args) _update_config_vars(args) - _update_symbolizer_secrets() _validate_config(parser) _set_logging_config() @@ -55,9 +51,14 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) - if args.additional_feature_flags_file and not os.path.isfile( - args.additional_feature_flags_file): - parser.error("The specified additional feature flags file does not exist.") + if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests: + if not os.path.isfile(ALL_FEATURE_FLAG_FILE): + parser.error( + "To run tests with all feature flags, the %s file must exist and be placed in" + " your working directory. The file can be downloaded from the artifacts tarball" + " in Evergreen. Alternatively, if you know which feature flags you want to enable," + " you can use the --additionalFeatureFlags command line argument" % + ALL_FEATURE_FLAG_FILE) def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) @@ -183,36 +184,28 @@ be invoked as either: - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""") raise RuntimeError(err) - def process_feature_flag_file(path): - with open(path) as fd: - return fd.read().split() - def setup_feature_flags(): _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") - _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests") - _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") + _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests") + # Running all feature flag tests implies running the fixtures with feature flags. if _config.RUN_ALL_FEATURE_FLAG_TESTS: - print("Generating: ", ALL_FEATURE_FLAG_FILE) - gen_all_feature_flag_list.gen_all_feature_flags_file() + _config.RUN_ALL_FEATURE_FLAGS = True all_ff = [] enabled_feature_flags = [] try: - all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE) + with open(ALL_FEATURE_FLAG_FILE) as fd: + all_ff = fd.read().split() except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. - if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS: raise - if _config.RUN_ALL_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS: enabled_feature_flags = all_ff[:] - if _config.ADDITIONAL_FEATURE_FLAGS_FILE: - enabled_feature_flags.extend( - process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE)) - # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags")) @@ -236,7 +229,7 @@ be invoked as either: _config.EXCLUDE_WITH_ANY_TAGS.extend( utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - if _config.RUN_NO_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS: # Don't run any feature flag tests. _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags) else: @@ -254,7 +247,6 @@ be invoked as either: _config.JOBS = config.pop("jobs") _config.LINEAR_CHAIN = config.pop("linear_chain") == "on" _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on" - _config.ENABLE_ENTERPRISE_TESTS = config.pop("enable_enterprise_tests") _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions") if _config.MIXED_BIN_VERSIONS is not None: _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-") @@ -310,8 +302,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG, _config.WT_COLL_CONFIG, \ _config.WT_INDEX_CONFIG = mongod_fuzzer_configs.fuzz_set_parameters( _config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS) - _config.EXCLUDE_WITH_ANY_TAGS.extend(["uses_compact"]) - _config.EXCLUDE_WITH_ANY_TAGS.extend(["requires_emptycapped"]) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) mongos_set_parameters = config.pop("mongos_set_parameters") @@ -332,6 +322,7 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") + _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config") @@ -362,7 +353,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.EVERGREEN_TASK_DOC = config.pop("task_doc") _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") _config.EVERGREEN_VERSION_ID = config.pop("version_id") - _config.EVERGREEN_WORK_DIR = config.pop("work_dir") # Archival options. Archival is enabled only when running on evergreen. if not _config.EVERGREEN_TASK_ID: @@ -512,36 +502,3 @@ def _tags_from_list(tags_list): tags.extend([t for t in tag.split(",") if t != ""]) return tags return None - - -def _update_symbolizer_secrets(): - """Open `expansions.yml`, get values for symbolizer secrets and update their values inside config.py .""" - if not _config.EVERGREEN_TASK_ID: - # not running on Evergreen - return - yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) - _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") - _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") - - -def detect_evergreen_config(parsed_args: argparse.Namespace, - expansions_file: str = "../expansions.yml"): - """Detect evergreen expansions.""" - if not os.path.exists(expansions_file): - return - - expansions = read_config_file(expansions_file) - - parsed_args.build_id = expansions.get("build_id", None) - parsed_args.distro_id = expansions.get("distro_id", None) - parsed_args.execution_number = expansions.get("execution", None) - parsed_args.project_name = expansions.get("project", None) - parsed_args.git_revision = expansions.get("revision", None) - parsed_args.revision_order_id = expansions.get("revision_order_id", None) - parsed_args.task_id = expansions.get("task_id", None) - parsed_args.task_name = expansions.get("task_name", None) - parsed_args.variant_name = expansions.get("build_variant", None) - parsed_args.version_id = expansions.get("version_id", None) - parsed_args.work_dir = expansions.get("workdir", None) - parsed_args.evg_project_config_path = expansions.get("evergreen_config_file_path", None) - parsed_args.requester = expansions.get("requester", None) diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index fa9a2e98ebd..19354d4c314 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -6,9 +6,7 @@ Handles all the nitty-gritty parameter conversion. import json import os import os.path -import re import stat -from packaging import version from buildscripts.resmokelib import config from buildscripts.resmokelib import utils @@ -45,31 +43,6 @@ def get_path_env_var(env_vars): return path -def get_binary_version(executable): - """Return the string for the binary version of the given executable.""" - - # pylint: disable=wrong-import-position - from buildscripts.resmokelib.multiversionconstants import LATEST_FCV - - split_executable = executable.split("-") - version_regex = re.compile(version.VERSION_PATTERN, re.VERBOSE | re.IGNORECASE) - if len(split_executable) > 1 and version_regex.match(split_executable[-1]): - return split_executable[-1] - return LATEST_FCV - - -def remove_set_parameter_if_before_version(set_parameters, parameter_name, bin_version, - required_bin_version): - """ - Used for removing a server parameter that does not exist prior to a specified version. - - Remove 'parameter_name' from the 'set_parameters' dictionary if 'bin_version' is older than - 'required_bin_version'. - """ - if version.parse(bin_version) < version.parse(required_bin_version): - set_parameters.pop(parameter_name, None) - - def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): """ Return a Process instance that starts mongod arguments constructed from 'mongod_options'. @@ -81,17 +54,12 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): @param mongod_options - A HistoryDict describing the various options to pass to the mongod. """ - bin_version = get_binary_version(executable) args = [executable] mongod_options = mongod_options.copy() if "port" not in mongod_options: mongod_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongod_options.get("set_parameters", {}) - remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", - bin_version, "6.0") - remove_set_parameter_if_before_version( - suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongod_options.pop("set_parameters") @@ -110,7 +78,6 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos_options=None): # pylint: disable=too-many-arguments """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" - bin_version = get_binary_version(executable) args = [executable] mongos_options = mongos_options.copy() @@ -118,10 +85,6 @@ def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos if "port" not in mongos_options: mongos_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongos_options.get("set_parameters", {}) - remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", - bin_version, "6.0") - remove_set_parameter_if_before_version( - suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongos_options.pop("set_parameters") @@ -288,11 +251,6 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');") - if config.FUZZ_MONGOD_CONFIGS is not None and config.FUZZ_MONGOD_CONFIGS is not False: - # Prevent commands from running with the config fuzzer. - eval_sb.append( - "load('jstests/libs/override_methods/config_fuzzer_incompatible_commands.js');") - # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');") diff --git a/buildscripts/resmokelib/flags.py b/buildscripts/resmokelib/flags.py deleted file mode 100644 index 6aff6961666..00000000000 --- a/buildscripts/resmokelib/flags.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Global flags used by resmoke.""" - -import threading - -HANG_ANALYZER_CALLED = threading.Event() diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 97788b0469b..e5112b870b3 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -5,14 +5,12 @@ import logging import os import sys import tempfile -from datetime import datetime from abc import ABCMeta, abstractmethod from collections import namedtuple from distutils import spawn # pylint: disable=no-name-in-module from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program from buildscripts.resmokelib.hang_analyzer.process_list import Pinfo -from buildscripts.resmokelib import config as resmoke_config Dumpers = namedtuple('Dumpers', ['dbg', 'jstack']) @@ -332,20 +330,6 @@ class LLDBDumper(Dumper): class GDBDumper(Dumper): """GDBDumper class.""" - def __init__(self, root_logger: logging.Logger, dbg_output: str, - timeout_seconds_for_gdb_process=720): - """Initialize GDBDumper.""" - if resmoke_config.EVERGREEN_TASK_ID is None: - # Set 24 hours time out for hang analyzer being run in locally - timeout_seconds_for_gdb_process = 86400 - #Timeout for hang analyzer, default timeout is 12mins(out of total 15mins) in Evergreen - self._timeout_seconds_for_gdb_process = timeout_seconds_for_gdb_process - super().__init__(root_logger, dbg_output) - - def _reduce_timeout_for_gdb_process(self, timeout_period: int): - """Reduce timeout for remaining gdb processes.""" - self._timeout_seconds_for_gdb_process -= timeout_period - def _find_debugger(self, debugger): """Find the installed debugger.""" return find_program(debugger, ['/opt/mongodbtoolchain/v3/bin', '/usr/bin']) @@ -461,19 +445,12 @@ class GDBDumper(Dumper): debugger = "gdb" dbg = self._find_debugger(debugger) logger = _get_process_logger(self._dbg_output, pinfo.name) - _start_time = datetime.now() if dbg is None: self._root_logger.warning("Debugger %s not found, skipping dumping of %s", debugger, str(pinfo.pidv)) return - if self._timeout_seconds_for_gdb_process <= 0: - self._root_logger.warning( - "Skipping dumping of %s processes with PIDs %s because the time limit expired", - pinfo.name, str(pinfo.pidv)) - return - self._root_logger.info("Debugger %s, analyzing %s processes with PIDs %s", dbg, pinfo.name, str(pinfo.pidv)) @@ -481,20 +458,9 @@ class GDBDumper(Dumper): cmds = self._prefix() + self._process_specific(pinfo, take_dump, logger) + self._postfix() - # gcore is both a command within GDB and a script packaged alongside gdb. The gcore script - # invokes the gdb binary with --readnever to avoid spending time loading the debug symbols - # prior to taking the core dump. The debug symbols are unneeded to generate the core dump. - # - # For reference - # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/gcore.in;h=34860de630cf0ee766e102eb82f7a3fddba6b368#l101 - skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] - - call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( - itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger, - self._timeout_seconds_for_gdb_process, pinfo) + call([dbg, "--quiet", "--nx"] + list( + itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) - time_period = (datetime.now() - _start_time).total_seconds() - self._reduce_timeout_for_gdb_process(time_period) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, str(pinfo.pidv)) diff --git a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py index ba259955444..44d0b23e54e 100755 --- a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py +++ b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py @@ -68,13 +68,7 @@ class HangAnalyzer(Subcommand): self._configure_processes() self._setup_logging(logger) - def kill_rogue_processes(self): - """Kill any processes that are currently being analyzed.""" - processes = process_list.get_processes(self.process_ids, self.interesting_processes, - self.options.process_match, self.root_logger) - process.teardown_processes(self.root_logger, processes, dump_pids={}) - - def execute(self): # pylint: disable=too-many-branches + def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements """ Execute hang analysis. diff --git a/buildscripts/resmokelib/hang_analyzer/process.py b/buildscripts/resmokelib/hang_analyzer/process.py index ee528aa6de6..629f78a2594 100644 --- a/buildscripts/resmokelib/hang_analyzer/process.py +++ b/buildscripts/resmokelib/hang_analyzer/process.py @@ -22,7 +22,7 @@ if _IS_WINDOWS: PROCS_TIMEOUT_SECS = 60 -def call(args, logger, timeout_seconds=None, pinfo=None): +def call(args, logger): """Call subprocess on args list.""" logger.info(str(args)) @@ -31,16 +31,7 @@ def call(args, logger, timeout_seconds=None, pinfo=None): logger_pipe = core.pipe.LoggerPipe(logger, logging.INFO, process.stdout) logger_pipe.wait_until_started() - try: - ret = process.wait(timeout=timeout_seconds) - except subprocess.TimeoutExpired: - logger.error("Killing %s processes with PIDs %s because time limit expired", pinfo.name, - str(pinfo.pidv)) - process.kill() - process.wait() - logger_pipe.wait_until_finished() - return - + ret = process.wait() logger_pipe.wait_until_finished() if ret != 0: diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index 8aa6b6bd983..6325ecabb89 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -14,7 +14,6 @@ CREATE_BUILD_ENDPOINT = "/build" APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" -PARSLEY_LOGS_URL = "https://parsley.mongodb.com/resmoke/%(build_id)s/test/%(test_id)s" _BUILDLOGGER_CONFIG = os.getenv("BUILDLOGGER_CREDENTIALS", "mci.buildlogger") @@ -299,7 +298,6 @@ class BuildloggerServer(object): "builder": builder, "buildnum": build_num, "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -318,7 +316,6 @@ class BuildloggerServer(object): "command": test_command, "phase": self.config.get("build_phase", "unknown"), "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -344,8 +341,3 @@ class BuildloggerServer(object): base_url = _config.BUILDLOGGER_URL.rstrip("/") endpoint = APPEND_TEST_LOGS_ENDPOINT % {"build_id": build_id, "test_id": test_id} return "%s/%s" % (base_url, endpoint.strip("/")) - - @staticmethod - def get_parsley_log_url(build_id, test_id): - """Return the parsley log URL.""" - return PARSLEY_LOGS_URL % {"build_id": build_id, "test_id": test_id} diff --git a/buildscripts/resmokelib/logging/formatters.py b/buildscripts/resmokelib/logging/formatters.py index b4ee2abd4b5..2998756a27e 100644 --- a/buildscripts/resmokelib/logging/formatters.py +++ b/buildscripts/resmokelib/logging/formatters.py @@ -19,17 +19,3 @@ class TimestampFormatter(logging.Formatter): formatted_time = time.strftime("%H:%M:%S", converted_time) return "%s.%03dZ" % (formatted_time, record.msecs) - - -class EvergreenLogFormatter(logging.Formatter): - """Log line formatter for Evergreen log messages. - - See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` - for more info. - """ - - def format(self, record): - """Return formatted line.""" - ts = int(record.created * 1e9) - - return "\n".join([f"{ts} {line}" for line in super().format(record).split("\n")]) diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 07bc214517a..692b4532c47 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -161,29 +161,6 @@ class BufferedHandler(logging.Handler): logging.Handler.close(self) -class BufferedFileHandler(BufferedHandler): - """File handler with in-memory buffering.""" - - def __init__(self, filename, capacity=2000, interval_secs=600): - """Initialize the handler with the filename and buffer capacity and flush interval.""" - super().__init__(capacity, interval_secs) - self.file = open(filename, "a", encoding="utf-8") - - def process_record(self, record): - """Return the formatted record message appended with a newline.""" - return self.format(record) + "\n" - - def _flush_buffer_with_lock(self, buf, close_called): - """Write the buffered log lines to the destination file.""" - self.file.writelines(buf) - - def close(self): - """Close the handler and the file descriptor.""" - super().close() - - self.file.close() - - class HTTPHandler(object): """A class which sends data to a web server using POST requests.""" diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index 8775c830669..6d5fee06912 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -1,19 +1,16 @@ """Module to hold the logger instances themselves.""" import logging -import os import re import shutil import subprocess import sys -import yaml from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import redirect as redirect_lib from buildscripts.resmokelib.logging import buildlogger from buildscripts.resmokelib.logging import formatters -from buildscripts.resmokelib.logging.handlers import BufferedFileHandler _DEFAULT_FORMAT = "[%(name)s] %(message)s" @@ -97,13 +94,6 @@ def configure_loggers(): global ROOT_EXECUTOR_LOGGER # pylint: disable=global-statement ROOT_EXECUTOR_LOGGER = new_root_logger(EXECUTOR_LOGGER_NAME) - _write_evergreen_log_spec() - - -def get_evergreen_log_name(job_num, test_id=None): - """Return the log name, relative to the reserved test log directory, on the Evergreen task host.""" - return f"job{job_num}/" + (f"{test_id}.log" if test_id else "global.log") - def new_root_logger(name): """ @@ -180,7 +170,6 @@ def new_fixture_logger(fixture_class, job_num): logger = FixtureLogger(_shorten(full_name), full_name) logger.parent = ROOT_FIXTURE_LOGGER _add_build_logger_handler(logger, job_num) - _add_evergreen_handler(logger, job_num) _FIXTURE_LOGGER_REGISTRY[job_num] = logger return logger @@ -210,7 +199,6 @@ def new_test_logger(test_shortname, test_basename, command, parent, job_num, tes name = "%s:%s" % (parent.name, test_shortname) logger = logging.Logger(name) logger.parent = parent - _add_evergreen_handler(logger, job_num, test_id) def _get_test_endpoint(job_num, test_basename, command, meta_logger): """Get a new test endpoint for the buildlogger server.""" @@ -268,8 +256,6 @@ def _add_handler(logger, handler_info, formatter): handler = logging.StreamHandler(sys.stdout) elif handler_class == "buildlogger": return # Buildlogger handlers are applied when creating specific child loggers - elif handler_class == "evergreen": - return # Evergreen handlers are applied when creating specific child loggers else: raise ValueError("Unknown handler class '%s'" % handler_class) handler.setFormatter(formatter) @@ -339,55 +325,3 @@ def _shorten(logger_name): logger_name = re.sub(r"(^[:_]+|[:_]+$)", "", logger_name) return logger_name - - -# Utility functions for Evergreen file system logging. -# See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` -# for more information. - - -def _write_evergreen_log_spec(): - """Configure file system logging for Evergreen tasks.""" - if not config.EVERGREEN_WORK_DIR: - return - - fp = f"{_get_evergreen_log_dirname()}/log_spec.yaml" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - ROOT_EXECUTOR_LOGGER.info("Writing Evergreen test log spec to %s.", fp) - - log_spec = { - "version": 0, - "format": "text-timestamp", - } - with open(fp, "w") as fd: - yaml.dump(log_spec, fd) - - -def _add_evergreen_handler(logger, job_num, test_id=None): - """Add a new evergreen handler to a logger.""" - logger_info = config.LOGGING_CONFIG[TESTS_LOGGER_NAME] - evergreen_handler_info = None - for handler_info in logger_info["handlers"]: - if handler_info["class"] == "evergreen": - evergreen_handler_info = handler_info - break - - if evergreen_handler_info: - fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - handler = BufferedFileHandler(fp) - handler.setFormatter( - formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT))) - logger.addHandler(handler) - - if test_id: - ROOT_EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp) - else: - ROOT_EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp) - - -def _get_evergreen_log_dirname(): - """Return the reserved directory for test logs on the Evergreen task host.""" - return f"{config.EVERGREEN_WORK_DIR}/build/TestLogs" diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongod_fuzzer_configs.py index 874bc7764cf..ec84d6c5a4e 100644 --- a/buildscripts/resmokelib/mongod_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongod_fuzzer_configs.py @@ -11,16 +11,13 @@ def generate_eviction_configs(rng): eviction_trigger = rng.randint(eviction_target + 1, 99) # Fuzz eviction_dirty_target and trigger both as relative and absolute values - target_bytes_min = 50 * 1024 * 1024 # 50MB # 5% of 1GB default cache size on Evergreen + target_bytes_min = 10 * 1024 * 1024 # 10MB target_bytes_max = 256 * 1024 * 1024 # 256MB # 1GB default cache size on Evergreen eviction_dirty_target = rng.choice( [rng.randint(5, 50), rng.randint(target_bytes_min, target_bytes_max)]) trigger_max = 75 if eviction_dirty_target <= 50 else target_bytes_max eviction_dirty_trigger = rng.randint(eviction_dirty_target + 1, trigger_max) - assert eviction_dirty_trigger > eviction_dirty_target - assert eviction_dirty_trigger <= trigger_max - close_idle_time_secs = rng.randint(1, 100) close_handle_minimum = rng.randint(0, 1000) close_scan_interval = rng.randint(1, 100) @@ -92,8 +89,6 @@ def generate_independent_parameters(rng): # The old retryable writes format is used by other variants. Weight towards turning on the # new retryable writes format on in this one. ret["storeFindAndModifyImagesInSideCollection"] = True - # TODO (SERVER-75632): Uncomment this to enable passthrough testing. - # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) return ret diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 7160d1ff026..2c9bae45766 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -1,5 +1,4 @@ """Subcommand for multiversion config.""" -import argparse from typing import List, Optional import yaml @@ -37,19 +36,10 @@ class MultiversionConfig(BaseModel): class MultiversionConfigSubcommand(Subcommand): """Subcommand for discovering multiversion configuration.""" - def __init__(self, options: argparse.Namespace) -> None: - """Initialize the class.""" - self.config_file_output = options.config_file_output - def execute(self): """Execute the subcommand.""" mv_config = self.determine_multiversion_config() - yaml_output = yaml.safe_dump(mv_config.dict()) - print(yaml_output) - - if self.config_file_output: - with open(self.config_file_output, "w") as file: - file.write(yaml_output) + print(yaml.safe_dump(mv_config.dict())) @staticmethod def determine_multiversion_config() -> MultiversionConfig: @@ -59,14 +49,14 @@ class MultiversionConfigSubcommand(Subcommand): mongo_version=MongoVersion.from_yaml_file(multiversionconstants.MONGO_VERSION_YAML), mongo_releases=MongoReleases.from_yaml_file(multiversionconstants.RELEASES_YAML), ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() return MultiversionConfig( last_versions=multiversionconstants.OLD_VERSIONS, - requires_fcv_tag=version_constants.get_fcv_tag_list(), - requires_fcv_tag_lts=version_constants.get_lts_fcv_tag_list(), - requires_fcv_tag_continuous=version_constants.get_continuous_fcv_tag_list(), - last_lts_fcv=version_constants.get_last_lts_fcv(), - last_continuous_fcv=version_constants.get_last_continuous_fcv(), + requires_fcv_tag=fcv_constants.get_fcv_tag_list(), + requires_fcv_tag_lts=fcv_constants.get_lts_fcv_tag_list(), + requires_fcv_tag_continuous=fcv_constants.get_continuous_fcv_tag_list(), + last_lts_fcv=fcv_constants.get_last_lts_fcv(), + last_continuous_fcv=fcv_constants.get_last_continuous_fcv(), ) @@ -79,14 +69,10 @@ class MultiversionPlugin(PluginInterface): :param subparsers: argparse subparsers """ - parser = subparsers.add_parser(MULTIVERSION_SUBCOMMAND, - help="Display configuration for multiversion testing") - - parser.add_argument("--config-file-output", '-f', action="store", type=str, default=None, - help="File to write the multiversion config to.") + subparsers.add_parser(MULTIVERSION_SUBCOMMAND, + help="Display configuration for multiversion testing") - def parse(self, subcommand: str, parser: argparse.ArgumentParser, - parsed_args: argparse.Namespace, **kwargs) -> Optional[Subcommand]: + def parse(self, subcommand, parser, parsed_args, **kwargs) -> Optional[Subcommand]: """ Resolve command-line options to a Subcommand or None. @@ -96,8 +82,7 @@ class MultiversionPlugin(PluginInterface): :param kwargs: additional args. :return: None or a Subcommand. """ + configure_resmoke.validate_and_update_config(parser, parsed_args) if subcommand == MULTIVERSION_SUBCOMMAND: - configure_resmoke.detect_evergreen_config(parsed_args) - configure_resmoke.validate_and_update_config(parser, parsed_args) - return MultiversionConfigSubcommand(parsed_args) + return MultiversionConfigSubcommand() return None diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 621d5ade9ae..3597cc2177f 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -25,9 +25,9 @@ def version_str(version: Version) -> str: return f"{version.major}.{version.minor}" -class VersionConstantValues(NamedTuple): +class FcvConstantValues(NamedTuple): """ - Object to hold the calculated Version constants. + Object to hold the calculated FCV constants. * latest: Latest FCV. * last_continuous: Last continuous FCV. @@ -36,7 +36,6 @@ class VersionConstantValues(NamedTuple): * requires_fcv_tag_list_continuous: List of FCVs that we need to generate a tag for against continuous versions. * fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0. - * eols: List of stable MongoDB versions since v2.0 that have been EOL'd. """ latest: Version @@ -45,7 +44,6 @@ class VersionConstantValues(NamedTuple): requires_fcv_tag_list: List[Version] requires_fcv_tag_list_continuous: List[Version] fcvs_less_than_latest: List[Version] - eols: List[Version] def get_fcv_tag_list(self) -> str: """Get a comma joined string of all the fcv tags.""" @@ -98,10 +96,6 @@ class VersionConstantValues(NamedTuple): last_continuous = self.get_last_continuous_fcv() return f"{base_name}-{last_continuous}" - def get_eols(self) -> List[str]: - """Get EOL'd versions as list of strings.""" - return [version_str(eol) for eol in self.eols] - class MongoVersion(BaseModel): """ @@ -138,14 +132,12 @@ class MongoReleases(BaseModel): * feature_compatibility_version: All FCVs starting with 4.0. * long_term_support_releases: All LTS releases starting with 4.0. - * eol_versions: List of stable MongoDB versions since 2.0 that have been EOL'd. * generate_fcv_lower_bound_override: Extend FCV generation down to the previous value of last LTS. """ feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions") long_term_support_releases: List[str] = Field(alias="longTermSupportReleases") - eol_versions: List[str] = Field(alias="eolVersions") generate_fcv_lower_bound_override: Optional[str] = Field(None, alias="generateFCVLowerBoundOverride") @@ -167,11 +159,7 @@ class MongoReleases(BaseModel): def get_lts_versions(self) -> List[Version]: """Get the Version representation of the lts versions.""" - return [Version(lts) for lts in self.long_term_support_releases] - - def get_eol_versions(self) -> List[Version]: - """Get the Version representation of the EOL versions.""" - return [Version(eol) for eol in self.eol_versions] + return [Version(fcv) for fcv in self.long_term_support_releases] class MultiversionService: @@ -187,12 +175,11 @@ class MultiversionService: self.mongo_version = mongo_version self.mongo_releases = mongo_releases - def calculate_version_constants(self) -> VersionConstantValues: + def calculate_fcv_constants(self) -> FcvConstantValues: """Calculate multiversion constants from data files.""" latest = self.mongo_version.get_version() fcvs = self.mongo_releases.get_fcv_versions() lts = self.mongo_releases.get_lts_versions() - eols = self.mongo_releases.get_eol_versions() lower_bound_override = self.mongo_releases.generate_fcv_lower_bound_override # Highest release less than latest. @@ -213,12 +200,7 @@ class MultiversionService: # All FCVs less than latest. fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)] - return VersionConstantValues( - latest=latest, - last_continuous=last_continuous, - last_lts=last_lts, - requires_fcv_tag_list=requires_fcv_tag_list, - requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, - fcvs_less_than_latest=fcvs_less_than_latest, - eols=eols, - ) + return FcvConstantValues(latest=latest, last_continuous=last_continuous, last_lts=last_lts, + requires_fcv_tag_list=requires_fcv_tag_list, + requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, + fcvs_less_than_latest=fcvs_less_than_latest) diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 6d31e11dc47..0cac4fbc95b 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,27 +2,26 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output -import http -import requests -from retry import retry import structlog -from buildscripts.resmokelib.multiversion.multiversion_service import ( - MongoReleases, MongoVersion, MultiversionService, MONGO_VERSION_YAML, RELEASES_YAML) -from buildscripts.resmokelib.multiversionsetupconstants import \ - USE_EXISTING_RELEASES_FILE - -LAST_LTS = "last_lts" -LAST_CONTINUOUS = "last_continuous" - -# We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date -# with the latest EOL versions. If a "last-continuous" version is EOL, we don't include -# it in the multiversion config and therefore don't test against it. -MASTER_RELEASES_REMOTE_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" +try: + # when running resmoke + from buildscripts.resmokelib.multiversion.multiversion_service import ( + MongoReleases, MongoVersion, MultiversionService) + from buildscripts.resmokelib.multiversionsetupconstants import \ + USE_EXISTING_RELEASES_FILE +except ImportError: + # when running db-contrib-tool + from multiversion.multiversion_service import (MongoReleases, MongoVersion, MultiversionService) + from multiversionsetupconstants import USE_EXISTING_RELEASES_FILE LOGGER = structlog.getLogger(__name__) +# These values must match the include paths for artifacts.tgz in evergreen.yml. +MONGO_VERSION_YAML = ".resmoke_mongo_version.yml" +RELEASES_YAML = ".resmoke_mongo_release_values.yml" + def generate_mongo_version_file(): """Generate the mongo version data file. Should only be called in the root of the mongo directory.""" @@ -38,16 +37,17 @@ def generate_mongo_version_file(): mongo_version_fh.write("mongo_version: " + res) -@retry(tries=5, delay=3) def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. - with open(RELEASES_YAML, "wb") as file: - response = requests.get(MASTER_RELEASES_REMOTE_FILE) - if response.status_code != http.HTTPStatus.OK: - raise RuntimeError("Http response for releases yml file was not 200 but was " + - response.status_code) - file.write(response.content) + releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml") + if not os.path.isfile(releases_yaml_path): + LOGGER.info( + 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.' + .format(releases_yaml_path)) + return + + shutil.copyfile(releases_yaml_path, RELEASES_YAML) def in_git_root_dir(): @@ -86,34 +86,33 @@ multiversion_service = MultiversionService( mongo_releases=MongoReleases.from_yaml_file(RELEASES_YAML), ) -version_constants = multiversion_service.calculate_version_constants() +fcv_constants = multiversion_service.calculate_fcv_constants() -LAST_LTS_BIN_VERSION = version_constants.get_last_lts_fcv() -LAST_CONTINUOUS_BIN_VERSION = version_constants.get_last_continuous_fcv() +LAST_LTS_BIN_VERSION = fcv_constants.get_last_lts_fcv() +LAST_CONTINUOUS_BIN_VERSION = fcv_constants.get_last_continuous_fcv() -LAST_LTS_FCV = version_constants.get_last_lts_fcv() -LAST_CONTINUOUS_FCV = version_constants.get_last_continuous_fcv() -LATEST_FCV = version_constants.get_latest_fcv() +LAST_LTS_FCV = fcv_constants.get_last_lts_fcv() +LAST_CONTINUOUS_FCV = fcv_constants.get_last_continuous_fcv() +LATEST_FCV = fcv_constants.get_latest_fcv() -LAST_CONTINUOUS_MONGO_BINARY = version_constants.build_last_continuous_binary("mongo") -LAST_CONTINUOUS_MONGOD_BINARY = version_constants.build_last_continuous_binary("mongod") -LAST_CONTINUOUS_MONGOS_BINARY = version_constants.build_last_continuous_binary("mongos") +LAST_CONTINUOUS_MONGO_BINARY = fcv_constants.build_last_continuous_binary("mongo") +LAST_CONTINUOUS_MONGOD_BINARY = fcv_constants.build_last_continuous_binary("mongod") +LAST_CONTINUOUS_MONGOS_BINARY = fcv_constants.build_last_continuous_binary("mongos") -LAST_LTS_MONGO_BINARY = version_constants.build_last_lts_binary("mongo") -LAST_LTS_MONGOD_BINARY = version_constants.build_last_lts_binary("mongod") -LAST_LTS_MONGOS_BINARY = version_constants.build_last_lts_binary("mongos") +LAST_LTS_MONGO_BINARY = fcv_constants.build_last_lts_binary("mongo") +LAST_LTS_MONGOD_BINARY = fcv_constants.build_last_lts_binary("mongod") +LAST_LTS_MONGOS_BINARY = fcv_constants.build_last_lts_binary("mongos") -REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() +REQUIRES_FCV_TAG_LATEST = fcv_constants.get_latest_tag() # Generate tags for all FCVS in (lastLTS, latest], or (lowerBoundOverride, latest] if requested. # All multiversion tests should be run with these tags excluded. -REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() +REQUIRES_FCV_TAG = fcv_constants.get_fcv_tag_list() # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] -EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) +EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest]) -OLD_VERSIONS = [ - LAST_LTS -] if LAST_CONTINUOUS_FCV == LAST_LTS_FCV or LAST_CONTINUOUS_FCV in version_constants.get_eols( -) else [LAST_LTS, LAST_CONTINUOUS] +OLD_VERSIONS = ["last_lts"] +if LAST_LTS_FCV != LAST_CONTINUOUS_FCV: + OLD_VERSIONS.append("last_continuous") diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index e450540f293..5f5bb7f3765 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -11,6 +11,7 @@ from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.multiversion import MultiversionPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin +from buildscripts.resmokelib.symbolizer import SymbolizerPlugin from buildscripts.resmokelib.undodb import UndoDbPlugin _PLUGINS = [ @@ -18,6 +19,7 @@ _PLUGINS = [ HangAnalyzerPlugin(), UndoDbPlugin(), PowercyclePlugin(), + SymbolizerPlugin(), GenerateFCVConstantsPlugin(), DiscoveryPlugin(), MultiversionPlugin(), diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index e18099de679..20ff72c2c79 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -175,8 +175,7 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes def generate_multiversion_exclude_tags(self): """Generate multiversion exclude tags file.""" generate_multiversion_exclude_tags.generate_exclude_yaml( - config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, config.EXPANSIONS_FILE, - self._resmoke_logger) + config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, self._resmoke_logger) @staticmethod def _find_suites_by_test(suites): @@ -386,12 +385,25 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ + UNRELIABLE_TAG = _TagInfo( + tag_name="unreliable", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + report_failure_status="silentfail")) + RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) + RETRY_ON_FAILURE_TAG = _TagInfo( + tag_name="retry_on_failure", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, + report_failure_status="silentfail")) + @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -426,8 +438,29 @@ class TestRunnerEvg(TestRunner): combinations = [] - combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) - combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) + if config.EVERGREEN_PATCH_BUILD: + combinations.append(("unreliable and resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("unreliable and not resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) + combinations.append(("reliable and resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("reliable and not resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + else: + combinations.append(("retry on failure and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("retry on failure and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + combinations.append(("run once and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("run once and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) return combinations @@ -707,19 +740,15 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", - help=("Do not run any tests tagged with enabled feature flags." - " This argument has precedence over --runAllFeatureFlagTests" - "; used for multiversion suites")) + "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", + action="store_true", help= + "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" + ) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") - parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", - action="store", metavar="FILE", - help="The path to a file with feature flags, delimited by newlines.") - parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -761,11 +790,6 @@ class RunPlugin(PluginInterface): metavar="ON|OFF", help=("Enable or disable majority read concern support." " Defaults to %(default)s.")) - mongodb_server_options.add_argument( - "--enableEnterpriseTests", action="store", dest="enable_enterprise_tests", default="on", - choices=("on", "off"), metavar="ON|OFF", - help=("Enable or disable enterprise tests. Defaults to 'on'.")) - mongodb_server_options.add_argument("--flowControl", action="store", dest="flow_control", choices=("on", "off"), metavar="ON|OFF", help=("Enable or disable flow control.")) @@ -855,6 +879,13 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( + "--reportFailureStatus", action="store", dest="report_failure_status", + choices=("fail", "silentfail"), metavar="STATUS", + help="Controls if the test failure status should be reported as failed" + " or be silently ignored (STATUS=silentfail). Dynamic test failures will" + " never be silently ignored. Defaults to STATUS=%(default)s.") + + internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") @@ -952,9 +983,6 @@ class RunPlugin(PluginInterface): evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") - evergreen_options.add_argument("--taskWorkDir", dest="work_dir", metavar="TASK_WORK_DIR", - help="Sets the working directory of the task.") - benchmark_options = parser.add_argument_group( title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests") @@ -1068,6 +1096,15 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) + # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on + # all_feature_flags.txt which may not exist in the local dev environment. + run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) + if run_all_feature_flag_tests is not None: + setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) + del parsed_args.run_all_feature_flag_tests + + del parsed_args.run_all_feature_flags_no_tests + # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py index d5c904bb2c3..98be3db0fa8 100755 --- a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py +++ b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py @@ -5,8 +5,6 @@ import re import tempfile from collections import defaultdict from subprocess import check_output -from typing import Optional -from github import GithubIntegration import requests @@ -15,35 +13,13 @@ from buildscripts.resmokelib.config import MultiversionOptions from buildscripts.resmokelib.core.programs import get_path_env_var from buildscripts.resmokelib.utils import is_windows from buildscripts.util.fileops import read_yaml_file -from buildscripts.util.read_config import read_config_file BACKPORT_REQUIRED_TAG = "backport_required_multiversion" # The directory in which BACKPORTS_REQUIRED_FILE resides. ETC_DIR = "etc" BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml" -BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo" - - -def get_installation_access_token(app_id: int, private_key: str, - installation_id: int) -> Optional[str]: # noqa: D406,D407,D413 - """ - Obtain an installation access token using JWT. - - Args: - - app_id: The application ID for GitHub App. - - private_key: The private key associated with the GitHub App. - - installation_id: The installation ID of the GitHub App for a particular account. - - Returns: - - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token. - """ - integration = GithubIntegration(app_id, private_key) - auth = integration.get_access_token(installation_id) - if auth: - return auth.token - else: - raise Exception("Error obtaining installation token") +BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo" def get_backports_required_hash_for_shell_version(mongo_shell_path=None): @@ -75,24 +51,10 @@ def get_backports_required_hash_for_shell_version(mongo_shell_path=None): f"Could not find a valid commit hash from the {mongo_shell_path} mongo binary.") -def get_old_yaml(commit_hash, expansions_file): +def get_old_yaml(commit_hash): """Download BACKPORTS_REQUIRED_FILE from the old commit and return the yaml.""" - - if not os.path.exists(expansions_file): - raise FileNotFoundError(f"The specified file does not exist: {expansions_file}") - expansions = read_config_file(expansions_file) - - # Obtain installation access tokens using app credentials - access_token_10gen_mongo = get_installation_access_token( - expansions["app_id_10gen_mongo"], expansions["private_key_10gen_mongo"], - expansions["installation_id_10gen_mongo"]) - response = requests.get( - f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}', - headers={ - 'Authorization': f'token {access_token_10gen_mongo}', - }) - + f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}') # If the response was successful, no exception will be raised. response.raise_for_status() @@ -106,8 +68,7 @@ def get_old_yaml(commit_hash, expansions_file): return backports_required_old -def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: str, - logger: logging.Logger) -> None: +def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Logger) -> None: """ Create a tag file associating multiversion tests to tags for exclusion. @@ -138,7 +99,7 @@ def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: st # Get the yaml contents from the old commit. logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}") - backports_required_old = get_old_yaml(old_version_commit_hash, expansions_file) + backports_required_old = get_old_yaml(old_version_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 5194c097f62..b9a06e0d742 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -19,8 +19,6 @@ from buildscripts.resmokelib import utils from buildscripts.resmokelib.utils import globstar from buildscripts.resmokelib.utils import jscomment -ENTERPRISE_TEST_DIR = os.path.normpath("src/mongo/db/modules/enterprise/jstests") - ######################## # Test file explorer # ######################## @@ -217,13 +215,6 @@ class _TestList(object): format(path))) self._filtered.discard(path) - def filter_enterprise_tests(self): - """Exclude tests that start with the enterprise module directory from the test list.""" - self._filtered = { - test - for test in self._filtered if not os.path.normpath(test).startswith(ENTERPRISE_TEST_DIR) - } - def match_tag_expression(self, tag_expression, get_tags): """Filter the test list to only include tests that match the tag expression. @@ -462,9 +453,6 @@ class _Selector(object): # 5. Apply the include files last with force=True to take precedence over the tags. if self._tests_are_files and selector_config.include_files: test_list.include_files(selector_config.include_files, force=True) - # 6: Apply the enterprise tests filter - if self.get_enterprise_tests_status() == "off": - test_list.filter_enterprise_tests() return self.sort_tests(*test_list.get_tests()) @@ -480,11 +468,6 @@ class _Selector(object): """Retrieve the tags associated with the give test file.""" return [] - @staticmethod - def get_enterprise_tests_status() -> str: - """Get the status of enterprise tests from the configuration.""" - return config.ENABLE_ENTERPRISE_TESTS - class _JSTestSelectorConfig(_SelectorConfig): """_SelectorConfig subclass for JavaScript tests.""" diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py index 609ea8a6e3e..5df67812d06 100644 --- a/buildscripts/resmokelib/sighandler.py +++ b/buildscripts/resmokelib/sighandler.py @@ -10,10 +10,10 @@ import traceback import psutil -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib import reportfile from buildscripts.resmokelib import testing from buildscripts.resmokelib import config +from buildscripts.resmokelib.hang_analyzer import hang_analyzer from buildscripts.resmokelib import parser _IS_WINDOWS = (sys.platform == "win32") @@ -32,8 +32,8 @@ def register(logger, suites, start_time): log suite summaries. """ - HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to SIGUSR1 signal" + _dump_and_log(header_msg) def _handle_set_event(event_handle): @@ -53,7 +53,6 @@ def register(logger, suites, start_time): except win32event.error as err: logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) else: - HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to signal from win32event.SetEvent" _dump_and_log(header_msg) @@ -160,26 +159,4 @@ def _analyze_pids(logger, pids): if not os.getenv('ASAN_OPTIONS'): hang_analyzer_args.append('-c') _hang_analyzer = parser.parse_command_line(hang_analyzer_args, logger=logger) - - # Evergreen has a 15 minute timeout for task timeout commands - # Limit the hang analyzer to 12 minutes so there is time for other tasks. - hang_analyzer_hard_timeout = None - if config.EVERGREEN_TASK_ID: - hang_analyzer_hard_timeout = 60 * 12 - logger.info( - "Limit the resmoke invoked hang analyzer to 12 minutes so there is time for resmoke to finish up." - ) - - hang_analyzer_thread = threading.Thread(target=_hang_analyzer.execute, daemon=True) - hang_analyzer_thread.start() - hang_analyzer_thread.join(hang_analyzer_hard_timeout) - - if hang_analyzer_thread.is_alive(): - logger.warning( - "Resmoke invoked hang analyzer thread did not finish, but will continue running in the background. The thread may be disruputed and may show extraneous output." - ) - logger.warning("Cleaning up resmoke child processes so that resmoke can fail gracefully.") - _hang_analyzer.kill_rogue_processes() - - else: - logger.info("Done running resmoke invoked hang analyzer thread.") + _hang_analyzer.execute() diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 55c23923ecc..394c025d510 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -4,8 +4,6 @@ import os import os.path import time import shutil -import uuid - import yaml import pymongo @@ -25,9 +23,6 @@ class MongoDFixture(interface.Fixture): self.mongod_options = self.fixturelib.make_historic( self.fixturelib.default_if_none(mongod_options, {})) - if "set_parameters" not in self.mongod_options: - self.mongod_options["set_parameters"] = {} - if add_feature_flags: for ff in self.config.ENABLED_FEATURE_FLAGS: self.mongod_options["set_parameters"][ff] = "true" @@ -57,11 +52,6 @@ class MongoDFixture(interface.Fixture): self.port = port or fixturelib.get_next_port(job_num) self.mongod_options["port"] = self.port - # Always log backtraces to a file in the dbpath in our testing. - backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(), - uuid.uuid4().hex + ".stacktrace") - self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name - def setup(self): """Set up the mongod.""" if not self.preserve_dbpath and os.path.lexists(self._dbpath): @@ -71,8 +61,7 @@ class MongoDFixture(interface.Fixture): launcher = MongodLauncher(self.fixturelib) # Second return val is the port, which we ignore because we explicitly created the port above. - # The port is used to set other mongod_option's here: - # https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 + # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num, executable=self.mongod_executable, mongod_options=self.mongod_options) @@ -179,7 +168,7 @@ class MongoDFixture(interface.Fixture): def get_driver_connection_url(self): """Return the driver connection URL.""" - return "mongodb://" + self.get_internal_connection_string() + "/?directConnection=true" + return "mongodb://" + self.get_internal_connection_string() # The below parameters define the default 'logComponentVerbosity' object passed to mongod processes @@ -191,16 +180,15 @@ class MongoDFixture(interface.Fixture): # The default verbosity setting for any tests that are not started with an Evergreen task id. This # will apply to any tests run locally. DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, - "transaction": 4, "tenantMigration": 4 + "replication": {"rollback": 2}, "sharding": {"migration": 2}, "transaction": 4, + "tenantMigration": 4 } # The default verbosity setting for any mongod processes running in Evergreen i.e. started with an # Evergreen task id. DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, - "rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, - "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 + "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, "rollback": 2}, + "sharding": {"migration": 2}, "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 } @@ -254,15 +242,6 @@ class MongodLauncher(object): if "shardsvr" in mongod_options and "orphanCleanupDelaySecs" not in suite_set_parameters: suite_set_parameters["orphanCleanupDelaySecs"] = 1 - # receiveChunkWaitForRangeDeleterTimeoutMS controls the amount of time an incoming migration - # will wait for an intersecting range with data in it to be cleared up before failing. The - # default is 10 seconds, but in some slower variants this is not enough time for the range - # deleter to finish so we increase it here to 90 seconds. Setting a value for this parameter - # in the .yml file overrides this. - if (("shardsvr" in mongod_options or "configsvr" in mongod_options) - and "receiveChunkWaitForRangeDeleterTimeoutMS" not in suite_set_parameters): - suite_set_parameters["receiveChunkWaitForRangeDeleterTimeoutMS"] = 90000 - # The LogicalSessionCache does automatic background refreshes in the server. This is # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off # background refreshing for tests. Set in the .yml file to override this. @@ -313,13 +292,6 @@ class MongodLauncher(object): "mode": "alwaysOn", "data": {"numTickets": self.config.FLOW_CONTROL_TICKETS} } - # The internalQueryForceClassicEngine parameter was renamed to - # internalQueryFrameworkControl starting in version 6.1. - if "internalQueryFrameworkControl" in suite_set_parameters and suite_set_parameters[ - "internalQueryFrameworkControl"] == "forceClassicEngine": - del suite_set_parameters["internalQueryFrameworkControl"] - suite_set_parameters["internalQueryForceClassicEngine"] = True - _add_testing_set_parameters(suite_set_parameters) shortcut_opts = { @@ -396,7 +368,3 @@ def _add_testing_set_parameters(suite_set_parameters): """ suite_set_parameters.setdefault("testingDiagnosticsEnabled", True) suite_set_parameters.setdefault("enableTestCommands", True) - # The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up. - # Set it to true for now as a placeholder that will error if no further processing is done. - # The placeholder is needed so older versions don't have this option won't have this value set. - suite_set_parameters.setdefault("backtraceLogFile", True) diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index 23ebfda8de3..38909056ce3 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -6,7 +6,6 @@ import threading from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib import utils -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib.utils import globstar @@ -106,9 +105,5 @@ class HookTestArchival(object): else: logger.info("Archive succeeded for %s: %s", test_name, message) - if HANG_ANALYZER_CALLED.is_set(): - logger.info("Hang Analyzer has been called. Fixtures will not be restarted.") - raise errors.StopExecution( - "Hang analyzer has been called. Stopping further execution of tests.") - elif not manager.setup_fixture(logger): + if not manager.setup_fixture(logger): raise errors.StopExecution("Error while restarting test fixtures after archiving.") diff --git a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py index c45f93192eb..d392c6ae0b8 100644 --- a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py +++ b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py @@ -4,70 +4,70 @@ This hook runs continuously, but the run_aggregate_metrics_background.js file it internally sleep for 1 second between runs. """ -import random -import pymongo +import os.path -from buildscripts.resmokelib.testing.hooks.bghook import BGHook +from buildscripts.resmokelib import errors +from buildscripts.resmokelib.testing.hooks import jsfile +from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase -def verify_metrics(doc): - """Checks whether the output from $operatiomMetrics has the schema we expect.""" +class AggregateResourceConsumptionMetricsInBackground(jsfile.JSHook): + """A hook to run $operationMetrics stage in the background.""" - top_level_fields = [ - "docBytesWritten", "docUnitsWritten", "idxEntryBytesWritten", "idxEntryUnitsWritten", - "totalUnitsWritten", "cpuNanos", "db", "primaryMetrics", "secondaryMetrics" - ] - read_fields = [ - "docBytesRead", "docUnitsRead", "idxEntryBytesRead", "idxEntryUnitsRead", "keysSorted", - "docUnitsReturned" - ] + IS_BACKGROUND = True - for key in top_level_fields: - assert key in doc, ("The metrics output is missing the property: " + key) + def __init__(self, hook_logger, fixture, shell_options=None): + """Initialize AggregateResourceConsumptionMetricsInBackground.""" + description = "Run background $operationMetrics on all mongods while a test is running" + js_filename = os.path.join("jstests", "hooks", "run_aggregate_metrics_background.js") + jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description, + shell_options=shell_options) + self._background_job = None - primary_metrics = doc["primaryMetrics"] - for key in read_fields: - assert key in primary_metrics, ( - "The metrics output is missing the property: primaryMetrics." + key) + def before_suite(self, test_report): + """Start the background thread.""" + self._background_job = _BackgroundJob("AggregateResourceConsumptionMetricsInBackground") + self.logger.info("Starting the background aggregate metrics thread.") + self._background_job.start() - secondary_metrics = doc["secondaryMetrics"] - for key in read_fields: - assert key in secondary_metrics, ( - "The metrics output is missing the property: secondaryMetrics." + key) + def after_suite(self, test_report, teardown_flag=None): + """Signal the background aggregate metrics thread to exit, and wait until it does.""" + if self._background_job is None: + return + self.logger.info("Stopping the background aggregate metrics thread.") + self._background_job.stop() -class AggregateResourceConsumptionMetricsInBackground(BGHook): - """A hook to run $operationMetrics stage in the background.""" + def before_test(self, test, test_report): + """Instruct the background aggregate metrics thread to run while 'test' is also running.""" + if self._background_job is None: + return - def __init__(self, hook_logger, fixture): - """Initialize AggregateResourceConsumptionMetricsInBackground.""" + hook_test_case = _ContinuousDynamicJSTestCase.create_before_test( + test.logger, test, self, self._js_filename, self._shell_options) + hook_test_case.configure(self.fixture) - description = "Run background $operationMetrics on all mongods while a test is running" - super().__init__(hook_logger, fixture, description, tests_per_cycle=None, - loop_delay_ms=1000) - - def run_action(self): - """Collects $operationMetrics on all non-arbiter nodes in the fixture.""" - for node_info in self.fixture.get_node_info(): - conn = pymongo.MongoClient(port=node_info.port) - # Filter out arbiters. - if "arbiterOnly" in conn.admin.command({"isMaster": 1}): - self.logger.info( - "Skipping background aggregation against test node: %s because it is an " + - "arbiter and has no data.", node_info.full_name) - return - - # Clear the metrics about 10% of the time. - clear_metrics = random.random() < 0.1 - self.logger.info("Running $operationMetrics with {clearMetrics: %s} on host: %s", - clear_metrics, node_info.full_name) - with conn.admin.aggregate( - [{"$operationMetrics": {"clearMetrics": clear_metrics}}]) as cursor: - for doc in cursor: - try: - verify_metrics(doc) - except: - self.logger.info( - "caught exception while verifying that all expected fields are in the" + - " metrics output: ", doc) - raise + self.logger.info("Resuming the background aggregate metrics thread.") + self._background_job.resume(hook_test_case, test_report) + + def after_test(self, test, test_report): # noqa: D205,D400 + """Instruct the background aggregate metrics thread to stop running now that 'test' has + finished running. + """ + if self._background_job is None: + return + + self.logger.info("Pausing the background aggregate metrics thread.") + self._background_job.pause() + + if self._background_job.exc_info is not None: + if isinstance(self._background_job.exc_info[1], errors.TestFailure): + # If the mongo shell process running the JavaScript file exited with a non-zero + # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's + # test execution to stop. + raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) + else: + self.logger.error( + "Encountered an error inside the background aggregate metrics thread.", + exc_info=self._background_job.exc_info) + raise self._background_job.exc_info[1] diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py index cd775aabd4a..92dbb5437a4 100644 --- a/buildscripts/resmokelib/testing/hooks/background_job.py +++ b/buildscripts/resmokelib/testing/hooks/background_job.py @@ -3,7 +3,6 @@ import sys import threading -from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing.hooks import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/bghook.py b/buildscripts/resmokelib/testing/hooks/bghook.py index 1b3fed3b104..b61f37b4e2d 100644 --- a/buildscripts/resmokelib/testing/hooks/bghook.py +++ b/buildscripts/resmokelib/testing/hooks/bghook.py @@ -13,13 +13,11 @@ class BGJob(threading.Thread): BGJob will call 'run_action' without any delay and expects the 'run_action' function to add some form of delay. """ - def __init__(self, hook, loop_delay_ms=None): + def __init__(self, hook): """Initialize the background job.""" threading.Thread.__init__(self, name=f"BGJob-{hook.__class__.__name__}") - self._loop_delay_ms = loop_delay_ms self.daemon = True self._hook = hook - self._interrupt_event = threading.Event() self.__is_alive = True self.err = None @@ -31,14 +29,6 @@ class BGJob(threading.Thread): try: self._hook.run_action() - if self._loop_delay_ms is not None: - # The configured loop delay asked us to wait before running the action again. Do - # that wait, but listen to see if we finish running the test or are killed in - # the meantime. - interrupted = self._interrupt_event.wait(self._loop_delay_ms / 1000.0) - if interrupted: - self._hook.logger.info("interrupted") - break except Exception as err: # pylint: disable=broad-except self._hook.logger.error("Background thread caught exception: %s.", err) self.err = err @@ -47,7 +37,6 @@ class BGJob(threading.Thread): def kill(self): """Kill the background job.""" self.__is_alive = False - self._interrupt_event.set() class BGHook(interface.Hook): @@ -57,13 +46,8 @@ class BGHook(interface.Hook): # By default, we continuously run the background hook for the duration of the suite. DEFAULT_TESTS_PER_CYCLE = math.inf - def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None, loop_delay_ms=None): - """ - Initialize the background hook. - - 'tests_per_cycle' or 'loop_delay_ms' can be used to configure how often the background job - is restarted, and how often run_action() is called, respectively. - """ + def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None): + """Initialize the background hook.""" interface.Hook.__init__(self, hook_logger, fixture, desc) self.logger = hook_logger @@ -73,21 +57,15 @@ class BGHook(interface.Hook): self._test_num = 0 # The number of tests we execute before restarting the background hook. self._tests_per_cycle = self.DEFAULT_TESTS_PER_CYCLE if tests_per_cycle is None else tests_per_cycle - self._loop_delay_ms = loop_delay_ms def run_action(self): - """ - Perform an action. This function will be called continuously in the BgJob. - - If a sleep_delay_ms was given, that many milliseconds of sleep will happen between each - invocation. - """ + """Perform an action. This function will be called continuously in the BgJob.""" raise NotImplementedError def before_suite(self, test_report): """Start the background thread.""" self.logger.info("Starting the background thread.") - self._background_job = BGJob(self, self._loop_delay_ms) + self._background_job = BGJob(self) self._background_job.start() def after_suite(self, test_report, teardown_flag=None): @@ -108,7 +86,7 @@ class BGHook(interface.Hook): return self.logger.info("Restarting the background thread.") - self._background_job = BGJob(self, self._loop_delay_ms) + self._background_job = BGJob(self) self._background_job.start() def after_test(self, test, test_report): diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index 5278819da8f..b0ff858c5dc 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -276,15 +276,16 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): secondary.await_ready() client = secondary.mongo_client() + minvalid_doc = client.local["replset.minvalid"].find_one() oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one() recovery_timestamp_res = client.admin.command("replSetTest", getLastStableRecoveryTimestamp=True) latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural", pymongo.DESCENDING)]) - self.logger.info("Checking replication invariants. oplogTruncateAfterPoint: {}," + self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {}," " stable recovery timestamp: {}, latest oplog doc: {}".format( - oplog_truncate_after_doc, recovery_timestamp_res, + minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res, latest_oplog_doc)) null_ts = bson.Timestamp(0, 0) @@ -298,6 +299,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): raise errors.ServerFailure( "Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc)) + # The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default + # it to null. + oplog_truncate_after_ts = null_ts + if oplog_truncate_after_doc is not None: + oplog_truncate_after_ts = oplog_truncate_after_doc.get( + "oplogTruncateAfterPoint", null_ts) + # The "lastStableRecoveryTimestamp" field is present if the storage engine supports # "recover to a timestamp". If it's a null timestamp on a durable storage engine, that # means we do not yet have a stable checkpoint timestamp and must be restarting at the @@ -320,6 +328,94 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): recovery_timestamp, latest_oplog_entry_ts, recovery_timestamp_res, latest_oplog_doc)) + if minvalid_doc is not None: + applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts) + minvalid_ts = minvalid_doc.get("ts", null_ts) + + # The "appliedThrough" value should always equal the "last stable recovery + # timestamp", AKA the stable checkpoint for durable engines, on server restart. + # + # The written "appliedThrough" time is updated with the latest timestamp at the end + # of each batch application, and batch boundaries are the only valid stable + # timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must + # equal the checkpoint timestamp, because any stable timestamp that the checkpoint + # could use includes an equal persisted appliedThrough timestamp. + if (recovery_timestamp != null_ts and applied_through_ts != null_ts + and (not recovery_timestamp == applied_through_ts)): + raise errors.ServerFailure( + "The condition last stable recovery timestamp ({}) == appliedThrough ({})" + " doesn't hold: minValid document={}," + " getLastStableRecoveryTimestamp result={}, last oplog entry={}".format( + recovery_timestamp, applied_through_ts, minvalid_doc, + recovery_timestamp_res, latest_oplog_doc)) + + if applied_through_ts == null_ts: + # We clear "appliedThrough" to represent having applied through the top of the + # oplog in PRIMARY state or immediately after "rollback via refetch". + # If we are using a storage engine that supports "recover to a timestamp," + # then we will have a "last stable recovery timestamp" and we should use that + # as our "appliedThrough" (similarly to why we assert their equality above). + # If both are null, then we are in PRIMARY state on a storage engine that does + # not support "recover to a timestamp" or in RECOVERING immediately after + # "rollback via refetch". Since we do not update "minValid" in PRIMARY state, + # we leave "appliedThrough" as null so that the invariants below hold, rather + # than substituting the latest oplog entry for the "appliedThrough" value. + applied_through_ts = recovery_timestamp + + if minvalid_ts == null_ts: + # The server treats the "ts" field in the minValid document as missing when its + # value is the null timestamp. + minvalid_ts = applied_through_ts + + if latest_oplog_entry_ts == null_ts: + # If the oplog is empty, we treat the "minValid" as the latest oplog entry. + latest_oplog_entry_ts = minvalid_ts + + if oplog_truncate_after_ts == null_ts: + # The server treats the "oplogTruncateAfterPoint" field as missing when its + # value is the null timestamp. When it is null, the oplog is complete and + # should not be truncated, so it is effectively the top of the oplog. + oplog_truncate_after_ts = latest_oplog_entry_ts + + # Check the ordering invariants before the secondary has reconciled the end of + # its oplog. + # The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of + # oplog entries before they are written to the oplog. Thus, it can be ahead + # of the top of the oplog before any oplog entries are written, and behind it + # after some are written. Thus, we cannot compare it to the top of the oplog. + + # appliedThrough <= minValid + # appliedThrough represents the end of the previous batch, so it is always the + # earliest. + if applied_through_ts > minvalid_ts: + raise errors.ServerFailure( + "The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid" + " document={}, latest oplog entry={}".format( + applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc)) + + # minValid <= oplogTruncateAfterPoint + # This is true because this hook is never run after a rollback. Thus, we only + # move "minValid" to the end of each batch after the batch is written to the oplog. + # We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from + # the end of the previous batch to the end of the current batch. Thus "minValid" + # must be less than or equal to the "oplogTruncateAfterPoint". + if minvalid_ts > oplog_truncate_after_ts: + raise errors.ServerFailure( + "The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't" + " hold: minValid document={}, oplogTruncateAfterPoint document={}," + " latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts, + minvalid_doc, oplog_truncate_after_doc, + latest_oplog_doc)) + + # minvalid <= latest oplog entry + # "minValid" is set to the end of a batch after the batch is written to the oplog. + # Thus it is always less than or equal to the top of the oplog. + if minvalid_ts > latest_oplog_entry_ts: + raise errors.ServerFailure( + "The condition minValid <= top of oplog ({} <= {}) doesn't" + " hold: minValid document={}, latest oplog entry={}".format( + minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc)) + try: secondary.teardown() except errors.ServerFailure: diff --git a/buildscripts/resmokelib/testing/hooks/run_query_stats.py b/buildscripts/resmokelib/testing/hooks/run_query_stats.py deleted file mode 100644 index 736c5e261c7..00000000000 --- a/buildscripts/resmokelib/testing/hooks/run_query_stats.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Test hook for verifying $queryStats collects expected metrics and can redact query shapes. - -This runs in the background as other tests are ongoing. -""" - -from bson import binary -import pymongo.errors -from buildscripts.resmokelib.testing.hooks.interface import Hook - -QUERY_STATS_NOT_ENABLED_CODES = [224, 7373500, 6579000] - - -class RunQueryStats(Hook): - """Runs $queryStats after every test, and clears the query stats store before every test.""" - - IS_BACKGROUND = False - - def __init__(self, hook_logger, fixture, allow_feature_not_supported=False): - """Initialize the RunQueryStats hook. - - Args: - hook_logger: the logger instance for this hook. - fixture: the target fixture (replica sets or a sharded cluster). - allow_feature_not_supported: absorb 'QueryFeatureNotAllowed' errors when calling - $queryStats. This is to support fuzzer suites that may manipulate the FCV. - """ - description = "Read query stats data after each test." - super().__init__(hook_logger, fixture, description) - self.client = self.fixture.mongo_client() - self.hmac_key = binary.Binary(("0" * 32).encode('utf-8'), 8) - self.allow_feature_not_supported = allow_feature_not_supported - - def verify_query_stats(self, querystats_spec): - """Verify a $queryStats call has all the right properties.""" - query_stats_pipeline = [ - {"$queryStats": querystats_spec}, - # SERVER-90921: The pymongo version we use on this branch has trouble parsing invalid - # DBRefs, which can be produced by the 'key' field. We'll overwrite that field with a - # dummy one, since the contents aren't important for this test/check. - {"$set": {"key": "Redacted due to issues with DBRefs"}} - ] - try: - with self.client.admin.aggregate(query_stats_pipeline) as cursor: - nreturned = 0 - for operation in cursor: - assert "key" in operation - assert "metrics" in operation - assert "asOf" in operation - nreturned += 1 - self.logger.info("Found %d query stats entries.", nreturned) - except pymongo.errors.OperationFailure as err: - if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: - self.logger.info("Encountered an error while running $queryStats. " - "$queryStats will not be run for this test.") - else: - raise err - - def after_test(self, test, test_report): - """After the test, make sure we can ingest the query stats, with and without hmac.""" - self.verify_query_stats({}) - self.verify_query_stats( - {"transformIdentifiers": {"algorithm": "hmac-sha-256", "hmacKey": self.hmac_key}}) - - # Log the number of evictions we encountered. - server_status = self.client.admin.command({"serverStatus": 1}) - num_evicted_entries = server_status["metrics"]["queryStats"]["numEvicted"] - if num_evicted_entries > 0: - self.logger.info("Evicted %d query stats entries during test execution.", - num_evicted_entries) - - def before_test(self, test, test_report): - """Before the test, reset the contents of the query stats store.""" - try: - # Clear out all existing entries, then reset the size cap. - self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="0%") - self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="1%") - except pymongo.errors.OperationFailure as err: - if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: - self.logger.info("Encountered an error while configuring the query stats store. " - "Query stats will not be collected for this test.") - else: - raise err diff --git a/buildscripts/resmokelib/testing/hooks/simulate_crash.py b/buildscripts/resmokelib/testing/hooks/simulate_crash.py index 68407a66192..c7f8bc4026c 100644 --- a/buildscripts/resmokelib/testing/hooks/simulate_crash.py +++ b/buildscripts/resmokelib/testing/hooks/simulate_crash.py @@ -94,14 +94,7 @@ class SimulateCrash(bghook.BGHook): rel = fqfn[len(root):] os.makedirs(new_root + "/journal", exist_ok=True) out_fd = os.open(new_root + rel, os.O_WRONLY | os.O_CREAT) - - total_bytes_sent = 0 - while total_bytes_sent < in_bytes: - bytes_sent = os.sendfile(out_fd, in_fd, total_bytes_sent, in_bytes - total_bytes_sent) - if bytes_sent == 0: - raise ValueError("Unexpectedly reached EOF copying file") - total_bytes_sent += bytes_sent - + os.sendfile(out_fd, in_fd, 0, in_bytes) os.close(out_fd) os.close(in_fd) diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index 8fb565cd1d8..c09602baee8 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -363,7 +363,7 @@ class FileBasedStepdownLifecycle(object): # We remove the "permitted" file to revoke permission for the stepdown thread to continue # performing stepdowns. - fs.remove_if_exists(self.__stepdown_files.permitted) + os.remove(self.__stepdown_files.permitted) class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-attributes @@ -627,10 +627,6 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at break except pymongo.errors.NotMasterError: pass - except pymongo.errors.OperationFailure as ex: - if ex.code == 166: # CommandNotSupportedOnView - # listCollections return also views and collStats is not supported on views - break retarget_time = time.time() - start_time if retarget_time >= 60: raise RuntimeError( diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index e2655e8bf81..38fe409b656 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -4,14 +4,12 @@ This is used to support additional test status and timing information for the re """ import copy -import os import threading import time import unittest from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import logging -from buildscripts.resmokelib.testing.symbolizer_service import ResmokeSymbolizer # pylint: disable=attribute-defined-outside-init @@ -56,9 +54,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for report in reports: if not isinstance(report, TestReport): - raise TypeError( - f"reports must be a list of TestReport instances, current report is {type(report)}" - ) + raise TypeError("reports must be a list of TestReport instances") with report._lock: # pylint: disable=protected-access for test_info in report.test_infos: @@ -110,7 +106,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr unittest.TestResult.startTest(self, test) test_info = TestInfo(test.id(), test.test_name, test.dynamic) - test_info.group_id = f"job{self.job_num}" basename = test.basename() command = test.as_command() @@ -129,53 +124,40 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test.basename(), command, test.logger, self.job_num, test.id(), self.job_logger) - test_info.log_info = { - "log_name": logging.loggers.get_evergreen_log_name(self.job_num, test.id()), - "logs_to_merge": [logging.loggers.get_evergreen_log_name(self.job_num)], - "rendering_type": "resmoke", "version": 0 - } test_info.url_endpoint = url_endpoint if self.logging_prefix is not None: test_logger.info(self.logging_prefix) # Set job_num in test. test.job_num = self.job_num + test.override_logger(test_logger) test_info.start_time = time.time() def stopTest(self, test): # pylint: disable=invalid-name """Call after 'test' has run.""" - try: - # check if there are stacktrace files, if so, invoke the symbolizer here. - # log symbolized output to test.logger.info() - symbolizer = ResmokeSymbolizer() - symbolizer.symbolize_test_logs(test) - - unittest.TestResult.stopTest(self, test) - - with self._lock: - test_info = self.find_test_info(test) - test_info.end_time = time.time() - test_status = "no failures detected" if test_info.status == "pass" else "failed" - - time_taken = test_info.end_time - test_info.start_time - self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, - test_status) - - finally: - # This is a failsafe. In the event that 'stopTest' fails, - # any rogue logger handlers will be removed from this test. - # If not cleaned up, these will trigger 'setup failures' -- - # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE. - for handler in test.logger.handlers: - # We ignore the cancellation token returned by close_later() since we always want the - # logs to eventually get flushed. - logging.flush.close_later(handler) - - # Restore the original logger for the test. - test.reset_logger() - - def addError(self, test, err): + unittest.TestResult.stopTest(self, test) + + with self._lock: + test_info = self.find_test_info(test) + test_info.end_time = time.time() + test_status = "no failures detected" if test_info.status == "pass" else "failed" + + time_taken = test_info.end_time - test_info.start_time + self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, + test_status) + + # Asynchronously closes the buildlogger test handler to avoid having too many threads open + # on 32-bit systems. + for handler in test.logger.handlers: + # We ignore the cancellation token returned by close_later() since we always want the + # logs to eventually get flushed. + logging.flush.close_later(handler) + + # Restore the original logger for the test. + test.reset_logger() + + def addError(self, test, err): # pylint: disable=invalid-name """Call when a non-failureException was raised during the execution of 'test'.""" unittest.TestResult.addError(self, test, err) @@ -219,7 +201,12 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = self.find_test_info(test) test_info.status = "fail" - test_info.evergreen_status = "fail" + if test_info.dynamic: + # Dynamic tests are used for data consistency checks, so the failures are never + # silenced. + test_info.evergreen_status = "fail" + else: + test_info.evergreen_status = self.suite_options.report_failure_status test_info.return_code = test.return_code def setFailure(self, test, return_code=1): # pylint: disable=invalid-name @@ -231,7 +218,12 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr raise ValueError("stopTest was not called on %s" % (test.basename())) test_info.status = "fail" - test_info.evergreen_status = "fail" + if test_info.dynamic: + # Dynamic tests are used for data consistency checks, so the failures are never + # silenced. + test_info.evergreen_status = "fail" + else: + test_info.evergreen_status = self.suite_options.report_failure_status test_info.return_code = return_code # Recompute number of success, failures, and errors. @@ -294,18 +286,19 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for test_info in self.test_infos: result = { "test_file": test_info.test_file, - "group_id": test_info.group_id, "status": test_info.evergreen_status, "exit_code": test_info.return_code, "start": test_info.start_time, "end": test_info.end_time, "elapsed": test_info.end_time - test_info.start_time, - "log_info": test_info.log_info, } if test_info.display_test_name is not None: result["display_test_name"] = test_info.display_test_name + if test_info.group_id is not None: + result["group_id"] = test_info.group_id + if test_info.url_endpoint is not None: result["url"] = test_info.url_endpoint result["url_raw"] = test_info.url_endpoint + "?raw=1" @@ -335,7 +328,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = TestInfo(test_file, test_file, is_dynamic) test_info.display_test_name = result.get("display_test_name") test_info.group_id = result.get("group_id") - test_info.log_info = result.get("log_info") test_info.url_endpoint = result.get("url") test_info.status = result["status"] test_info.evergreen_status = test_info.status @@ -389,16 +381,15 @@ class TestInfo(object): # pylint: disable=too-many-instance-attributes self.test_id = test_id self.test_file = test_file - self.group_id = None self.display_test_name = None self.dynamic = dynamic + self.group_id = None self.start_time = None self.end_time = None self.status = None self.evergreen_status = None self.return_code = None - self.log_info = None self.url_endpoint = None diff --git a/buildscripts/resmokelib/testing/symbolizer_service.py b/buildscripts/resmokelib/testing/symbolizer_service.py deleted file mode 100644 index 9e78b0dd6b7..00000000000 --- a/buildscripts/resmokelib/testing/symbolizer_service.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Symbolize stacktraces inside test logs.""" -from __future__ import annotations - -import os -import subprocess -import sys -import time -from datetime import timedelta -from threading import Lock - -from typing import List, Optional, NamedTuple - -from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED -from buildscripts.resmokelib.testing.testcases.interface import TestCase - -# This lock prevents different resmoke jobs from symbolizing stacktraces concurrently, -# which includes downloading the debug symbols, that can be reused by other resmoke jobs -_lock = Lock() - -STACKTRACE_FILE_EXTENSION = ".stacktrace" -SYMBOLIZE_RETRY_TIMEOUT_SECS = timedelta(minutes=4).total_seconds() - - -class ResmokeSymbolizerConfig(NamedTuple): - """ - Resmoke symbolizer config. - - * evg_task_id: evergreen task ID resmoke runs on - * client_id: symbolizer client ID - * client_secret: symbolizer client secret - """ - - evg_task_id: Optional[str] - client_id: Optional[str] - client_secret: Optional[str] - - @classmethod - def from_resmoke_config(cls) -> ResmokeSymbolizerConfig: - """ - Make resmoke symbolizer config from a global resmoke config. - - :return: resmoke symbolizer config - """ - return cls( - evg_task_id=_config.EVERGREEN_TASK_ID, - client_id=_config.SYMBOLIZER_CLIENT_ID, - client_secret=_config.SYMBOLIZER_CLIENT_SECRET, - ) - - @staticmethod - def is_windows() -> bool: - """ - Whether we are on Windows. - - :return: True if on Windows - """ - return sys.platform == "win32" or sys.platform == "cygwin" - - @staticmethod - def is_macos() -> bool: - """ - Whether we are on MacOS. - - :return: True if on MacOS. - """ - return sys.platform == "darwin" - - -class ResmokeSymbolizer: - """Symbolize stacktraces inside test logs.""" - - def __init__(self, config: Optional[ResmokeSymbolizerConfig] = None, - symbolizer_service: Optional[SymbolizerService] = None, - file_service: Optional[FileService] = None): - """Initialize instance.""" - - self.config = config if config is not None else ResmokeSymbolizerConfig.from_resmoke_config( - ) - self.symbolizer_service = symbolizer_service if symbolizer_service is not None else SymbolizerService( - ) - self.file_service = file_service if file_service is not None else FileService() - - def symbolize_test_logs(self, test: TestCase, - symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS) -> None: - """ - Perform all necessary actions to symbolize and write output to test logs. - - :param test: resmoke test case - :param symbolize_retry_timeout: the timeout for symbolizer retries - """ - if not self.should_symbolize(test): - return - - dbpath = self.get_stacktrace_dir(test) - if dbpath is None: - return - - test.logger.info("Looking for stacktrace files in '%s'", dbpath) - files = self.collect_stacktrace_files(dbpath) - if not files: - test.logger.info("No failure logs/stacktrace files found, skipping symbolization") - return - - with _lock: - test.logger.info("Found stacktrace files. \nBEGIN Symbolization") - test.logger.info("Stacktrace files: %s", files) - - start_time = time.perf_counter() - for file_path in files: - test.logger.info("Working on: %s", file_path) - symbolizer_script_timeout = int(symbolize_retry_timeout - - (time.perf_counter() - start_time)) - symbolized_out = self.symbolizer_service.run_symbolizer_script( - file_path, symbolizer_script_timeout) - test.logger.info(symbolized_out) - if time.perf_counter() - start_time > symbolize_retry_timeout: - break - - # To avoid performing the same actions on these files again, we remove them - self.file_service.remove_all(files) - - test.logger.info("\nEND Symbolization \nSymbolization process completed. ") - - def should_symbolize(self, test: TestCase) -> bool: - """ - Check whether we should perform symbolization process. - - :param test: resmoke test case - :return: whether we should symbolize - """ - if self.config.evg_task_id is None: - test.logger.info("Not running in Evergreen, skipping symbolization") - return False - - if self.config.client_id is None or self.config.client_secret is None: - test.logger.info("Symbolizer client secret and/or client ID are absent," - " skipping symbolization") - return False - - if self.config.is_windows(): - test.logger.info("Running on Windows, skipping symbolization") - return False - - if self.config.is_macos(): - test.logger.info("Running on MacOS, skipping symbolization") - return False - - if HANG_ANALYZER_CALLED.is_set(): - test.logger.info( - "Hang analyzer has been called, skipping symbolization to meet timeout constraints." - ) - return False - - return True - - def get_stacktrace_dir(self, test: TestCase) -> Optional[str]: - """ - Get dbpath from test case. - - :param test: resmoke test case - :return: dbpath or None - """ - if not hasattr(test, "fixture") or test.fixture is None: - test.logger.info("Test fixture is not available, could not get dbpath") - return None - - dbpath = test.fixture.get_dbpath_prefix() - if not self.file_service.check_path_exists(dbpath): - test.logger.info("dbpath '%s' directory not found", dbpath) - return None - - return dbpath - - def collect_stacktrace_files(self, dir_path: str) -> List[str]: - """ - Collect all stacktrace files which are not empty and return their full paths. - - :param dir_path: directory to look into - :return: list of stacktrace files paths - """ - - files = self.file_service.find_all_children_recursively(dir_path) - files = self.file_service.filter_by_extension(files, STACKTRACE_FILE_EXTENSION) - self.file_service.remove_empty(files) - files = self.file_service.filter_out_non_files(files) - - return files - - -class FileService: - """A service for working with files.""" - - @staticmethod - def find_all_children_recursively(dir_path: str) -> List[str]: - """ - Find all children files in directory recursively. - - :param dir_path: directory path - :return: list of all children files - """ - children_in_dir = [] - for parent, _, children in os.walk(dir_path): - children_in_dir.extend(os.path.join(parent, child) for child in children) - return children_in_dir - - @staticmethod - def filter_by_extension(files: List[str], extension: str) -> List[str]: - """ - Filter files by extension. - - :param files: list of file paths - :param extension: file extension - :return: filtered list of file paths - """ - return [f for f in files if f.endswith(extension)] - - @staticmethod - def filter_out_non_files(files: List[str]) -> List[str]: - """ - Filter out non files. - - :param files: list of paths - :return: filtered list of file paths - """ - return [f for f in files if os.path.isfile(f)] - - @staticmethod - def remove_empty(files: List[str]) -> None: - """ - Delete files that are empty. - - :param files: list of paths - """ - for file in [f for f in files if os.stat(f).st_size == 0]: - os.remove(file) - - @staticmethod - def remove_all(files: List[str]) -> None: - """ - Delete all files. - - :param files: list of paths - """ - for file in files: - os.remove(file) - - @staticmethod - def check_path_exists(path: str) -> bool: - """ - Check that file or directory exists. - - :param path: file or directory path - :return: whether path exists - """ - return os.path.exists(path) - - -class SymbolizerService: - """Wrapper around symbolizer script.""" - - @staticmethod - def run_symbolizer_script(full_file_path: str, retry_timeout_secs: int) -> str: - """ - Symbolize given file and return symbolized output as string. - - :param full_file_path: stacktrace file path - :param retry_timeout_secs: the timeout for symbolizer to retry - :return: symbolized output as string - """ - - symbolizer_args = [ - "db-contrib-tool", - "symbolize", - "--client-secret", - _config.SYMBOLIZER_CLIENT_SECRET, - "--client-id", - _config.SYMBOLIZER_CLIENT_ID, - "--total-seconds-for-retries", - str(retry_timeout_secs), - ] - - with open(full_file_path) as file_obj: - symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, - stdin=file_obj, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - try: - output, _ = symbolizer_process.communicate(timeout=retry_timeout_secs) - except subprocess.TimeoutExpired: - symbolizer_process.kill() - output, _ = symbolizer_process.communicate() - - return output.strip().decode() diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 81202daceb8..7658a415fa6 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -85,7 +85,7 @@ def get_task_name_without_suffix(task_name, variant_name): """Return evergreen task name without suffix added to the generated task. Remove evergreen variant name, numerical suffix and underscores between them from evergreen task name. - Example: "noPassthrough_0_enterprise-rhel-8-64-bit-dynamic-required" -> "noPassthrough" + Example: "noPassthrough_0_enterprise-rhel-80-64-bit-dynamic-required" -> "noPassthrough" """ task_name = task_name if task_name else "" return re.sub(fr"(_[0-9]+)?(_{variant_name})?$", "", task_name) diff --git a/buildscripts/resmokelib/utils/check_has_tag.py b/buildscripts/resmokelib/utils/check_has_tag.py index 18ae19bc21b..aebdd5644fe 100755 --- a/buildscripts/resmokelib/utils/check_has_tag.py +++ b/buildscripts/resmokelib/utils/check_has_tag.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """CLI interface for jscomment.""" -import re import sys import jscomment @@ -17,21 +16,11 @@ try: print(' 3 if any error happened during check') print('Usage:') print(' check_has_tag.py <jsfile> <tag>') - print('Notice: <tag> is a regex, not search string') sys.exit(2) else: tags = jscomment.get_tags(sys.argv[1]) print(sys.argv[1], "has tags:", tags) - - prog = re.compile(sys.argv[2]) - for tag in tags: - if prog.match(tag): - print("tag matches:", tag) - sys.exit(0) - - print("no tags match", sys.argv[2]) - sys.exit(1) - + sys.exit(0 if sys.argv[2] in tags else 1) except Exception as err: # pylint: disable=W0703 print(err) sys.exit(3) diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py index 6153349b6af..5857870e627 100644 --- a/buildscripts/resmokelib/utils/globstar.py +++ b/buildscripts/resmokelib/utils/globstar.py @@ -1,9 +1,11 @@ """Filename globbing utility.""" import glob as _glob +import os import os.path import re +_GLOBSTAR = "**" _CONTAINS_GLOB_PATTERN = re.compile("[*?[]") @@ -33,6 +35,155 @@ def iglob(globbed_pathname): expanded to match zero or more subdirectories. """ - for pathname in _glob.iglob(globbed_pathname, recursive=True): - # Normalize 'pathname' so exact string comparison can be used later. - yield os.path.normpath(pathname) + parts = _split_path(globbed_pathname) + parts = _canonicalize(parts) + + index = _find_globstar(parts) + if index == -1: + for pathname in _glob.iglob(globbed_pathname): + # Normalize 'pathname' so exact string comparison can be used later. + yield os.path.normpath(pathname) + return + + # **, **/, or **/a + if index == 0: + expand = _expand_curdir + + # a/** or a/**/ or a/**/b + else: + expand = _expand + + prefix_parts = parts[:index] + suffix_parts = parts[index + 1:] + + prefix = os.path.join(*prefix_parts) if prefix_parts else os.curdir + suffix = os.path.join(*suffix_parts) if suffix_parts else "" + + for (kind, path) in expand(prefix): + if not suffix_parts: + yield path + + # Avoid following symlinks to avoid an infinite loop + elif suffix_parts and kind == "dir" and not os.path.islink(path): + path = os.path.join(path, suffix) + for pathname in iglob(path): + yield pathname + + +def _split_path(pathname): + """Return 'pathname' as a list of path components.""" + + parts = [] + + while True: + (dirname, basename) = os.path.split(pathname) + parts.append(basename) + if pathname == dirname: + parts.append(dirname) + break + if not dirname: + break + pathname = dirname + + parts.reverse() + return parts + + +def _canonicalize(parts): + """Return a copy of 'parts' with consecutive "**"s coalesced. + + Raise a ValueError for unsupported uses of "**". + """ + + res = [] + + prev_was_globstar = False + for part in parts: + if part == _GLOBSTAR: + # Skip consecutive **'s + if not prev_was_globstar: + prev_was_globstar = True + res.append(part) + elif _GLOBSTAR in part: # a/b**/c or a/**b/c + raise ValueError("Can only specify glob patterns of the form a/**/b") + else: + prev_was_globstar = False + res.append(part) + + return res + + +def _find_globstar(parts): + """Return the index of the first occurrence of "**" in 'parts'. + + Return -1 if "**" is not found in the list. + """ + + for (idx, part) in enumerate(parts): + if part == _GLOBSTAR: + return idx + return -1 + + +def _list_dir(pathname): + """Return a pair of subdirectory names and filenames contained within the 'pathname' directory. + + If 'pathname' does not exist, then None is returned. + """ + + try: + (_root, dirs, files) = next(os.walk(pathname)) + return (dirs, files) + except StopIteration: + return None # 'pathname' directory does not exist + + +def _expand(pathname): + """Emit tuples of the form ("dir", dirname) and ("file", filename). + + The result is for all directories and files contained within the 'pathname' directory. + """ + + res = _list_dir(pathname) + if res is None: + return + + (dirs, files) = res + + # Zero expansion + if os.path.basename(pathname): + yield ("dir", os.path.join(pathname, "")) + + for fname in files: + path = os.path.join(pathname, fname) + yield ("file", path) + + for dname in dirs: + path = os.path.join(pathname, dname) + for xpath in _expand(path): + yield xpath + + +def _expand_curdir(pathname): + """Emit tuples of the form ("dir", dirname) and ("file", filename). + + The result is for all directories and files contained within the 'pathname' directory. + + The returned pathnames omit a "./" prefix. + """ + + res = _list_dir(pathname) + if res is None: + return + + (dirs, files) = res + + # Zero expansion + yield ("dir", "") + + for fname in files: + yield ("file", fname) + + for dname in dirs: + for xdir in _expand(dname): + yield xdir diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py index d7c2c295492..7af28d11ed8 100644 --- a/buildscripts/resmokelib/utils/jscomment.py +++ b/buildscripts/resmokelib/utils/jscomment.py @@ -36,18 +36,6 @@ def get_tags(pathname): tags = yaml.safe_load(_strip_jscomments(match.group(1))) if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) - - for tag in tags: - if '//' in tag: - raise ValueError(("Found a JS line comment '%s'. "\ - "Use '#' YAML style comments instead in a tags array %s") - % (tag, pathname)) - - if ' ' in tag: - raise ValueError(("Found an empty space in tag '%s'. "\ - "This is not permitted and may indicate a missing comma in %s") - % (tag, pathname)) - return tags except yaml.YAMLError as err: raise ValueError( |