diff options
Diffstat (limited to 'buildscripts/resmokelib')
24 files changed, 596 insertions, 442 deletions
diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py index 2cb85721e9e..3dc6a0a522d 100644 --- a/buildscripts/resmokelib/cli.py +++ b/buildscripts/resmokelib/cli.py @@ -18,6 +18,6 @@ def main(argv): usage="Resmoke is MongoDB's correctness testing orchestrator.\n" "For more information, see the help message for each subcommand.\n" "For example: resmoke.py run -h\n" - "Note: bisect and setup-multiversion subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" + "Note: bisect, setup-multiversion and symbolize subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" ) subcommand.execute() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index c1cc1048585..4b46d11f4df 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -55,7 +55,7 @@ DEFAULTS = { "archive_limit_tests": 10, "base_port": 20000, "backup_on_restart_dir": None, - "buildlogger_url": "https://logkeeper.mongodb.org", + "buildlogger_url": "https://logkeeper2.build.10gen.cc", "continue_on_failure": False, "dbpath_prefix": None, "dbtest_executable": None, @@ -88,11 +88,11 @@ DEFAULTS = { "repeat_tests_min": None, "repeat_tests_secs": None, "replay_file": None, - "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, - "run_all_feature_flags_no_tests": False, + "run_no_feature_flag_tests": False, "additional_feature_flags": None, + "additional_feature_flags_file": None, "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, @@ -166,7 +166,6 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "num_repeat_tests_max", "num_repeat_tests_min", "time_repeat_tests_secs", - "report_failure_status", ]) @@ -233,7 +232,6 @@ class SuiteOptions(_SuiteOptions): REPEAT_TESTS_MAX, REPEAT_TESTS_MIN, REPEAT_TESTS_SECS, - REPORT_FAILURE_STATUS, ]))) options = self._asdict() @@ -375,8 +373,11 @@ INSTALL_DIR = None # Whether to run tests for feature flags. RUN_ALL_FEATURE_FLAG_TESTS = None -# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true. -RUN_ALL_FEATURE_FLAGS = None +# Whether to run the tests with enabled feature flags +RUN_NO_FEATURE_FLAG_TESTS = None + +# the path to a file containing feature flags +ADDITIONAL_FEATURE_FLAGS_FILE = None # List of enabled feature flags. ENABLED_FEATURE_FLAGS = [] @@ -433,9 +434,6 @@ REPEAT_TESTS_MIN = None # If set, then each test is repeated the specified time (seconds) inside the suites. REPEAT_TESTS_SECS = None -# Controls if the test failure status should be reported as failed or be silently ignored. -REPORT_FAILURE_STATUS = None - # If set, then resmoke.py will write out a report file with the status of each test that ran. REPORT_FILE = None @@ -576,3 +574,12 @@ SHORTEN_LOGGER_NAME_CONFIG: dict = {} # that get loaded for multiversion tests can behave correctly on master and on v5.0; the latter # case runs 5.0 and 4.4 binaries and has this value set to True. Can be removed after 6.0. USE_LEGACY_MULTIVERSION = True + +# Expansions file location +# in CI, the expansions file is located in the ${workdir}, one dir up +# from src, the checkout directory +EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else None + +# Symbolizer secrets +SYMBOLIZER_CLIENT_SECRET = None +SYMBOLIZER_CLIENT_ID = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index db1319b8343..23c7ec69597 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -15,6 +15,7 @@ import shlex import pymongo.uri_parser +from buildscripts.idl import gen_all_feature_flag_list from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config @@ -27,6 +28,7 @@ def validate_and_update_config(parser, args): """Validate inputs and update config module.""" _validate_options(parser, args) _update_config_vars(args) + _update_symbolizer_secrets() _validate_config(parser) _set_logging_config() @@ -51,14 +53,9 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) - if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests: - if not os.path.isfile(ALL_FEATURE_FLAG_FILE): - parser.error( - "To run tests with all feature flags, the %s file must exist and be placed in" - " your working directory. The file can be downloaded from the artifacts tarball" - " in Evergreen. Alternatively, if you know which feature flags you want to enable," - " you can use the --additionalFeatureFlags command line argument" % - ALL_FEATURE_FLAG_FILE) + if args.additional_feature_flags_file and not os.path.isfile( + args.additional_feature_flags_file): + parser.error("The specified additional feature flags file does not exist.") def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) @@ -184,28 +181,36 @@ be invoked as either: - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""") raise RuntimeError(err) + def process_feature_flag_file(path): + with open(path) as fd: + return fd.read().split() + def setup_feature_flags(): _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") - _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests") + _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests") + _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") - # Running all feature flag tests implies running the fixtures with feature flags. if _config.RUN_ALL_FEATURE_FLAG_TESTS: - _config.RUN_ALL_FEATURE_FLAGS = True + print("Generating: ", ALL_FEATURE_FLAG_FILE) + gen_all_feature_flag_list.gen_all_feature_flags_file() all_ff = [] enabled_feature_flags = [] try: - with open(ALL_FEATURE_FLAG_FILE) as fd: - all_ff = fd.read().split() + all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE) except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS: raise - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS: enabled_feature_flags = all_ff[:] + if _config.ADDITIONAL_FEATURE_FLAGS_FILE: + enabled_feature_flags.extend( + process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE)) + # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags")) @@ -229,7 +234,7 @@ be invoked as either: _config.EXCLUDE_WITH_ANY_TAGS.extend( utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS: + if _config.RUN_NO_FEATURE_FLAG_TESTS: # Don't run any feature flag tests. _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags) else: @@ -302,6 +307,8 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG, _config.WT_COLL_CONFIG, \ _config.WT_INDEX_CONFIG = mongod_fuzzer_configs.fuzz_set_parameters( _config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS) + _config.EXCLUDE_WITH_ANY_TAGS.extend(["uses_compact"]) + _config.EXCLUDE_WITH_ANY_TAGS.extend(["requires_emptycapped"]) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) mongos_set_parameters = config.pop("mongos_set_parameters") @@ -322,7 +329,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") - _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config") @@ -502,3 +508,13 @@ def _tags_from_list(tags_list): tags.extend([t for t in tag.split(",") if t != ""]) return tags return None + + +def _update_symbolizer_secrets(): + """Open `expansions.yml`, get values for symbolizer secrets and update their values inside config.py .""" + if not _config.EVERGREEN_TASK_ID: + # not running on Evergreen + return + yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) + _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") + _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 19354d4c314..2a22d177a29 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -251,6 +251,11 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');") + if config.FUZZ_MONGOD_CONFIGS is not None and config.FUZZ_MONGOD_CONFIGS is not False: + # Prevent commands from running with the config fuzzer. + eval_sb.append( + "load('jstests/libs/override_methods/config_fuzzer_incompatible_commands.js');") + # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');") diff --git a/buildscripts/resmokelib/flags.py b/buildscripts/resmokelib/flags.py new file mode 100644 index 00000000000..6aff6961666 --- /dev/null +++ b/buildscripts/resmokelib/flags.py @@ -0,0 +1,5 @@ +"""Global flags used by resmoke.""" + +import threading + +HANG_ANALYZER_CALLED = threading.Event() diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index e5112b870b3..2c4d4b761d7 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -458,7 +458,15 @@ class GDBDumper(Dumper): cmds = self._prefix() + self._process_specific(pinfo, take_dump, logger) + self._postfix() - call([dbg, "--quiet", "--nx"] + list( + # gcore is both a command within GDB and a script packaged alongside gdb. The gcore script + # invokes the gdb binary with --readnever to avoid spending time loading the debug symbols + # prior to taking the core dump. The debug symbols are unneeded to generate the core dump. + # + # For reference + # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/gcore.in;h=34860de630cf0ee766e102eb82f7a3fddba6b368#l101 + skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] + + call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, diff --git a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py index 44d0b23e54e..ba259955444 100755 --- a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py +++ b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py @@ -68,7 +68,13 @@ class HangAnalyzer(Subcommand): self._configure_processes() self._setup_logging(logger) - def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements + def kill_rogue_processes(self): + """Kill any processes that are currently being analyzed.""" + processes = process_list.get_processes(self.process_ids, self.interesting_processes, + self.options.process_match, self.root_logger) + process.teardown_processes(self.root_logger, processes, dump_pids={}) + + def execute(self): # pylint: disable=too-many-branches """ Execute hang analysis. diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index 6325ecabb89..fbd3f1a9bfc 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -298,6 +298,7 @@ class BuildloggerServer(object): "builder": builder, "buildnum": build_num, "task_id": _config.EVERGREEN_TASK_ID, + "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -316,6 +317,7 @@ class BuildloggerServer(object): "command": test_command, "phase": self.config.get("build_phase", "unknown"), "task_id": _config.EVERGREEN_TASK_ID, + "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongod_fuzzer_configs.py index ec84d6c5a4e..874bc7764cf 100644 --- a/buildscripts/resmokelib/mongod_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongod_fuzzer_configs.py @@ -11,13 +11,16 @@ def generate_eviction_configs(rng): eviction_trigger = rng.randint(eviction_target + 1, 99) # Fuzz eviction_dirty_target and trigger both as relative and absolute values - target_bytes_min = 10 * 1024 * 1024 # 10MB + target_bytes_min = 50 * 1024 * 1024 # 50MB # 5% of 1GB default cache size on Evergreen target_bytes_max = 256 * 1024 * 1024 # 256MB # 1GB default cache size on Evergreen eviction_dirty_target = rng.choice( [rng.randint(5, 50), rng.randint(target_bytes_min, target_bytes_max)]) trigger_max = 75 if eviction_dirty_target <= 50 else target_bytes_max eviction_dirty_trigger = rng.randint(eviction_dirty_target + 1, trigger_max) + assert eviction_dirty_trigger > eviction_dirty_target + assert eviction_dirty_trigger <= trigger_max + close_idle_time_secs = rng.randint(1, 100) close_handle_minimum = rng.randint(0, 1000) close_scan_interval = rng.randint(1, 100) @@ -89,6 +92,8 @@ def generate_independent_parameters(rng): # The old retryable writes format is used by other variants. Weight towards turning on the # new retryable writes format on in this one. ret["storeFindAndModifyImagesInSideCollection"] = True + # TODO (SERVER-75632): Uncomment this to enable passthrough testing. + # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) return ret diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 2c9bae45766..713dedf0f82 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -49,14 +49,14 @@ class MultiversionConfigSubcommand(Subcommand): mongo_version=MongoVersion.from_yaml_file(multiversionconstants.MONGO_VERSION_YAML), mongo_releases=MongoReleases.from_yaml_file(multiversionconstants.RELEASES_YAML), ) - fcv_constants = multiversion_service.calculate_fcv_constants() + version_constants = multiversion_service.calculate_version_constants() return MultiversionConfig( last_versions=multiversionconstants.OLD_VERSIONS, - requires_fcv_tag=fcv_constants.get_fcv_tag_list(), - requires_fcv_tag_lts=fcv_constants.get_lts_fcv_tag_list(), - requires_fcv_tag_continuous=fcv_constants.get_continuous_fcv_tag_list(), - last_lts_fcv=fcv_constants.get_last_lts_fcv(), - last_continuous_fcv=fcv_constants.get_last_continuous_fcv(), + requires_fcv_tag=version_constants.get_fcv_tag_list(), + requires_fcv_tag_lts=version_constants.get_lts_fcv_tag_list(), + requires_fcv_tag_continuous=version_constants.get_continuous_fcv_tag_list(), + last_lts_fcv=version_constants.get_last_lts_fcv(), + last_continuous_fcv=version_constants.get_last_continuous_fcv(), ) diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 3597cc2177f..621d5ade9ae 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -25,9 +25,9 @@ def version_str(version: Version) -> str: return f"{version.major}.{version.minor}" -class FcvConstantValues(NamedTuple): +class VersionConstantValues(NamedTuple): """ - Object to hold the calculated FCV constants. + Object to hold the calculated Version constants. * latest: Latest FCV. * last_continuous: Last continuous FCV. @@ -36,6 +36,7 @@ class FcvConstantValues(NamedTuple): * requires_fcv_tag_list_continuous: List of FCVs that we need to generate a tag for against continuous versions. * fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0. + * eols: List of stable MongoDB versions since v2.0 that have been EOL'd. """ latest: Version @@ -44,6 +45,7 @@ class FcvConstantValues(NamedTuple): requires_fcv_tag_list: List[Version] requires_fcv_tag_list_continuous: List[Version] fcvs_less_than_latest: List[Version] + eols: List[Version] def get_fcv_tag_list(self) -> str: """Get a comma joined string of all the fcv tags.""" @@ -96,6 +98,10 @@ class FcvConstantValues(NamedTuple): last_continuous = self.get_last_continuous_fcv() return f"{base_name}-{last_continuous}" + def get_eols(self) -> List[str]: + """Get EOL'd versions as list of strings.""" + return [version_str(eol) for eol in self.eols] + class MongoVersion(BaseModel): """ @@ -132,12 +138,14 @@ class MongoReleases(BaseModel): * feature_compatibility_version: All FCVs starting with 4.0. * long_term_support_releases: All LTS releases starting with 4.0. + * eol_versions: List of stable MongoDB versions since 2.0 that have been EOL'd. * generate_fcv_lower_bound_override: Extend FCV generation down to the previous value of last LTS. """ feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions") long_term_support_releases: List[str] = Field(alias="longTermSupportReleases") + eol_versions: List[str] = Field(alias="eolVersions") generate_fcv_lower_bound_override: Optional[str] = Field(None, alias="generateFCVLowerBoundOverride") @@ -159,7 +167,11 @@ class MongoReleases(BaseModel): def get_lts_versions(self) -> List[Version]: """Get the Version representation of the lts versions.""" - return [Version(fcv) for fcv in self.long_term_support_releases] + return [Version(lts) for lts in self.long_term_support_releases] + + def get_eol_versions(self) -> List[Version]: + """Get the Version representation of the EOL versions.""" + return [Version(eol) for eol in self.eol_versions] class MultiversionService: @@ -175,11 +187,12 @@ class MultiversionService: self.mongo_version = mongo_version self.mongo_releases = mongo_releases - def calculate_fcv_constants(self) -> FcvConstantValues: + def calculate_version_constants(self) -> VersionConstantValues: """Calculate multiversion constants from data files.""" latest = self.mongo_version.get_version() fcvs = self.mongo_releases.get_fcv_versions() lts = self.mongo_releases.get_lts_versions() + eols = self.mongo_releases.get_eol_versions() lower_bound_override = self.mongo_releases.generate_fcv_lower_bound_override # Highest release less than latest. @@ -200,7 +213,12 @@ class MultiversionService: # All FCVs less than latest. fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)] - return FcvConstantValues(latest=latest, last_continuous=last_continuous, last_lts=last_lts, - requires_fcv_tag_list=requires_fcv_tag_list, - requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, - fcvs_less_than_latest=fcvs_less_than_latest) + return VersionConstantValues( + latest=latest, + last_continuous=last_continuous, + last_lts=last_lts, + requires_fcv_tag_list=requires_fcv_tag_list, + requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, + fcvs_less_than_latest=fcvs_less_than_latest, + eols=eols, + ) diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 0cac4fbc95b..3646d139b1f 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,25 +2,24 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output +import requests import structlog -try: - # when running resmoke - from buildscripts.resmokelib.multiversion.multiversion_service import ( - MongoReleases, MongoVersion, MultiversionService) - from buildscripts.resmokelib.multiversionsetupconstants import \ - USE_EXISTING_RELEASES_FILE -except ImportError: - # when running db-contrib-tool - from multiversion.multiversion_service import (MongoReleases, MongoVersion, MultiversionService) - from multiversionsetupconstants import USE_EXISTING_RELEASES_FILE +from buildscripts.resmokelib.multiversion.multiversion_service import ( + MongoReleases, MongoVersion, MultiversionService, MONGO_VERSION_YAML, RELEASES_YAML) +from buildscripts.resmokelib.multiversionsetupconstants import \ + USE_EXISTING_RELEASES_FILE -LOGGER = structlog.getLogger(__name__) +LAST_LTS = "last_lts" +LAST_CONTINUOUS = "last_continuous" + +# We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date +# with the latest EOL versions. If a "last-continuous" version is EOL, we don't include +# it in the multiversion config and therefore don't test against it. +MASTER_RELEASES_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" -# These values must match the include paths for artifacts.tgz in evergreen.yml. -MONGO_VERSION_YAML = ".resmoke_mongo_version.yml" -RELEASES_YAML = ".resmoke_mongo_release_values.yml" +LOGGER = structlog.getLogger(__name__) def generate_mongo_version_file(): @@ -40,14 +39,8 @@ def generate_mongo_version_file(): def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. - releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml") - if not os.path.isfile(releases_yaml_path): - LOGGER.info( - 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.' - .format(releases_yaml_path)) - return - - shutil.copyfile(releases_yaml_path, RELEASES_YAML) + with open(RELEASES_YAML, "wb") as file: + file.write(requests.get(MASTER_RELEASES_FILE).content) def in_git_root_dir(): @@ -86,33 +79,34 @@ multiversion_service = MultiversionService( mongo_releases=MongoReleases.from_yaml_file(RELEASES_YAML), ) -fcv_constants = multiversion_service.calculate_fcv_constants() +version_constants = multiversion_service.calculate_version_constants() -LAST_LTS_BIN_VERSION = fcv_constants.get_last_lts_fcv() -LAST_CONTINUOUS_BIN_VERSION = fcv_constants.get_last_continuous_fcv() +LAST_LTS_BIN_VERSION = version_constants.get_last_lts_fcv() +LAST_CONTINUOUS_BIN_VERSION = version_constants.get_last_continuous_fcv() -LAST_LTS_FCV = fcv_constants.get_last_lts_fcv() -LAST_CONTINUOUS_FCV = fcv_constants.get_last_continuous_fcv() -LATEST_FCV = fcv_constants.get_latest_fcv() +LAST_LTS_FCV = version_constants.get_last_lts_fcv() +LAST_CONTINUOUS_FCV = version_constants.get_last_continuous_fcv() +LATEST_FCV = version_constants.get_latest_fcv() -LAST_CONTINUOUS_MONGO_BINARY = fcv_constants.build_last_continuous_binary("mongo") -LAST_CONTINUOUS_MONGOD_BINARY = fcv_constants.build_last_continuous_binary("mongod") -LAST_CONTINUOUS_MONGOS_BINARY = fcv_constants.build_last_continuous_binary("mongos") +LAST_CONTINUOUS_MONGO_BINARY = version_constants.build_last_continuous_binary("mongo") +LAST_CONTINUOUS_MONGOD_BINARY = version_constants.build_last_continuous_binary("mongod") +LAST_CONTINUOUS_MONGOS_BINARY = version_constants.build_last_continuous_binary("mongos") -LAST_LTS_MONGO_BINARY = fcv_constants.build_last_lts_binary("mongo") -LAST_LTS_MONGOD_BINARY = fcv_constants.build_last_lts_binary("mongod") -LAST_LTS_MONGOS_BINARY = fcv_constants.build_last_lts_binary("mongos") +LAST_LTS_MONGO_BINARY = version_constants.build_last_lts_binary("mongo") +LAST_LTS_MONGOD_BINARY = version_constants.build_last_lts_binary("mongod") +LAST_LTS_MONGOS_BINARY = version_constants.build_last_lts_binary("mongos") -REQUIRES_FCV_TAG_LATEST = fcv_constants.get_latest_tag() +REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() # Generate tags for all FCVS in (lastLTS, latest], or (lowerBoundOverride, latest] if requested. # All multiversion tests should be run with these tags excluded. -REQUIRES_FCV_TAG = fcv_constants.get_fcv_tag_list() +REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] -EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest]) +EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) -OLD_VERSIONS = ["last_lts"] -if LAST_LTS_FCV != LAST_CONTINUOUS_FCV: - OLD_VERSIONS.append("last_continuous") +OLD_VERSIONS = [ + LAST_LTS +] if LAST_CONTINUOUS_FCV == LAST_LTS_FCV or LAST_CONTINUOUS_FCV in version_constants.get_eols( +) else [LAST_LTS, LAST_CONTINUOUS] diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 5f5bb7f3765..e450540f293 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -11,7 +11,6 @@ from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.multiversion import MultiversionPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin -from buildscripts.resmokelib.symbolizer import SymbolizerPlugin from buildscripts.resmokelib.undodb import UndoDbPlugin _PLUGINS = [ @@ -19,7 +18,6 @@ _PLUGINS = [ HangAnalyzerPlugin(), UndoDbPlugin(), PowercyclePlugin(), - SymbolizerPlugin(), GenerateFCVConstantsPlugin(), DiscoveryPlugin(), MultiversionPlugin(), diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 20ff72c2c79..420825bfec4 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -385,25 +385,12 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ - UNRELIABLE_TAG = _TagInfo( - tag_name="unreliable", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - report_failure_status="silentfail")) - RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) - RETRY_ON_FAILURE_TAG = _TagInfo( - tag_name="retry_on_failure", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, - report_failure_status="silentfail")) - @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -438,29 +425,8 @@ class TestRunnerEvg(TestRunner): combinations = [] - if config.EVERGREEN_PATCH_BUILD: - combinations.append(("unreliable and resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("unreliable and not resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) - combinations.append(("reliable and resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("reliable and not resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - else: - combinations.append(("retry on failure and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("retry on failure and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - combinations.append(("run once and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("run once and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) + combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) + combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) return combinations @@ -740,15 +706,19 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", - action="store_true", help= - "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" - ) + "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", + help=("Do not run any tests tagged with enabled feature flags." + " This argument has precedence over --runAllFeatureFlagTests" + "; used for multiversion suites")) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") + parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", + action="store", metavar="FILE", + help="The path to a file with feature flags, delimited by newlines.") + parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -879,13 +849,6 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( - "--reportFailureStatus", action="store", dest="report_failure_status", - choices=("fail", "silentfail"), metavar="STATUS", - help="Controls if the test failure status should be reported as failed" - " or be silently ignored (STATUS=silentfail). Dynamic test failures will" - " never be silently ignored. Defaults to STATUS=%(default)s.") - - internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") @@ -1096,15 +1059,6 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) - # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on - # all_feature_flags.txt which may not exist in the local dev environment. - run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) - if run_all_feature_flag_tests is not None: - setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) - del parsed_args.run_all_feature_flag_tests - - del parsed_args.run_all_feature_flags_no_tests - # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py index 5df67812d06..609ea8a6e3e 100644 --- a/buildscripts/resmokelib/sighandler.py +++ b/buildscripts/resmokelib/sighandler.py @@ -10,10 +10,10 @@ import traceback import psutil +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib import reportfile from buildscripts.resmokelib import testing from buildscripts.resmokelib import config -from buildscripts.resmokelib.hang_analyzer import hang_analyzer from buildscripts.resmokelib import parser _IS_WINDOWS = (sys.platform == "win32") @@ -32,8 +32,8 @@ def register(logger, suites, start_time): log suite summaries. """ + HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to SIGUSR1 signal" - _dump_and_log(header_msg) def _handle_set_event(event_handle): @@ -53,6 +53,7 @@ def register(logger, suites, start_time): except win32event.error as err: logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) else: + HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to signal from win32event.SetEvent" _dump_and_log(header_msg) @@ -159,4 +160,26 @@ def _analyze_pids(logger, pids): if not os.getenv('ASAN_OPTIONS'): hang_analyzer_args.append('-c') _hang_analyzer = parser.parse_command_line(hang_analyzer_args, logger=logger) - _hang_analyzer.execute() + + # Evergreen has a 15 minute timeout for task timeout commands + # Limit the hang analyzer to 12 minutes so there is time for other tasks. + hang_analyzer_hard_timeout = None + if config.EVERGREEN_TASK_ID: + hang_analyzer_hard_timeout = 60 * 12 + logger.info( + "Limit the resmoke invoked hang analyzer to 12 minutes so there is time for resmoke to finish up." + ) + + hang_analyzer_thread = threading.Thread(target=_hang_analyzer.execute, daemon=True) + hang_analyzer_thread.start() + hang_analyzer_thread.join(hang_analyzer_hard_timeout) + + if hang_analyzer_thread.is_alive(): + logger.warning( + "Resmoke invoked hang analyzer thread did not finish, but will continue running in the background. The thread may be disruputed and may show extraneous output." + ) + logger.warning("Cleaning up resmoke child processes so that resmoke can fail gracefully.") + _hang_analyzer.kill_rogue_processes() + + else: + logger.info("Done running resmoke invoked hang analyzer thread.") diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 394c025d510..55c23923ecc 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -4,6 +4,8 @@ import os import os.path import time import shutil +import uuid + import yaml import pymongo @@ -23,6 +25,9 @@ class MongoDFixture(interface.Fixture): self.mongod_options = self.fixturelib.make_historic( self.fixturelib.default_if_none(mongod_options, {})) + if "set_parameters" not in self.mongod_options: + self.mongod_options["set_parameters"] = {} + if add_feature_flags: for ff in self.config.ENABLED_FEATURE_FLAGS: self.mongod_options["set_parameters"][ff] = "true" @@ -52,6 +57,11 @@ class MongoDFixture(interface.Fixture): self.port = port or fixturelib.get_next_port(job_num) self.mongod_options["port"] = self.port + # Always log backtraces to a file in the dbpath in our testing. + backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(), + uuid.uuid4().hex + ".stacktrace") + self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name + def setup(self): """Set up the mongod.""" if not self.preserve_dbpath and os.path.lexists(self._dbpath): @@ -61,7 +71,8 @@ class MongoDFixture(interface.Fixture): launcher = MongodLauncher(self.fixturelib) # Second return val is the port, which we ignore because we explicitly created the port above. - # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 + # The port is used to set other mongod_option's here: + # https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num, executable=self.mongod_executable, mongod_options=self.mongod_options) @@ -168,7 +179,7 @@ class MongoDFixture(interface.Fixture): def get_driver_connection_url(self): """Return the driver connection URL.""" - return "mongodb://" + self.get_internal_connection_string() + return "mongodb://" + self.get_internal_connection_string() + "/?directConnection=true" # The below parameters define the default 'logComponentVerbosity' object passed to mongod processes @@ -180,15 +191,16 @@ class MongoDFixture(interface.Fixture): # The default verbosity setting for any tests that are not started with an Evergreen task id. This # will apply to any tests run locally. DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"rollback": 2}, "sharding": {"migration": 2}, "transaction": 4, - "tenantMigration": 4 + "replication": {"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, + "transaction": 4, "tenantMigration": 4 } # The default verbosity setting for any mongod processes running in Evergreen i.e. started with an # Evergreen task id. DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, "rollback": 2}, - "sharding": {"migration": 2}, "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 + "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, + "rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, + "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 } @@ -242,6 +254,15 @@ class MongodLauncher(object): if "shardsvr" in mongod_options and "orphanCleanupDelaySecs" not in suite_set_parameters: suite_set_parameters["orphanCleanupDelaySecs"] = 1 + # receiveChunkWaitForRangeDeleterTimeoutMS controls the amount of time an incoming migration + # will wait for an intersecting range with data in it to be cleared up before failing. The + # default is 10 seconds, but in some slower variants this is not enough time for the range + # deleter to finish so we increase it here to 90 seconds. Setting a value for this parameter + # in the .yml file overrides this. + if (("shardsvr" in mongod_options or "configsvr" in mongod_options) + and "receiveChunkWaitForRangeDeleterTimeoutMS" not in suite_set_parameters): + suite_set_parameters["receiveChunkWaitForRangeDeleterTimeoutMS"] = 90000 + # The LogicalSessionCache does automatic background refreshes in the server. This is # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off # background refreshing for tests. Set in the .yml file to override this. @@ -292,6 +313,13 @@ class MongodLauncher(object): "mode": "alwaysOn", "data": {"numTickets": self.config.FLOW_CONTROL_TICKETS} } + # The internalQueryForceClassicEngine parameter was renamed to + # internalQueryFrameworkControl starting in version 6.1. + if "internalQueryFrameworkControl" in suite_set_parameters and suite_set_parameters[ + "internalQueryFrameworkControl"] == "forceClassicEngine": + del suite_set_parameters["internalQueryFrameworkControl"] + suite_set_parameters["internalQueryForceClassicEngine"] = True + _add_testing_set_parameters(suite_set_parameters) shortcut_opts = { @@ -368,3 +396,7 @@ def _add_testing_set_parameters(suite_set_parameters): """ suite_set_parameters.setdefault("testingDiagnosticsEnabled", True) suite_set_parameters.setdefault("enableTestCommands", True) + # The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up. + # Set it to true for now as a placeholder that will error if no further processing is done. + # The placeholder is needed so older versions don't have this option won't have this value set. + suite_set_parameters.setdefault("backtraceLogFile", True) diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index 38909056ce3..23ebfda8de3 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -6,6 +6,7 @@ import threading from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib import utils +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib.utils import globstar @@ -105,5 +106,9 @@ class HookTestArchival(object): else: logger.info("Archive succeeded for %s: %s", test_name, message) - if not manager.setup_fixture(logger): + if HANG_ANALYZER_CALLED.is_set(): + logger.info("Hang Analyzer has been called. Fixtures will not be restarted.") + raise errors.StopExecution( + "Hang analyzer has been called. Stopping further execution of tests.") + elif not manager.setup_fixture(logger): raise errors.StopExecution("Error while restarting test fixtures after archiving.") diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index b0ff858c5dc..5278819da8f 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -276,16 +276,15 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): secondary.await_ready() client = secondary.mongo_client() - minvalid_doc = client.local["replset.minvalid"].find_one() oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one() recovery_timestamp_res = client.admin.command("replSetTest", getLastStableRecoveryTimestamp=True) latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural", pymongo.DESCENDING)]) - self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {}," + self.logger.info("Checking replication invariants. oplogTruncateAfterPoint: {}," " stable recovery timestamp: {}, latest oplog doc: {}".format( - minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res, + oplog_truncate_after_doc, recovery_timestamp_res, latest_oplog_doc)) null_ts = bson.Timestamp(0, 0) @@ -299,13 +298,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): raise errors.ServerFailure( "Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc)) - # The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default - # it to null. - oplog_truncate_after_ts = null_ts - if oplog_truncate_after_doc is not None: - oplog_truncate_after_ts = oplog_truncate_after_doc.get( - "oplogTruncateAfterPoint", null_ts) - # The "lastStableRecoveryTimestamp" field is present if the storage engine supports # "recover to a timestamp". If it's a null timestamp on a durable storage engine, that # means we do not yet have a stable checkpoint timestamp and must be restarting at the @@ -328,94 +320,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): recovery_timestamp, latest_oplog_entry_ts, recovery_timestamp_res, latest_oplog_doc)) - if minvalid_doc is not None: - applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts) - minvalid_ts = minvalid_doc.get("ts", null_ts) - - # The "appliedThrough" value should always equal the "last stable recovery - # timestamp", AKA the stable checkpoint for durable engines, on server restart. - # - # The written "appliedThrough" time is updated with the latest timestamp at the end - # of each batch application, and batch boundaries are the only valid stable - # timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must - # equal the checkpoint timestamp, because any stable timestamp that the checkpoint - # could use includes an equal persisted appliedThrough timestamp. - if (recovery_timestamp != null_ts and applied_through_ts != null_ts - and (not recovery_timestamp == applied_through_ts)): - raise errors.ServerFailure( - "The condition last stable recovery timestamp ({}) == appliedThrough ({})" - " doesn't hold: minValid document={}," - " getLastStableRecoveryTimestamp result={}, last oplog entry={}".format( - recovery_timestamp, applied_through_ts, minvalid_doc, - recovery_timestamp_res, latest_oplog_doc)) - - if applied_through_ts == null_ts: - # We clear "appliedThrough" to represent having applied through the top of the - # oplog in PRIMARY state or immediately after "rollback via refetch". - # If we are using a storage engine that supports "recover to a timestamp," - # then we will have a "last stable recovery timestamp" and we should use that - # as our "appliedThrough" (similarly to why we assert their equality above). - # If both are null, then we are in PRIMARY state on a storage engine that does - # not support "recover to a timestamp" or in RECOVERING immediately after - # "rollback via refetch". Since we do not update "minValid" in PRIMARY state, - # we leave "appliedThrough" as null so that the invariants below hold, rather - # than substituting the latest oplog entry for the "appliedThrough" value. - applied_through_ts = recovery_timestamp - - if minvalid_ts == null_ts: - # The server treats the "ts" field in the minValid document as missing when its - # value is the null timestamp. - minvalid_ts = applied_through_ts - - if latest_oplog_entry_ts == null_ts: - # If the oplog is empty, we treat the "minValid" as the latest oplog entry. - latest_oplog_entry_ts = minvalid_ts - - if oplog_truncate_after_ts == null_ts: - # The server treats the "oplogTruncateAfterPoint" field as missing when its - # value is the null timestamp. When it is null, the oplog is complete and - # should not be truncated, so it is effectively the top of the oplog. - oplog_truncate_after_ts = latest_oplog_entry_ts - - # Check the ordering invariants before the secondary has reconciled the end of - # its oplog. - # The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of - # oplog entries before they are written to the oplog. Thus, it can be ahead - # of the top of the oplog before any oplog entries are written, and behind it - # after some are written. Thus, we cannot compare it to the top of the oplog. - - # appliedThrough <= minValid - # appliedThrough represents the end of the previous batch, so it is always the - # earliest. - if applied_through_ts > minvalid_ts: - raise errors.ServerFailure( - "The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid" - " document={}, latest oplog entry={}".format( - applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc)) - - # minValid <= oplogTruncateAfterPoint - # This is true because this hook is never run after a rollback. Thus, we only - # move "minValid" to the end of each batch after the batch is written to the oplog. - # We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from - # the end of the previous batch to the end of the current batch. Thus "minValid" - # must be less than or equal to the "oplogTruncateAfterPoint". - if minvalid_ts > oplog_truncate_after_ts: - raise errors.ServerFailure( - "The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't" - " hold: minValid document={}, oplogTruncateAfterPoint document={}," - " latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts, - minvalid_doc, oplog_truncate_after_doc, - latest_oplog_doc)) - - # minvalid <= latest oplog entry - # "minValid" is set to the end of a batch after the batch is written to the oplog. - # Thus it is always less than or equal to the top of the oplog. - if minvalid_ts > latest_oplog_entry_ts: - raise errors.ServerFailure( - "The condition minValid <= top of oplog ({} <= {}) doesn't" - " hold: minValid document={}, latest oplog entry={}".format( - minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc)) - try: secondary.teardown() except errors.ServerFailure: diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index c09602baee8..8fb565cd1d8 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -363,7 +363,7 @@ class FileBasedStepdownLifecycle(object): # We remove the "permitted" file to revoke permission for the stepdown thread to continue # performing stepdowns. - os.remove(self.__stepdown_files.permitted) + fs.remove_if_exists(self.__stepdown_files.permitted) class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-attributes @@ -627,6 +627,10 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at break except pymongo.errors.NotMasterError: pass + except pymongo.errors.OperationFailure as ex: + if ex.code == 166: # CommandNotSupportedOnView + # listCollections return also views and collStats is not supported on views + break retarget_time = time.time() - start_time if retarget_time >= 60: raise RuntimeError( diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 38fe409b656..0aa87390e59 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -10,6 +10,7 @@ import unittest from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import logging +from buildscripts.resmokelib.testing.symbolizer_service import ResmokeSymbolizer # pylint: disable=attribute-defined-outside-init @@ -54,7 +55,9 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for report in reports: if not isinstance(report, TestReport): - raise TypeError("reports must be a list of TestReport instances") + raise TypeError( + f"reports must be a list of TestReport instances, current report is {type(report)}" + ) with report._lock: # pylint: disable=protected-access for test_info in report.test_infos: @@ -136,28 +139,37 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr def stopTest(self, test): # pylint: disable=invalid-name """Call after 'test' has run.""" - unittest.TestResult.stopTest(self, test) - - with self._lock: - test_info = self.find_test_info(test) - test_info.end_time = time.time() - test_status = "no failures detected" if test_info.status == "pass" else "failed" - - time_taken = test_info.end_time - test_info.start_time - self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, - test_status) - - # Asynchronously closes the buildlogger test handler to avoid having too many threads open - # on 32-bit systems. - for handler in test.logger.handlers: - # We ignore the cancellation token returned by close_later() since we always want the - # logs to eventually get flushed. - logging.flush.close_later(handler) - - # Restore the original logger for the test. - test.reset_logger() - - def addError(self, test, err): # pylint: disable=invalid-name + try: + # check if there are stacktrace files, if so, invoke the symbolizer here. + # log symbolized output to test.logger.info() + symbolizer = ResmokeSymbolizer() + symbolizer.symbolize_test_logs(test) + + unittest.TestResult.stopTest(self, test) + + with self._lock: + test_info = self.find_test_info(test) + test_info.end_time = time.time() + test_status = "no failures detected" if test_info.status == "pass" else "failed" + + time_taken = test_info.end_time - test_info.start_time + self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, + test_status) + + finally: + # This is a failsafe. In the event that 'stopTest' fails, + # any rogue logger handlers will be removed from this test. + # If not cleaned up, these will trigger 'setup failures' -- + # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE. + for handler in test.logger.handlers: + # We ignore the cancellation token returned by close_later() since we always want the + # logs to eventually get flushed. + logging.flush.close_later(handler) + + # Restore the original logger for the test. + test.reset_logger() + + def addError(self, test, err): """Call when a non-failureException was raised during the execution of 'test'.""" unittest.TestResult.addError(self, test, err) @@ -201,12 +213,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = self.find_test_info(test) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = test.return_code def setFailure(self, test, return_code=1): # pylint: disable=invalid-name @@ -218,12 +225,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr raise ValueError("stopTest was not called on %s" % (test.basename())) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = return_code # Recompute number of success, failures, and errors. diff --git a/buildscripts/resmokelib/testing/symbolizer_service.py b/buildscripts/resmokelib/testing/symbolizer_service.py new file mode 100644 index 00000000000..9e78b0dd6b7 --- /dev/null +++ b/buildscripts/resmokelib/testing/symbolizer_service.py @@ -0,0 +1,294 @@ +"""Symbolize stacktraces inside test logs.""" +from __future__ import annotations + +import os +import subprocess +import sys +import time +from datetime import timedelta +from threading import Lock + +from typing import List, Optional, NamedTuple + +from buildscripts.resmokelib import config as _config +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED +from buildscripts.resmokelib.testing.testcases.interface import TestCase + +# This lock prevents different resmoke jobs from symbolizing stacktraces concurrently, +# which includes downloading the debug symbols, that can be reused by other resmoke jobs +_lock = Lock() + +STACKTRACE_FILE_EXTENSION = ".stacktrace" +SYMBOLIZE_RETRY_TIMEOUT_SECS = timedelta(minutes=4).total_seconds() + + +class ResmokeSymbolizerConfig(NamedTuple): + """ + Resmoke symbolizer config. + + * evg_task_id: evergreen task ID resmoke runs on + * client_id: symbolizer client ID + * client_secret: symbolizer client secret + """ + + evg_task_id: Optional[str] + client_id: Optional[str] + client_secret: Optional[str] + + @classmethod + def from_resmoke_config(cls) -> ResmokeSymbolizerConfig: + """ + Make resmoke symbolizer config from a global resmoke config. + + :return: resmoke symbolizer config + """ + return cls( + evg_task_id=_config.EVERGREEN_TASK_ID, + client_id=_config.SYMBOLIZER_CLIENT_ID, + client_secret=_config.SYMBOLIZER_CLIENT_SECRET, + ) + + @staticmethod + def is_windows() -> bool: + """ + Whether we are on Windows. + + :return: True if on Windows + """ + return sys.platform == "win32" or sys.platform == "cygwin" + + @staticmethod + def is_macos() -> bool: + """ + Whether we are on MacOS. + + :return: True if on MacOS. + """ + return sys.platform == "darwin" + + +class ResmokeSymbolizer: + """Symbolize stacktraces inside test logs.""" + + def __init__(self, config: Optional[ResmokeSymbolizerConfig] = None, + symbolizer_service: Optional[SymbolizerService] = None, + file_service: Optional[FileService] = None): + """Initialize instance.""" + + self.config = config if config is not None else ResmokeSymbolizerConfig.from_resmoke_config( + ) + self.symbolizer_service = symbolizer_service if symbolizer_service is not None else SymbolizerService( + ) + self.file_service = file_service if file_service is not None else FileService() + + def symbolize_test_logs(self, test: TestCase, + symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS) -> None: + """ + Perform all necessary actions to symbolize and write output to test logs. + + :param test: resmoke test case + :param symbolize_retry_timeout: the timeout for symbolizer retries + """ + if not self.should_symbolize(test): + return + + dbpath = self.get_stacktrace_dir(test) + if dbpath is None: + return + + test.logger.info("Looking for stacktrace files in '%s'", dbpath) + files = self.collect_stacktrace_files(dbpath) + if not files: + test.logger.info("No failure logs/stacktrace files found, skipping symbolization") + return + + with _lock: + test.logger.info("Found stacktrace files. \nBEGIN Symbolization") + test.logger.info("Stacktrace files: %s", files) + + start_time = time.perf_counter() + for file_path in files: + test.logger.info("Working on: %s", file_path) + symbolizer_script_timeout = int(symbolize_retry_timeout - + (time.perf_counter() - start_time)) + symbolized_out = self.symbolizer_service.run_symbolizer_script( + file_path, symbolizer_script_timeout) + test.logger.info(symbolized_out) + if time.perf_counter() - start_time > symbolize_retry_timeout: + break + + # To avoid performing the same actions on these files again, we remove them + self.file_service.remove_all(files) + + test.logger.info("\nEND Symbolization \nSymbolization process completed. ") + + def should_symbolize(self, test: TestCase) -> bool: + """ + Check whether we should perform symbolization process. + + :param test: resmoke test case + :return: whether we should symbolize + """ + if self.config.evg_task_id is None: + test.logger.info("Not running in Evergreen, skipping symbolization") + return False + + if self.config.client_id is None or self.config.client_secret is None: + test.logger.info("Symbolizer client secret and/or client ID are absent," + " skipping symbolization") + return False + + if self.config.is_windows(): + test.logger.info("Running on Windows, skipping symbolization") + return False + + if self.config.is_macos(): + test.logger.info("Running on MacOS, skipping symbolization") + return False + + if HANG_ANALYZER_CALLED.is_set(): + test.logger.info( + "Hang analyzer has been called, skipping symbolization to meet timeout constraints." + ) + return False + + return True + + def get_stacktrace_dir(self, test: TestCase) -> Optional[str]: + """ + Get dbpath from test case. + + :param test: resmoke test case + :return: dbpath or None + """ + if not hasattr(test, "fixture") or test.fixture is None: + test.logger.info("Test fixture is not available, could not get dbpath") + return None + + dbpath = test.fixture.get_dbpath_prefix() + if not self.file_service.check_path_exists(dbpath): + test.logger.info("dbpath '%s' directory not found", dbpath) + return None + + return dbpath + + def collect_stacktrace_files(self, dir_path: str) -> List[str]: + """ + Collect all stacktrace files which are not empty and return their full paths. + + :param dir_path: directory to look into + :return: list of stacktrace files paths + """ + + files = self.file_service.find_all_children_recursively(dir_path) + files = self.file_service.filter_by_extension(files, STACKTRACE_FILE_EXTENSION) + self.file_service.remove_empty(files) + files = self.file_service.filter_out_non_files(files) + + return files + + +class FileService: + """A service for working with files.""" + + @staticmethod + def find_all_children_recursively(dir_path: str) -> List[str]: + """ + Find all children files in directory recursively. + + :param dir_path: directory path + :return: list of all children files + """ + children_in_dir = [] + for parent, _, children in os.walk(dir_path): + children_in_dir.extend(os.path.join(parent, child) for child in children) + return children_in_dir + + @staticmethod + def filter_by_extension(files: List[str], extension: str) -> List[str]: + """ + Filter files by extension. + + :param files: list of file paths + :param extension: file extension + :return: filtered list of file paths + """ + return [f for f in files if f.endswith(extension)] + + @staticmethod + def filter_out_non_files(files: List[str]) -> List[str]: + """ + Filter out non files. + + :param files: list of paths + :return: filtered list of file paths + """ + return [f for f in files if os.path.isfile(f)] + + @staticmethod + def remove_empty(files: List[str]) -> None: + """ + Delete files that are empty. + + :param files: list of paths + """ + for file in [f for f in files if os.stat(f).st_size == 0]: + os.remove(file) + + @staticmethod + def remove_all(files: List[str]) -> None: + """ + Delete all files. + + :param files: list of paths + """ + for file in files: + os.remove(file) + + @staticmethod + def check_path_exists(path: str) -> bool: + """ + Check that file or directory exists. + + :param path: file or directory path + :return: whether path exists + """ + return os.path.exists(path) + + +class SymbolizerService: + """Wrapper around symbolizer script.""" + + @staticmethod + def run_symbolizer_script(full_file_path: str, retry_timeout_secs: int) -> str: + """ + Symbolize given file and return symbolized output as string. + + :param full_file_path: stacktrace file path + :param retry_timeout_secs: the timeout for symbolizer to retry + :return: symbolized output as string + """ + + symbolizer_args = [ + "db-contrib-tool", + "symbolize", + "--client-secret", + _config.SYMBOLIZER_CLIENT_SECRET, + "--client-id", + _config.SYMBOLIZER_CLIENT_ID, + "--total-seconds-for-retries", + str(retry_timeout_secs), + ] + + with open(full_file_path) as file_obj: + symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, + stdin=file_obj, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + try: + output, _ = symbolizer_process.communicate(timeout=retry_timeout_secs) + except subprocess.TimeoutExpired: + symbolizer_process.kill() + output, _ = symbolizer_process.communicate() + + return output.strip().decode() diff --git a/buildscripts/resmokelib/utils/check_has_tag.py b/buildscripts/resmokelib/utils/check_has_tag.py index aebdd5644fe..18ae19bc21b 100755 --- a/buildscripts/resmokelib/utils/check_has_tag.py +++ b/buildscripts/resmokelib/utils/check_has_tag.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """CLI interface for jscomment.""" +import re import sys import jscomment @@ -16,11 +17,21 @@ try: print(' 3 if any error happened during check') print('Usage:') print(' check_has_tag.py <jsfile> <tag>') + print('Notice: <tag> is a regex, not search string') sys.exit(2) else: tags = jscomment.get_tags(sys.argv[1]) print(sys.argv[1], "has tags:", tags) - sys.exit(0 if sys.argv[2] in tags else 1) + + prog = re.compile(sys.argv[2]) + for tag in tags: + if prog.match(tag): + print("tag matches:", tag) + sys.exit(0) + + print("no tags match", sys.argv[2]) + sys.exit(1) + except Exception as err: # pylint: disable=W0703 print(err) sys.exit(3) diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py index 5857870e627..6153349b6af 100644 --- a/buildscripts/resmokelib/utils/globstar.py +++ b/buildscripts/resmokelib/utils/globstar.py @@ -1,11 +1,9 @@ """Filename globbing utility.""" import glob as _glob -import os import os.path import re -_GLOBSTAR = "**" _CONTAINS_GLOB_PATTERN = re.compile("[*?[]") @@ -35,155 +33,6 @@ def iglob(globbed_pathname): expanded to match zero or more subdirectories. """ - parts = _split_path(globbed_pathname) - parts = _canonicalize(parts) - - index = _find_globstar(parts) - if index == -1: - for pathname in _glob.iglob(globbed_pathname): - # Normalize 'pathname' so exact string comparison can be used later. - yield os.path.normpath(pathname) - return - - # **, **/, or **/a - if index == 0: - expand = _expand_curdir - - # a/** or a/**/ or a/**/b - else: - expand = _expand - - prefix_parts = parts[:index] - suffix_parts = parts[index + 1:] - - prefix = os.path.join(*prefix_parts) if prefix_parts else os.curdir - suffix = os.path.join(*suffix_parts) if suffix_parts else "" - - for (kind, path) in expand(prefix): - if not suffix_parts: - yield path - - # Avoid following symlinks to avoid an infinite loop - elif suffix_parts and kind == "dir" and not os.path.islink(path): - path = os.path.join(path, suffix) - for pathname in iglob(path): - yield pathname - - -def _split_path(pathname): - """Return 'pathname' as a list of path components.""" - - parts = [] - - while True: - (dirname, basename) = os.path.split(pathname) - parts.append(basename) - if pathname == dirname: - parts.append(dirname) - break - if not dirname: - break - pathname = dirname - - parts.reverse() - return parts - - -def _canonicalize(parts): - """Return a copy of 'parts' with consecutive "**"s coalesced. - - Raise a ValueError for unsupported uses of "**". - """ - - res = [] - - prev_was_globstar = False - for part in parts: - if part == _GLOBSTAR: - # Skip consecutive **'s - if not prev_was_globstar: - prev_was_globstar = True - res.append(part) - elif _GLOBSTAR in part: # a/b**/c or a/**b/c - raise ValueError("Can only specify glob patterns of the form a/**/b") - else: - prev_was_globstar = False - res.append(part) - - return res - - -def _find_globstar(parts): - """Return the index of the first occurrence of "**" in 'parts'. - - Return -1 if "**" is not found in the list. - """ - - for (idx, part) in enumerate(parts): - if part == _GLOBSTAR: - return idx - return -1 - - -def _list_dir(pathname): - """Return a pair of subdirectory names and filenames contained within the 'pathname' directory. - - If 'pathname' does not exist, then None is returned. - """ - - try: - (_root, dirs, files) = next(os.walk(pathname)) - return (dirs, files) - except StopIteration: - return None # 'pathname' directory does not exist - - -def _expand(pathname): - """Emit tuples of the form ("dir", dirname) and ("file", filename). - - The result is for all directories and files contained within the 'pathname' directory. - """ - - res = _list_dir(pathname) - if res is None: - return - - (dirs, files) = res - - # Zero expansion - if os.path.basename(pathname): - yield ("dir", os.path.join(pathname, "")) - - for fname in files: - path = os.path.join(pathname, fname) - yield ("file", path) - - for dname in dirs: - path = os.path.join(pathname, dname) - for xpath in _expand(path): - yield xpath - - -def _expand_curdir(pathname): - """Emit tuples of the form ("dir", dirname) and ("file", filename). - - The result is for all directories and files contained within the 'pathname' directory. - - The returned pathnames omit a "./" prefix. - """ - - res = _list_dir(pathname) - if res is None: - return - - (dirs, files) = res - - # Zero expansion - yield ("dir", "") - - for fname in files: - yield ("file", fname) - - for dname in dirs: - for xdir in _expand(dname): - yield xdir + for pathname in _glob.iglob(globbed_pathname, recursive=True): + # Normalize 'pathname' so exact string comparison can be used later. + yield os.path.normpath(pathname) diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py index 7af28d11ed8..d7c2c295492 100644 --- a/buildscripts/resmokelib/utils/jscomment.py +++ b/buildscripts/resmokelib/utils/jscomment.py @@ -36,6 +36,18 @@ def get_tags(pathname): tags = yaml.safe_load(_strip_jscomments(match.group(1))) if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) + + for tag in tags: + if '//' in tag: + raise ValueError(("Found a JS line comment '%s'. "\ + "Use '#' YAML style comments instead in a tags array %s") + % (tag, pathname)) + + if ' ' in tag: + raise ValueError(("Found an empty space in tag '%s'. "\ + "This is not permitted and may indicate a missing comma in %s") + % (tag, pathname)) + return tags except yaml.YAMLError as err: raise ValueError( |