diff options
Diffstat (limited to 'buildscripts')
134 files changed, 3114 insertions, 1925 deletions
diff --git a/buildscripts/benchmarks/analyze.py b/buildscripts/benchmarks/analyze.py index 924db911955..d8bbefcaaf4 100644 --- a/buildscripts/benchmarks/analyze.py +++ b/buildscripts/benchmarks/analyze.py @@ -26,15 +26,24 @@ def get_baseline_task_id(evg_api_config: str, current_task_id: str) -> str: for version in prior_versions: baseline_task_candidate = None - build = version.build_by_variant(current_task.build_variant) + + try: + build = version.build_by_variant(current_task.build_variant) + # If previous version doesn't contain the build variant of the current task + # assume that it's not possible to find the baseline task + except KeyError: + return "" + for task in build.get_tasks(): if task.display_name == current_task.display_name: baseline_task_candidate = task break + # If previous build doesn't contain the task with the same 'display_name' # assume that this is a new task and there is no baseline task if baseline_task_candidate is None: return "" + if baseline_task_candidate.is_success(): return baseline_task_candidate.task_id diff --git a/buildscripts/blackduck_hub.py b/buildscripts/blackduck_hub.py index 111f0d7b03c..324b8f37d53 100644 --- a/buildscripts/blackduck_hub.py +++ b/buildscripts/blackduck_hub.py @@ -70,11 +70,15 @@ BUILD_LOGGER_APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" BUILD_LOGGER_CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" BUILD_LOGGER_APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" -BUILD_LOGGER_DEFAULT_URL = "https://logkeeper.mongodb.org" +BUILD_LOGGER_DEFAULT_URL = "https://logkeeper2.build.10gen.cc" BUILD_LOGGER_TIMEOUT_SECS = 65 LOCAL_REPORTS_DIR = "bd_reports" +# Detect script constants +BLACKDUCK_DETECT_VERSION = "8" +BLACKDUCK_DETECT_CHECKSUM = "cc7daaf95e79bd06f249eb3d8a3413ec02e38db0b3388099becb72b13c110fd9" + ############################################################################ THIRD_PARTY_DIRECTORIES = [ @@ -86,7 +90,7 @@ THIRD_PARTY_COMPONENTS_FILE = "etc/third_party_components.yml" ############################################################################ -RE_LETTERS = re.compile("[A-Za-z]{2,}") +RE_LETTER = re.compile("[A-Za-z]") def default_if_none(value, default): @@ -331,8 +335,9 @@ class VersionInfo: if self.ver_str.endswith('-'): self.ver_str = self.ver_str[0:-1] - # Boost keeps varying the version strings so filter for anything with 2 or more ascii charaters - if RE_LETTERS.search(self.ver_str): + # Boost keeps varying the version strings so filter for anything with a letter + # Safeint has "3.0.26c" as a version number + if RE_LETTER.search(self.ver_str): self.production_version = False return @@ -558,8 +563,7 @@ class BlackDuckConfig: rc = json.loads(rfh.read()) self.url = rc["baseurl"] - self.username = rc["username"] - self.password = rc["password"] + self.token = rc["token"] def _run_scan(): @@ -568,7 +572,9 @@ def _run_scan(): with tempfile.NamedTemporaryFile() as fp: fp.write(f"""#/!bin/sh -curl --retry 5 -s -L https://detect.synopsys.com/detect.sh | bash -s -- --blackduck.url={bdc.url} --blackduck.username={bdc.username} --blackduck.password={bdc.password} --detect.report.timeout={BLACKDUCK_TIMEOUT_SECS} --snippet-matching --upload-source --detect.wait.for.results=true +curl --retry 5 -s -L -O https://detect.synopsys.com/detect{BLACKDUCK_DETECT_VERSION}.sh +echo "{BLACKDUCK_DETECT_CHECKSUM} detect{BLACKDUCK_DETECT_VERSION}.sh" | sha256sum --check && \ +bash detect{BLACKDUCK_DETECT_VERSION}.sh --blackduck.url={bdc.url} --blackduck.api.token={bdc.token} --detect.report.timeout={BLACKDUCK_TIMEOUT_SECS} --snippet-matching --upload-source --detect.wait.for.results=true --logging.level.detect=TRACE --detect.diagnostic=true --detect.cleanup=false """.encode()) fp.flush() diff --git a/buildscripts/build_system_options.py b/buildscripts/build_system_options.py index 66a303bf320..140d1e4dffd 100644 --- a/buildscripts/build_system_options.py +++ b/buildscripts/build_system_options.py @@ -9,7 +9,7 @@ class PathOptions: shared_library_folder_name = "lib" # extend the below list if there are new types of shared libraries - _shared_library_file_patterns = [r".+\.so(\.\d{1,})?$", r".+\.dylib$"] + _shared_library_file_patterns = [r".+\.so(\.(\d{1,}|debug))?$", r".+\.dylib$"] _compiled_shared_library_file_patterns = None @property diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py index 5acb2e75861..3e391fc1b11 100644 --- a/buildscripts/burn_in_tags.py +++ b/buildscripts/burn_in_tags.py @@ -92,7 +92,7 @@ def _create_evg_build_variant_map(expansions_file_data): :param expansions_file_data: Config data file to use. :return: Map of base buildvariants to their generated buildvariants. """ - burn_in_tag_build_variants = expansions_file_data["burn_in_tag_buildvariants"] + burn_in_tag_build_variants = expansions_file_data["burn_in_tag_include_build_variants"] if burn_in_tag_build_variants: return { @@ -160,7 +160,7 @@ def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject repeat_tests_max=config_options.repeat_tests_max, repeat_tests_secs=config_options.repeat_tests_secs) - burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config, evergreen_api) + burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config) burn_in_generator.generate_tasks_for_variant(tests_by_task, shrub_build_variant) shrub_project.add_build_variant(shrub_build_variant) diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 27da0996b63..2831b1a806c 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 """Command line utility for determining what jstests have been added or modified.""" +import collections import copy +import json import logging import os.path import shlex @@ -13,6 +15,7 @@ from typing import Optional, Set, Tuple, List, Dict, NamedTuple import click import yaml from git import Repo +from pydantic import BaseModel import structlog from structlog.stdlib import LoggerFactory @@ -41,12 +44,14 @@ DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" ENTERPRISE_MODULE_PATH = "src/mongo/db/modules/enterprise" DEFAULT_REPO_LOCATIONS = [".", f"./{ENTERPRISE_MODULE_PATH}"] REPEAT_SUITES = 2 -EVERGREEN_FILE = "etc/evergreen.yml" +DEFAULT_EVG_PROJECT_FILE = "etc/evergreen.yml" # The executor_file and suite_files defaults are required to make the suite resolver work # correctly. SELECTOR_FILE = "etc/burn_in_tests.yml" SUITE_FILES = ["with_server"] +BURN_IN_TEST_MEMBERSHIP_FILE = "burn_in_test_membership_map_file_for_ci.json" + SUPPORTED_TEST_KINDS = ("fsm_workload_test", "js_test", "json_schema_test", "multi_stmt_txn_passthrough", "parallel_fsm_workload_test", "all_versions_js_test") @@ -181,7 +186,13 @@ def create_executor_list(suites, exclude_suites): parameter. Returns a dict keyed by suite name / executor, value is tests to run under that executor. """ - test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) + try: + with open(BURN_IN_TEST_MEMBERSHIP_FILE) as file: + test_membership = collections.defaultdict(list, json.load(file)) + LOGGER.info(f"Using cached test membership file {BURN_IN_TEST_MEMBERSHIP_FILE}.") + except FileNotFoundError: + LOGGER.info("Getting test membership data.") + test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) memberships = defaultdict(list) for suite in suites: @@ -352,7 +363,8 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, - changed_tests: Set[str], install_dir: str) -> Dict[str, TaskInfo]: + changed_tests: Set[str], + install_dir: Optional[str]) -> Dict[str, TaskInfo]: """ Create a list of tests by task. @@ -367,7 +379,11 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*") changed_tests = filter_tests(changed_tests, exclude_tests) - buildscripts.resmokelib.parser.set_run_options(f"--installDir={shlex.quote(install_dir)}") + run_options = "" + if install_dir is not None: + run_options = f"--installDir={shlex.quote(install_dir)}" + buildscripts.resmokelib.parser.set_run_options(run_options) + if changed_tests: return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites, exclude_tasks) @@ -408,7 +424,7 @@ def _configure_logging(verbose: bool): logging.basicConfig( format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", level=level, - stream=sys.stdout, + stream=sys.stderr, ) for log_name in EXTERNAL_LOGGERS: logging.getLogger(log_name).setLevel(logging.WARNING) @@ -537,11 +553,45 @@ class LocalBurnInExecutor(BurnInExecutor): run_tests(tests_by_task, resmoke_cmd) +class DiscoveredTask(BaseModel): + """ + Model for a discovered task to run. + + * task_name: Name of discovered task. + * test_list: List of tests to run under discovered task. + """ + + task_name: str + test_list: List[str] + + +class DiscoveredTaskList(BaseModel): + """Model for a list of discovered tasks.""" + + discovered_tasks: List[DiscoveredTask] + + +class YamlBurnInExecutor(BurnInExecutor): + """A burn-in executor that outputs discovered tasks as YAML.""" + + def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + """ + Report the given tasks and their tests to stdout. + + :param tests_by_task: Dictionary of tasks to run with tests to run in each. + """ + discovered_tasks = DiscoveredTaskList(discovered_tasks=[ + DiscoveredTask(task_name=task_name, test_list=task_info.tests) + for task_name, task_info in tests_by_task.items() + ]) + print(yaml.safe_dump(discovered_tasks.dict())) + + class BurnInOrchestrator: """Orchestrate the execution of burn_in_tests.""" def __init__(self, change_detector: FileChangeDetector, burn_in_executor: BurnInExecutor, - evg_conf: EvergreenProjectConfig) -> None: + evg_conf: EvergreenProjectConfig, install_dir: Optional[str]) -> None: """ Create a new orchestrator. @@ -552,8 +602,9 @@ class BurnInOrchestrator: self.change_detector = change_detector self.burn_in_executor = burn_in_executor self.evg_conf = evg_conf + self.install_dir = install_dir - def burn_in(self, repos: List[Repo], build_variant: str, install_dir: str) -> None: + def burn_in(self, repos: List[Repo], build_variant: str) -> None: """ Execute burn in tests for the given git repositories. @@ -564,14 +615,19 @@ class BurnInOrchestrator: LOGGER.info("Found changed tests", files=changed_tests) tests_by_task = create_tests_by_task(build_variant, self.evg_conf, changed_tests, - install_dir) + self.install_dir) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) self.burn_in_executor.execute(tests_by_task) -# pylint: disable=too-many-function-args -@click.command(context_settings=dict(ignore_unknown_options=True)) +@click.group() +def cli(): + """Run the cli.""" + pass + + +@cli.command(context_settings=dict(ignore_unknown_options=True)) @click.option("--no-exec", "no_exec", default=False, is_flag=True, help="Do not execute the found tests.") @click.option("--build-variant", "build_variant", default=DEFAULT_VARIANT, metavar='BUILD_VARIANT', @@ -584,18 +640,22 @@ class BurnInOrchestrator: help="The maximum number of times to repeat tests if time option is specified.") @click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS", help="Repeat tests for the given time (in secs).") +@click.option("--yaml", "use_yaml", is_flag=True, default=False, + help="Output discovered tasks in YAML. Tests will not be run.") @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") @click.option( "--origin-rev", "origin_rev", default=None, help="The revision in the mongo repo that changes will be compared against if specified.") -@click.option("--install-dir", "install_dir", required=True, type=str, +@click.option("--install-dir", "install_dir", type=str, help="Path to bin directory of a testable installation") +@click.option("--evg-project-file", "evg_project_file", default=DEFAULT_EVG_PROJECT_FILE, + help="Evergreen project config file") @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED) -# pylint: disable=too-many-arguments,too-many-locals -def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], - repeat_tests_min: Optional[int], repeat_tests_max: Optional[int], - repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool, - origin_rev: Optional[str], install_dir: str) -> None: +def run(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], + repeat_tests_min: Optional[int], repeat_tests_max: Optional[int], + repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool, + origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool, + evg_project_file: Optional[str]) -> None: """ Run new or changed tests in repeated mode to validate their stability. @@ -626,6 +686,9 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], :param resmoke_args: Arguments to pass through to resmoke. :param verbose: Log extra debug information. :param origin_rev: The revision that local changes will be compared against. + :param install_dir: Path to bin directory of a testable installation. + :param use_yaml: Output discovered tasks in YAML. Tests will not be run. + :param evg_project_file: Evergreen project config file. """ _configure_logging(verbose) @@ -635,16 +698,40 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], repeat_tests_num=repeat_tests_num) # yapf: disable repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] - evg_conf = parse_evergreen_file(EVERGREEN_FILE) + evg_conf = parse_evergreen_file(evg_project_file) change_detector = LocalFileChangeDetector(origin_rev) executor = LocalBurnInExecutor(resmoke_args, repeat_config) - if no_exec: + if use_yaml: + executor = YamlBurnInExecutor() + elif no_exec: executor = NopBurnInExecutor() - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) - burn_in_orchestrator.burn_in(repos, build_variant, install_dir) + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) + burn_in_orchestrator.burn_in(repos, build_variant) + + +@cli.command() +def generate_test_membership_map_file_for_ci(): + """ + Generate a file to cache test membership data for CI. + + This command should only be used in CI. The task generator runs many iterations of this script + for many build variants. The bottleneck is that creating the test membership file takes a long time. + Instead, we can cache this data & reuse it in CI for a significant speedup. + + Run this command in CI before running the burn in task generator. + """ + _configure_logging(False) + buildscripts.resmokelib.parser.set_run_options() + + LOGGER.info("Generating burn_in test membership mapping file.") + test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) + with open(BURN_IN_TEST_MEMBERSHIP_FILE, "w") as file: + json.dump(test_membership, file) + LOGGER.info( + f"Finished writing burn_in test membership mapping to {BURN_IN_TEST_MEMBERSHIP_FILE}") if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter + cli() diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index fb63b2201ec..8f331b5e1ee 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -320,7 +320,7 @@ class Variant(object): def is_required_variant(self) -> bool: """Return True if the variant is a required variant.""" - return self.display_name.startswith("! ") + return self.display_name.startswith("!") def get_task(self, task_name): """Return the task with the given name as an instance of VariantTask. diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py index 8952a564943..dd5564f4454 100755 --- a/buildscripts/combine_reports.py +++ b/buildscripts/combine_reports.py @@ -25,7 +25,7 @@ def report_exit(combined_test_report): """Return report exit code. The exit code of this script is based on the following: - 0: All tests have status "pass", or only non-dynamic tests have status "silentfail". + 0: All tests have status "pass". 31: At least one test has status "fail" or "timeout". Note: A test can be considered dynamic if its name contains a ":" character. """ diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py index 977716a7c0f..0f79b4349fe 100644 --- a/buildscripts/debugsymb_mapper.py +++ b/buildscripts/debugsymb_mapper.py @@ -4,11 +4,13 @@ import json import logging import os import pathlib +import re import shutil import subprocess import sys import time -import typing +from json import JSONDecoder +from typing import Optional, Tuple, Generator, Dict, List, NamedTuple import requests @@ -20,30 +22,95 @@ from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download from buildscripts.build_system_options import PathOptions +BUILD_INFO_RE = re.compile(r"Build Info: ({(\n.*)*})") +MONGOD = "mongod" -class LinuxBuildIDExtractor: - """Parse readlef command output & extract Build ID.""" - default_executable_path = "readelf" +class CmdClient: + """Client to run commands.""" - def __init__(self, executable_path: str = None): - """Initialize instance.""" + @staticmethod + def run(args: List[str]) -> str: + """ + Run command with args. + + :param args: Argument list. + :return: Command output. + """ + + out = subprocess.run(args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False) + return out.stdout.strip().decode() + + +class BuildIdOutput(NamedTuple): + """ + Build ID and command output. + + * build_id: Build ID or None. + * cmd_output: Command output. + """ + + build_id: Optional[str] + cmd_output: str + + +class BinVersionOutput(NamedTuple): + """ + Mongodb bin version and command output. + + * mongodb_version: Bin version. + * cmd_output: Command output. + """ + + mongodb_version: Optional[str] + cmd_output: str + + +class CmdOutputExtractor: + """Data extractor from command output.""" + + def __init__(self, cmd_client: Optional[CmdClient] = None, + json_decoder: Optional[JSONDecoder] = None) -> None: + """ + Initialize. + + :param cmd_client: Client to run commands. + :param json_decoder: JSONDecoder object. + """ + self.cmd_client = cmd_client if cmd_client is not None else CmdClient() + self.json_decoder = json_decoder if json_decoder is not None else JSONDecoder() - self.executable_path = executable_path or self.default_executable_path + def get_build_id(self, bin_path: str) -> BuildIdOutput: + """ + Get build ID from readelf command. - def callreadelf(self, binary_path: str) -> str: - """Call readelf command for given binary & return string output.""" + :param bin_path: Path to binary of the build. + :return: Build ID or None and command output. + """ + out = self.cmd_client.run(["readelf", "-n", bin_path]) + build_id = self._extract_build_id(out) + return BuildIdOutput(build_id, out) - args = [self.executable_path, "-n", binary_path] - process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - process.wait() - return process.stdout.read().decode() + def get_bin_version(self, bin_path: str) -> BinVersionOutput: + """ + Get mongodb bin version from `{bin} --version` command. + + :param bin_path: Path to mongodb binary. + :return: Bin version or None and command output. + """ + out = self.cmd_client.run([os.path.abspath(bin_path), "--version"]) + mongodb_version = self._get_mongodb_version(out) + return BinVersionOutput(mongodb_version, out) @staticmethod - def extractbuildid(out: str) -> typing.Optional[str]: - """Parse readelf output and extract Build ID from it.""" + def _extract_build_id(out: str) -> Optional[str]: + """ + Parse readelf output and extract Build ID from it. + :param out: readelf command output. + :return: Build ID on None. + """ build_id = None for line in out.splitlines(): line = line.strip() @@ -53,13 +120,21 @@ class LinuxBuildIDExtractor: build_id = line.split(': ')[1] return build_id - def run(self, binary_path: str) -> typing.Tuple[str, str]: - """Perform all necessary actions to get Build ID.""" + def _get_mongodb_version(self, out: str) -> Optional[str]: + """ + Parse version command output and extract mongodb version. + + :param out: Version command output. + :return: Version or None. + """ + mongodb_version = None - readelfout = self.callreadelf(binary_path) - buildid = self.extractbuildid(readelfout) + search = BUILD_INFO_RE.search(out) + if search: + build_info = self.json_decoder.decode(search.group(1)) + mongodb_version = build_info.get("version") - return buildid, readelfout + return mongodb_version class DownloadOptions(object): @@ -83,24 +158,29 @@ class Mapper: default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com" default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache') - selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug') + selected_binaries = ('mongos', 'mongod', 'mongo') default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') - def __init__(self, version: str, client_id: str, client_secret: str, variant: str, - cache_dir: str = None, web_service_base_url: str = None, + def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str, + client_secret: str, cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. - :param version: version string - :param variant: build variant string - :param cache_dir: full path to cache directory as a string - :param web_service_base_url: URL of symbolizer web service + :param evg_version: Evergreen version ID. + :param evg_variant: Evergreen build variant name. + :param is_san_variant: Whether build variant is sanitizer build. + :param client_id: Client id for Okta Oauth. + :param client_secret: Secret key for Okta Oauth. + :param cache_dir: Full path to cache directory as a string. + :param web_service_base_url: URL of symbolizer web service. + :param logger: Debug symbols mapper logger. """ - self.version = version - self.variant = variant + self.evg_version = evg_version + self.evg_variant = evg_variant + self.is_san_variant = is_san_variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url @@ -113,8 +193,8 @@ class Mapper: self.http_client = requests.Session() self.multiversion_setup = SetupMultiversion( - DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant, - ignore_failed_push=True) + DownloadOptions(download_symbols=True, download_binaries=True), + variant=self.evg_variant, ignore_failed_push=True) self.debug_symbols_url = None self.url = None self.configs = Configs( @@ -139,7 +219,7 @@ class Mapper: data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get("expire_time") if time.time() < expire_time: - # credentials hasn't expired yet + # credentials haven't expired yet self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) return @@ -147,7 +227,7 @@ class Mapper: configs=self.configs) self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) - # write credentials to local file for further useage + # write credentials to local file for further usage with open(self.default_creds_file_path, "w") as cfile: cfile.write( json.dumps({ @@ -184,17 +264,19 @@ class Mapper: def setup_urls(self): """Set up URLs using multiversion.""" - urlinfo = self.multiversion_setup.get_urls(self.version, self.variant) + urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant) - download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) binaries_url = urlinfo.urls.get("Binaries", "") - - if not download_symbols_url: - download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None) + if self.is_san_variant: + # Sanitizer builds are not stripped and contain debug symbols + download_symbols_url = binaries_url + else: + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get( + "mongo-debugsymbols.zip") if not download_symbols_url: self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", - self.version, urlinfo.urls) + self.evg_version, urlinfo.urls) raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}") self.debug_symbols_url = download_symbols_url @@ -233,54 +315,64 @@ class Mapper: tarball_full_path = download.download_from_s3(url) return tarball_full_path - def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]: + def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]: """ Extract build id from binaries and creates new dict using them. :return: mapped data as dict """ - readelf_extractor = LinuxBuildIDExtractor() - - debug_symbols_path = self.download(self.debug_symbols_url) - debug_symbols_unpacked_path = self.unpack(debug_symbols_path) + extractor = CmdOutputExtractor() binaries_path = self.download(self.url) binaries_unpacked_path = self.unpack(binaries_path) - # we need to analyze two directories: main binary folder inside debug-symbols and + # we need to analyze two directories: main binary folder and # shared libraries folder inside binaries. # main binary folder holds main binaries, like mongos, mongod, mongo ... # shared libraries folder holds shared libraries, tons of them. # some build variants do not contain shared libraries. - debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test') binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test') - self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s", - os.listdir(debug_symbols_unpacked_path)) self.logger.info("INSIDE unpacked binaries/dist-test: %s", os.listdir(binaries_unpacked_path)) + mongod_bin = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name, + MONGOD) + bin_version_output = extractor.get_bin_version(mongod_bin) + + if bin_version_output.mongodb_version is None: + self.logger.error("mongodb version could not be extracted. \n`%s --version` output: %s", + mongod_bin, bin_version_output.cmd_output) + return + else: + self.logger.info("Extracted mongodb version: %s", bin_version_output.mongodb_version) + # start with main binary folder for binary in self.selected_binaries: - full_bin_path = os.path.join(debug_symbols_unpacked_path, + full_bin_path = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name, binary) if not os.path.exists(full_bin_path): self.logger.error("Could not find binary at %s", full_bin_path) - return + continue - build_id, readelf_out = readelf_extractor.run(full_bin_path) + build_id_output = extractor.get_build_id(full_bin_path) - if not build_id: + if not build_id_output.build_id: self.logger.error("Build ID couldn't be extracted. \nReadELF output %s", - readelf_out) - return + build_id_output.cmd_output) + continue + else: + self.logger.info("Extracted build ID: %s", build_id_output.build_id) yield { - 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, - 'file_name': binary, 'version': self.version + 'url': self.url, + 'debug_symbols_url': self.debug_symbols_url, + 'build_id': build_id_output.build_id, + 'file_name': binary, + 'version': bin_version_output.mongodb_version, } # move to shared libraries folder. @@ -304,20 +396,23 @@ class Mapper: if not os.path.exists(sofile_path): self.logger.error("Could not find binary at %s", sofile_path) - return + continue - build_id, readelf_out = readelf_extractor.run(sofile_path) + build_id_output = extractor.get_build_id(sofile_path) - if not build_id: - self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out) - return + if not build_id_output.build_id: + self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", + build_id_output.cmd_output) + continue + else: + self.logger.info("Extracted build ID: %s", build_id_output.build_id) yield { 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, - 'build_id': build_id, + 'build_id': build_id_output.build_id, 'file_name': sofile, - 'version': self.version, + 'version': bin_version_output.mongodb_version, } def run(self): @@ -330,6 +425,7 @@ class Mapper: # mappings is a generator, we iterate over to generate mappings on the go for mapping in mappings: + self.logger.info("Creating mapping %s", mapping) response = self.http_client.post('/'.join((self.web_service_base_url, 'add')), json=mapping) if response.status_code != 200: @@ -344,18 +440,20 @@ def make_argument_parser(parser=None, **kwargs): if parser is None: parser = argparse.ArgumentParser(**kwargs) - parser.add_argument('--version') - parser.add_argument('--client-id') - parser.add_argument('--client-secret') - parser.add_argument('--variant') - parser.add_argument('--web-service-base-url', default="") + parser.add_argument("--version") + parser.add_argument("--client-id") + parser.add_argument("--client-secret") + parser.add_argument("--variant") + parser.add_argument("--is-san-variant", action="store_true") + parser.add_argument("--web-service-base-url", default="") return parser def main(options): """Execute mapper here. Main entry point.""" - mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id, + mapper = Mapper(evg_version=options.version, evg_variant=options.variant, + is_san_variant=options.is_san_variant, client_id=options.client_id, client_secret=options.client_secret, web_service_base_url=options.web_service_base_url) diff --git a/buildscripts/download_sys_perf_binaries.py b/buildscripts/download_sys_perf_binaries.py new file mode 100644 index 00000000000..f94a328fd7b --- /dev/null +++ b/buildscripts/download_sys_perf_binaries.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Download the binaries from a previous sys-perf run.""" + +import argparse +import requests + +BASE_URI = 'https://evergreen.mongodb.com/rest/v2/' + + +def _get_auth_headers(evergreen_api_user, evergreen_api_key): + return { + 'Api-User': evergreen_api_user, + 'Api-Key': evergreen_api_key, + } + + +def _get_build_id(build_variant_name, version_id, auth_headers): + url = BASE_URI + 'versions/' + version_id + response = requests.get(url, headers=auth_headers) + if response.status_code != 200: + raise ValueError('Invalid version_id:', version_id) + + version_json = response.json() + build_variants = version_json['build_variants_status'] + for build_variant in build_variants: + if build_variant['build_variant'] == build_variant_name: + return build_variant['build_id'] + + raise RuntimeError('The compile-variant ' + build_variant_name + + ' does not exist for the build with version_id ' + version_id) + + +# All of our sys-perf compile variants have exactly one task, +# so we can safely select the first one from the build variants tasks. +def _get_task_id(build_id, auth_headers): + url = BASE_URI + 'builds/' + build_id + response = requests.get(url, headers=auth_headers) + if response.status_code != 200: + raise RuntimeError('Unexpected error when trying to reach' + url) + + build_json = response.json() + task_list = build_json['tasks'] + if len(task_list) != 1: + raise RuntimeError('Recieved unexpected tasklist:', task_list) + return task_list[0] + + +# The API used here always grabs the latest execution +def _get_binary_details(task_id, auth_headers): + url = BASE_URI + 'tasks/' + task_id + response = requests.get(url, headers=auth_headers) + if response.status_code != 200: + raise RuntimeError('Unexpected error when trying to reach' + url) + + task_json = response.json() + if task_json['status'] != 'success': + raise RuntimeError('The task ' + task_id + ' did not run sucessfully') + + # The binary will always be the first artifact, unless we make large changes to system_perf.yml + artifacts = task_json['artifacts'] + if len(artifacts) > 0 and artifacts[0]['name'].startswith('mongo'): + return artifacts[0] + raise RuntimeError('Unexpected list of artifacts:' + artifacts) + + +def _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key): + auth_headers = _get_auth_headers(evergreen_api_user, evergreen_api_key) + build_id = _get_build_id(build_variant, version_id, auth_headers) + task_id = _get_task_id(build_id, auth_headers) + binary_json = _get_binary_details(task_id, auth_headers) + return binary_json['url'] + + +def _download_binary_file(url, save_path): + response = requests.get(url, stream=True) + if response.status_code == 200: + with open(save_path, 'wb') as file: + for chunk in response.iter_content(chunk_size=1024): + file.write(chunk) + else: + raise RuntimeError('Failed to download the file ' + url) + + +def _download_sys_perf_binaries(version_id, build_variant, evergreen_api_user, evergreen_api_key): + url = _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key) + _download_binary_file(url, 'binary.tar.gz') + + +if __name__ == '__main__': + argParser = argparse.ArgumentParser() + argParser.add_argument("-v", "--version_id", + help="Evergreen version_id from which binaries will be downloaded") + argParser.add_argument("-b", "--build_variant", + help="Build variant for which binaries will be downloaded") + argParser.add_argument( + "-u", "--evergreen_api_user", + help="Evergreen API user, see https://spruce.mongodb.com/preferences/cli") + argParser.add_argument("-k", "--evergreen_api_key", + help="Evergreen API key, see https://spruce.mongodb.com/preferences/cli") + args = argParser.parse_args() + + _download_sys_perf_binaries(args.version_id, args.build_variant, args.evergreen_api_user, + args.evergreen_api_key) diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py index 8d9330b4240..bcabaa2e9cc 100755 --- a/buildscripts/errorcodes.py +++ b/buildscripts/errorcodes.py @@ -46,6 +46,7 @@ _CODE_PATTERNS = [ r"(?:StatusOK)?" r"(?:WithContext)?" r"\s*\(", + r"MONGO_UNREACHABLE_TASSERT\(", # DBException and AssertionException constructors r"(?:DB|Assertion)Exception\s*[({]", # Calls to all LOGV2* variants diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py index 07cbb0ddc5c..b4d1d97df5c 100755 --- a/buildscripts/eslint.py +++ b/buildscripts/eslint.py @@ -19,12 +19,13 @@ import sys import tarfile import tempfile import threading +import platform from typing import Optional import urllib.error import urllib.parse import urllib.request -from distutils import spawn # pylint: disable=no-name-in-module +from distutils import spawn from optparse import OptionParser import structlog @@ -50,14 +51,17 @@ ESLINT_VERSION = "7.22.0" # Name of ESLint as a binary. ESLINT_PROGNAME = "eslint" +# Arch of running system +ARCH = platform.machine() if platform.machine() != "aarch64" else "arm64" + # URL location of our provided ESLint binaries. ESLINT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ - ESLINT_VERSION + "-linux.tar.gz" + ESLINT_VERSION + "-linux-" + ARCH + ".tar.gz" ESLINT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ ESLINT_VERSION + "-darwin.tar.gz" # Path in the tarball to the ESLint binary. -ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$platform-$arch") +ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$operating_system-$arch") LOGGER = structlog.get_logger(__name__) @@ -76,15 +80,15 @@ def extract_eslint(tar_path, target_file): tarfp.close() -def get_eslint_from_cache(dest_file, platform, arch): +def get_eslint_from_cache(dest_file, operating_system, arch): """Get ESLint binary from mongodb's cache.""" # Get URL - if platform == "Linux": + if operating_system == "Linux": url = ESLINT_HTTP_LINUX_CACHE - elif platform == "Darwin": + elif operating_system == "Darwin": url = ESLINT_HTTP_DARWIN_CACHE else: - raise ValueError('ESLint is not available as a binary for ' + platform) + raise ValueError('ESLint is not available as a binary for ' + operating_system) dest_dir = tempfile.gettempdir() temp_tar_file = os.path.join(dest_dir, "temp.tar.gz") @@ -93,9 +97,9 @@ def get_eslint_from_cache(dest_file, platform, arch): print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) urllib.request.urlretrieve(url, temp_tar_file) - # pylint: disable=too-many-function-args print("Extracting ESLint %s to %s" % (ESLINT_VERSION, dest_file)) - eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) + eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(operating_system=operating_system, + arch=arch) extract_eslint(temp_tar_file, eslint_distfile) shutil.move(eslint_distfile, dest_file) @@ -109,7 +113,7 @@ class ESLint(object): # Initialize ESLint configuration information if sys.platform.startswith("linux"): - self.arch = "x86_64" + self.arch = ARCH self.tar_path = None elif sys.platform == "darwin": self.arch = "x86_64" diff --git a/buildscripts/eslint/README.md b/buildscripts/eslint/README.md index f8f6b692dfa..d4f0d7d8402 100644 --- a/buildscripts/eslint/README.md +++ b/buildscripts/eslint/README.md @@ -21,6 +21,7 @@ "pkg": { "scripts": [ "conf/**/*", "lib/**/*", "messages/**/*" ], "targets": [ "linux-x64", "macos-x64" ] + # "targets": [ "linux-arm64" ] }, ``` 6. Run pkg command to make ESLint executables. @@ -38,6 +39,10 @@ ``` eslint-macos --help ``` + or (if you are on arm) + ``` + eslint --help + ``` (*) If executable fails to find some .js files there are [extra steps](#extra-steps) required to be done before step 6. @@ -48,19 +53,25 @@ Rename produced files. ``` mv eslint-linux eslint-Linux-x86_64 mv eslint-macos eslint-Darwin-x86_64 +# arm +# mv eslint eslint-Linux-arm64 ``` -Archive files. +Archive files. (No leading v in version e.g. 8.28.0 NOT v8.28.0) ``` -tar -czvf eslint-${version}-linux.tar.gz eslint-Linux-x86_64 +tar -czvf eslint-${version}-linux-x86_64.tar.gz eslint-Linux-x86_64 tar -czvf eslint-${version}-darwin.tar.gz eslint-Darwin-x86_64 +# arm +# tar -czvf eslint-${version}-linux-arm64.tar.gz eslint-Linux-arm64 ``` ### Upload archives to `boxes.10gen.com` Archives should be available by the following links: ``` -https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux.tar.gz +https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux-x86_64.tar.gz https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-darwin.tar.gz +# arm +# https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux-arm64.tar.gz ``` Build team has an access to do that. You can create a build ticket in Jira for them to do it @@ -77,7 +88,7 @@ ESLINT_VERSION = "${version}" Unfortunately pkg doesn't work well with `require(variable)` statements and force include files using `assets` or `scripts` options might not help. -For the ESLint version 7.22.0 the following change was applied to the +For the ESLint version 7.22.0 and 8.28.0 the following change was applied to the source code to make everything work: ``` diff --git a/lib/cli-engine/cli-engine.js b/lib/cli-engine/cli-engine.js diff --git a/buildscripts/evergreen_activate_gen_tasks.py b/buildscripts/evergreen_activate_gen_tasks.py index 13eed325701..1ccd95642c5 100755 --- a/buildscripts/evergreen_activate_gen_tasks.py +++ b/buildscripts/evergreen_activate_gen_tasks.py @@ -21,6 +21,9 @@ from buildscripts.util.taskname import remove_gen_suffix LOGGER = structlog.getLogger(__name__) EVG_CONFIG_FILE = "./.evergreen.yml" +BURN_IN_TAGS = "burn_in_tags" +BURN_IN_TESTS = "burn_in_tests" +BURN_IN_VARIANT_SUFFIX = "generated-by-burn-in-tags" class EvgExpansions(BaseModel): @@ -28,10 +31,12 @@ class EvgExpansions(BaseModel): Evergreen expansions file contents. build_id: ID of build being run. + version_id: ID of version being run. task_name: Name of task creating the generated configuration. """ build_id: str + version_id: str task_name: str @classmethod @@ -45,7 +50,7 @@ class EvgExpansions(BaseModel): return remove_gen_suffix(self.task_name) -def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: +def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None: """ Activate the given task in the specified build. @@ -53,27 +58,28 @@ def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: :param task_name: Name of task to activate. :param evg_api: Evergreen API client. """ - build = evg_api.build_by_id(build_id) - task_list = build.get_tasks() - for task in task_list: - if task.display_name == task_name: - LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) - evg_api.configure_task(task.task_id, activated=True) - - # if any(ARCHIVE_DIST_TEST_TASK in dependency["id"] for dependency in task.depends_on): - # _activate_archive_debug_symbols(evg_api, task_list) - - -# def _activate_archive_debug_symbols(evg_api: EvergreenApi, task_list): -# debug_iter = filter(lambda tsk: tsk.display_name == ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK, -# task_list) -# activate_symbol_tasks = list(debug_iter) -# -# if len(activate_symbol_tasks) == 1: -# activated_symbol_task = activate_symbol_tasks[0] -# if not activated_symbol_task.activated: -# LOGGER.info("Activating debug symbols archival", task_id=activated_symbol_task.task_id) -# evg_api.configure_task(activated_symbol_task.task_id, activated=True) + if expansions.task == BURN_IN_TAGS: + version = evg_api.version_by_id(expansions.version_id) + burn_in_build_variants = [ + variant for variant in version.build_variants_map.keys() + if variant.endswith(BURN_IN_VARIANT_SUFFIX) + ] + for build_variant in burn_in_build_variants: + build_id = version.build_variants_map[build_variant] + task_list = evg_api.tasks_by_build(build_id) + + for task in task_list: + if task.display_name == BURN_IN_TESTS: + LOGGER.info("Activating task", task_id=task.task_id, + task_name=task.display_name) + evg_api.configure_task(task.task_id, activated=True) + + else: + task_list = evg_api.tasks_by_build(expansions.build_id) + for task in task_list: + if task.display_name == expansions.task: + LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) + evg_api.configure_task(task.task_id, activated=True) @click.command() @@ -96,7 +102,7 @@ def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None: expansions = EvgExpansions.from_yaml_file(expansion_file) evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config) - activate_task(expansions.build_id, expansions.task, evg_api) + activate_task(expansions, evg_api) if __name__ == "__main__": diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py index 37a0d2874ff..bc92a2fef33 100644 --- a/buildscripts/evergreen_burn_in_tests.py +++ b/buildscripts/evergreen_burn_in_tests.py @@ -34,7 +34,6 @@ DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen" BURN_IN_TESTS_TASK = "burn_in_tests" BURN_IN_ENV_VAR = "BURN_IN_TESTS" -AVG_TEST_RUNTIME_ANALYSIS_DAYS = 14 AVG_TEST_SETUP_SEC = 4 * 60 AVG_TEST_TIME_MULTIPLIER = 3 MIN_AVG_TEST_OVERFLOW_SEC = float(60) @@ -328,23 +327,17 @@ class GenerateBurnInExecutor(BurnInExecutor): # pylint: disable=too-many-arguments def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig, - evg_api: EvergreenApi, generate_tasks_file: Optional[str] = None, - history_end_date: Optional[datetime] = None) -> None: + generate_tasks_file: Optional[str] = None) -> None: """ Create a new generate burn-in executor. :param generate_config: Configuration for how to generate tasks. :param repeat_config: Configuration for how tests should be repeated. - :param evg_api: Evergreen API client. :param generate_tasks_file: File to write generated task configuration to. - :param history_end_date: End date of range to query for historic test data. """ self.generate_config = generate_config self.repeat_config = repeat_config - self.evg_api = evg_api self.generate_tasks_file = generate_tasks_file - self.history_end_date = history_end_date if history_end_date else datetime.utcnow()\ - .replace(microsecond=0) def get_task_runtime_history(self, task: str) -> List[TestRuntime]: """ @@ -353,21 +346,10 @@ class GenerateBurnInExecutor(BurnInExecutor): :param task: Task to query. :return: List of runtime histories for all tests in specified task. """ - try: - project = self.generate_config.project - variant = self.generate_config.build_variant - end_date = self.history_end_date - start_date = end_date - timedelta(days=AVG_TEST_RUNTIME_ANALYSIS_DAYS) - test_stats = HistoricTaskData.from_evg(self.evg_api, project, start_date=start_date, - end_date=end_date, task=task, variant=variant) - return test_stats.get_tests_runtimes() - except requests.HTTPError as err: - if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE: - # Evergreen may return a 503 when the service is degraded. - # We fall back to returning no test history - return [] - else: - raise + project = self.generate_config.project + variant = self.generate_config.build_variant + test_stats = HistoricTaskData.from_s3(project, task, variant) + return test_stats.get_tests_runtimes() def _get_existing_tasks(self) -> Optional[Set[ExistingTask]]: """Get any existing tasks that should be included in the generated display task.""" @@ -434,10 +416,10 @@ def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig, :param install_dir: Path to bin directory of a testable installation """ change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ) - executor = GenerateBurnInExecutor(generate_config, repeat_config, evg_api, generate_tasks_file) + executor = GenerateBurnInExecutor(generate_config, repeat_config, generate_tasks_file) - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) - burn_in_orchestrator.burn_in(repos, build_variant, install_dir) + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) + burn_in_orchestrator.burn_in(repos, build_variant) @click.command() diff --git a/buildscripts/evergreen_task_timeout.py b/buildscripts/evergreen_task_timeout.py index 5c0eabf7aef..7bee3a8e57b 100755 --- a/buildscripts/evergreen_task_timeout.py +++ b/buildscripts/evergreen_task_timeout.py @@ -7,7 +7,7 @@ import math import os import shlex import sys -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path from typing import Dict, List, Optional @@ -19,7 +19,7 @@ from evergreen import EvergreenApi, RetryingEvergreenApi from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, parse_evergreen_file) from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService, TimeoutSettings) +from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService) from buildscripts.util.cmdutils import enable_logging from buildscripts.util.taskname import determine_task_base_name @@ -28,20 +28,21 @@ DEFAULT_TIMEOUT_OVERRIDES = "etc/evergreen_timeouts.yml" DEFAULT_EVERGREEN_CONFIG = "etc/evergreen.yml" DEFAULT_EVERGREEN_AUTH_CONFIG = "~/.evergreen.yml" COMMIT_QUEUE_ALIAS = "__commit_queue" -UNITTEST_TASK = "run_unittests" IGNORED_SUITES = { - "integration_tests_replset", "integration_tests_replset_ssl_auth", "integration_tests_sharded", - "integration_tests_standalone", "integration_tests_standalone_audit", "mongos_test", - "server_selection_json_test" + "integration_tests_replset", + "integration_tests_replset_ssl_auth", + "integration_tests_sharded", + "integration_tests_standalone", + "integration_tests_standalone_audit", + "mongos_test", + "server_selection_json_test", + "sdam_json_test", } HISTORY_LOOKBACK = timedelta(weeks=2) -COMMIT_QUEUE_TIMEOUT = timedelta(minutes=40) +COMMIT_QUEUE_TIMEOUT = timedelta(minutes=20) DEFAULT_REQUIRED_BUILD_TIMEOUT = timedelta(hours=1, minutes=20) DEFAULT_NON_REQUIRED_BUILD_TIMEOUT = timedelta(hours=2) -# 2x the longest "run tests" phase for unittests as of c9bf1dbc9cc46e497b2f12b2d6685ef7348b0726, -# which is 5 mins 47 secs, excluding outliers below -UNITTESTS_TIMEOUT = timedelta(minutes=12) class TimeoutOverride(BaseModel): @@ -143,16 +144,6 @@ class TimeoutOverrides(BaseModel): return None -def _is_required_build_variant(build_variant: str) -> bool: - """ - Determine if the given build variants is a required build variant. - - :param build_variant: Name of build variant to check. - :return: True if the given build variant is required. - """ - return build_variant.endswith("-required") - - def output_timeout(exec_timeout: timedelta, idle_timeout: Optional[timedelta], output_file: Optional[str]) -> None: """ @@ -222,12 +213,7 @@ class TaskTimeoutOrchestrator: LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds()) determined_timeout = override - elif task_name == UNITTEST_TASK and override is None: - LOGGER.info("Overriding unittest timeout", - exec_timeout_secs=UNITTESTS_TIMEOUT.total_seconds()) - determined_timeout = UNITTESTS_TIMEOUT - - elif _is_required_build_variant( + elif self._is_required_build_variant( variant) and determined_timeout > DEFAULT_REQUIRED_BUILD_TIMEOUT: LOGGER.info("Overriding required-builder timeout", exec_timeout_secs=DEFAULT_REQUIRED_BUILD_TIMEOUT.total_seconds()) @@ -273,11 +259,12 @@ class TaskTimeoutOrchestrator: return determined_timeout - def determine_historic_timeout(self, task: str, variant: str, suite_name: str, + def determine_historic_timeout(self, project: str, task: str, variant: str, suite_name: str, exec_timeout_factor: Optional[float]) -> TimeoutOverride: """ Calculate the timeout based on historic test results. + :param project: Name of project to query. :param task: Name of task to query. :param variant: Name of build variant to query. :param suite_name: Name of test suite being run. @@ -287,7 +274,7 @@ class TaskTimeoutOrchestrator: return TimeoutOverride(task=task, exec_timeout=None, idle_timeout=None) timeout_params = TimeoutParams( - evg_project="mongodb-mongo-master", + evg_project=project, build_variant=variant, task_name=task, suite_name=suite_name, @@ -314,9 +301,20 @@ class TaskTimeoutOrchestrator: bv = self.evg_project_config.get_variant(build_variant) return bv.is_asan_build() + def _is_required_build_variant(self, build_variant: str) -> bool: + """ + Determine if the given build variants is a required build variant. + + :param build_variant: Name of build variant to check. + :param evergreen_project_config: Evergreen config to query the variant name. + :return: True if the given build variant is required. + """ + bv = self.evg_project_config.get_variant(build_variant) + return "!" in bv.display_name + def determine_timeouts(self, cli_idle_timeout: Optional[timedelta], - cli_exec_timeout: Optional[timedelta], outfile: Optional[str], task: str, - variant: str, evg_alias: str, suite_name: str, + cli_exec_timeout: Optional[timedelta], outfile: Optional[str], + project: str, task: str, variant: str, evg_alias: str, suite_name: str, exec_timeout_factor: Optional[float]) -> None: """ Determine the timeouts to use for the given task and write timeouts to expansion file. @@ -324,12 +322,14 @@ class TaskTimeoutOrchestrator: :param cli_idle_timeout: Idle timeout specified by the CLI. :param cli_exec_timeout: Exec timeout specified by the CLI. :param outfile: File to write timeout expansions to. + :param project: Evergreen project task is being run on. + :param task: Name of task. :param variant: Build variant task is being run on. :param evg_alias: Evergreen alias that triggered task. :param suite_name: Name of evergreen suite being run. :param exec_timeout_factor: Scaling factor to use when determining timeout. """ - historic_timeout = self.determine_historic_timeout(task, variant, suite_name, + historic_timeout = self.determine_historic_timeout(project, task, variant, suite_name, exec_timeout_factor) idle_timeout = self.determine_idle_timeout(task, variant, cli_idle_timeout, @@ -351,6 +351,8 @@ def main(): help="Resmoke suite being run against.") parser.add_argument("--build-variant", dest="variant", required=True, help="Build variant task is being executed on.") + parser.add_argument("--project", dest="project", required=True, + help="Evergreen project task is being executed on.") parser.add_argument("--evg-alias", dest="evg_alias", required=True, help="Evergreen alias used to trigger build.") parser.add_argument("--timeout", dest="timeout", type=int, help="Timeout to use (in sec).") @@ -369,9 +371,6 @@ def main(): options = parser.parse_args() - end_date = datetime.now() - start_date = end_date - HISTORY_LOOKBACK - timeout_override = timedelta(seconds=options.timeout) if options.timeout else None exec_timeout_override = timedelta( seconds=options.exec_timeout) if options.exec_timeout else None @@ -381,12 +380,12 @@ def main(): os.path.expanduser(options.timeout_overrides_file)) enable_logging(verbose=False) + LOGGER.info("Determining timeouts", cli_args=options) def dependencies(binder: inject.Binder) -> None: binder.bind( EvergreenApi, RetryingEvergreenApi.get_api(config_file=os.path.expanduser(options.evg_api_config))) - binder.bind(TimeoutSettings, TimeoutSettings(start_date=start_date, end_date=end_date)) binder.bind(TimeoutOverrides, timeout_overrides) binder.bind(EvergreenProjectConfig, parse_evergreen_file(os.path.expanduser(options.evg_project_config))) @@ -398,8 +397,8 @@ def main(): task_timeout_orchestrator = inject.instance(TaskTimeoutOrchestrator) task_timeout_orchestrator.determine_timeouts( - timeout_override, exec_timeout_override, options.outfile, task_name, options.variant, - options.evg_alias, options.suite_name, options.exec_timeout_factor) + timeout_override, exec_timeout_override, options.outfile, options.project, task_name, + options.variant, options.evg_alias, options.suite_name, options.exec_timeout_factor) if __name__ == "__main__": diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py index 2e1106bf121..ea3bf3c45d9 100644 --- a/buildscripts/gdb/mongo.py +++ b/buildscripts/gdb/mongo.py @@ -69,7 +69,7 @@ def get_current_thread_name(): fallback_name = '"%s"' % (gdb.selected_thread().name or '') try: # This goes through the pretty printer for StringData which adds "" around the name. - name = str(gdb.parse_and_eval("mongo::ThreadName::getStaticString()")) + name = str(gdb.parse_and_eval("mongo::getThreadName()")) if name == '""': return fallback_name return name diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index 2bf7e67727e..c4ca320641f 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml """ import argparse -import json import os -import re import sys import shlex import yaml @@ -25,7 +23,6 @@ def generate_expansions(): """ args = parse_args() expansions = {} - expansions.update(generate_version_expansions()) expansions.update(generate_scons_cache_expansions()) with open(args.out, "w") as out: @@ -40,39 +37,6 @@ def parse_args(): return parser.parse_args() -def generate_version_expansions(): - """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" - expansions = {} - - if os.path.exists(VERSION_JSON): - with open(VERSION_JSON, "r") as fh: - data = fh.read() - version_data = json.loads(data) - version_line = version_data['version'] - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version.json") - else: - if not os.getenv("MONGO_VERSION"): - raise Exception("$MONGO_VERSION not set and no version.json provided") - version_line = os.getenv("MONGO_VERSION").lstrip("r") - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version from stdin and no version.json provided") - - if version_parts[0]: - expansions["suffix"] = "v6.0-latest" - expansions["src_suffix"] = "v6.0-latest" - expansions["is_release"] = "false" - else: - expansions["suffix"] = version_line - expansions["src_suffix"] = "r{0}".format(version_line) - expansions["is_release"] = "true" - expansions["version"] = version_line - - return expansions - - def generate_scons_cache_expansions(): """Generate scons cache expansions from some files and environment variables.""" expansions = {} @@ -101,21 +65,5 @@ def generate_scons_cache_expansions(): return expansions -def match_verstr(verstr): - """Match a version string and capture the "extra" part. - - If the version is a release like "2.3.4" or "2.3.4-rc0", this will return - None. If the version is a pre-release like "2.3.4-325-githash" or - "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version - begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, - r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. - doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. - """ - res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) - if not res: - return False - return res.groups() - - if __name__ == "__main__": generate_expansions() diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py index 1f9fea582dd..5b6183fa6d9 100755 --- a/buildscripts/generate_compile_expansions_shared_cache.py +++ b/buildscripts/generate_compile_expansions_shared_cache.py @@ -7,9 +7,7 @@ $ python generate_compile_expansions.py --out compile_expansions.yml """ import argparse -import json import os -import re import sys import shlex import yaml @@ -25,7 +23,6 @@ def generate_expansions(): """ args = parse_args() expansions = {} - expansions.update(generate_version_expansions()) expansions.update(generate_scons_cache_expansions()) with open(args.out, "w") as out: @@ -40,39 +37,6 @@ def parse_args(): return parser.parse_args() -def generate_version_expansions(): - """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" - expansions = {} - - if os.path.exists(VERSION_JSON): - with open(VERSION_JSON, "r") as fh: - data = fh.read() - version_data = json.loads(data) - version_line = version_data['version'] - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version.json") - else: - if not os.getenv("MONGO_VERSION"): - raise Exception("$MONGO_VERSION not set and no version.json provided") - version_line = os.getenv("MONGO_VERSION").lstrip("r") - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version from stdin and no version.json provided") - - if version_parts[0]: - expansions["suffix"] = "v6.0-latest" - expansions["src_suffix"] = "v6.0-latest" - expansions["is_release"] = "false" - else: - expansions["suffix"] = version_line - expansions["src_suffix"] = "r{0}".format(version_line) - expansions["is_release"] = "true" - expansions["version"] = version_line - - return expansions - - def generate_scons_cache_expansions(): """Generate scons cache expansions from some files and environment variables.""" expansions = {} @@ -97,7 +61,7 @@ def generate_scons_cache_expansions(): if sys.platform.startswith("win"): shared_mount_root = 'X:\\' else: - shared_mount_root = '/efs' + shared_mount_root = '/efs/scons' default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache") expansions["scons_cache_path"] = default_cache_path expansions[ @@ -125,21 +89,5 @@ def generate_scons_cache_expansions(): return expansions -def match_verstr(verstr): - """Match a version string and capture the "extra" part. - - If the version is a release like "2.3.4" or "2.3.4-rc0", this will return - None. If the version is a pre-release like "2.3.4-325-githash" or - "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version - begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, - r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. - doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. - """ - res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) - if not res: - return False - return res.groups() - - if __name__ == "__main__": generate_expansions() diff --git a/buildscripts/generate_version_expansions.py b/buildscripts/generate_version_expansions.py new file mode 100755 index 00000000000..4b0f9ccc0a8 --- /dev/null +++ b/buildscripts/generate_version_expansions.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +Generate the version expansions file used by Evergreen as part of the push/release process. + +Invoke by specifying an output file. +$ python generate_build_expansions.py --out version_expansions.yml +""" + +import argparse +import json +import os +import re +import sys +import yaml + +VERSION_JSON = "version.json" + + +def generate_expansions(): + """Entry point for the script. + + This calls functions to generate version and scons cache expansions and + writes them to a file. + """ + args = parse_args() + expansions = {} + expansions.update(generate_version_expansions()) + + with open(args.out, "w") as out: + print("saving compile expansions to {0}: ({1})".format(args.out, expansions)) + yaml.safe_dump(expansions, out, default_flow_style=False) + + +def parse_args(): + """Parse program arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument("--out", required=True) + return parser.parse_args() + + +def generate_version_expansions(): + """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" + expansions = {} + + if os.path.exists(VERSION_JSON): + with open(VERSION_JSON, "r") as fh: + data = fh.read() + version_data = json.loads(data) + version_line = version_data['version'] + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version.json") + else: + version_line = os.getenv("MONGO_VERSION") + if not version_line: + raise Exception("$MONGO_VERSION not set and no version.json provided") + + version_line = version_line.lstrip("r") + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version from stdin and no version.json provided") + + if version_parts[0]: + expansions["suffix"] = "v6.0-latest" + expansions["src_suffix"] = "v6.0-latest" + expansions["is_release"] = "false" + else: + expansions["suffix"] = version_line + expansions["src_suffix"] = "r{0}".format(version_line) + expansions["is_release"] = "true" + expansions["version"] = version_line + + return expansions + + +def match_verstr(verstr): + """Match a version string and capture the "extra" part. + + If the version is a release like "2.3.4" or "2.3.4-rc0", this will return + None. If the version is a pre-release like "2.3.4-325-githash" or + "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version + begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, + r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. + doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. + """ + res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) + if not res: + return False + return res.groups() + + +if __name__ == "__main__": + generate_expansions() diff --git a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py index c18077ace76..2a9d3c72420 100644 --- a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py +++ b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py @@ -96,16 +96,16 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: if mongod_or_mongos == "mongod": logger = loggers.new_fixture_logger("MongoDFixture", 0) logger.parent = LOGGER - fixture: interface.Fixture = fixturelib.make_fixture("MongoDFixture", logger, 0, - dbpath_prefix=dbpath.name, - mongod_executable=mongod_executable) + fixture: interface.Fixture = fixturelib.make_fixture( + "MongoDFixture", logger, 0, dbpath_prefix=dbpath.name, + mongod_executable=mongod_executable, mongod_options={"set_parameters": {}}) else: logger = loggers.new_fixture_logger("ShardedClusterFixture", 0) logger.parent = LOGGER - fixture = fixturelib.make_fixture("ShardedClusterFixture", logger, 0, - dbpath_prefix=dbpath.name, - mongos_executable=mongos_executable, - mongod_executable=mongod_executable, mongod_options={}) + fixture = fixturelib.make_fixture( + "ShardedClusterFixture", logger, 0, dbpath_prefix=dbpath.name, + mongos_executable=mongos_executable, mongod_executable=mongod_executable, + mongod_options={"set_parameters": {}}) fixture.setup() fixture.await_ready() diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py index 518583898cb..c7496803b3b 100644 --- a/buildscripts/idl/gen_all_feature_flag_list.py +++ b/buildscripts/idl/gen_all_feature_flag_list.py @@ -30,7 +30,6 @@ Generate a file containing a list of disabled feature flags. Used by resmoke.py to run only feature flag tests. """ -import argparse import os import sys @@ -43,6 +42,7 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../ # pylint: disable=wrong-import-position import buildscripts.idl.lib as lib +from buildscripts.idl.idl import parser def is_third_party_idl(idl_path: str) -> bool: @@ -56,13 +56,14 @@ def is_third_party_idl(idl_path: str) -> bool: return False -def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]): +def gen_all_feature_flags(idl_dir: str = os.getcwd()): """Generate a list of all feature flags.""" all_flags = [] for idl_path in sorted(lib.list_idls(idl_dir)): if is_third_party_idl(idl_path): continue - for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags: + doc = parser.parse_file(open(idl_path), idl_path) + for feature_flag in doc.spec.feature_flags: if feature_flag.default.literal != "true": all_flags.append(feature_flag.name) @@ -72,18 +73,17 @@ def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]): return list(set(all_flags) - set(force_disabled_flags)) -def main(): - """Run the main function.""" - arg_parser = argparse.ArgumentParser(description=__doc__) - arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append", - help="Directory to search for IDL import files") +def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE): + """Output generated list of feature flags to specified file.""" + flags = gen_all_feature_flags() + with open(filename, "w") as output_file: + output_file.write("\n".join(flags)) + print("Generated: ", os.path.realpath(output_file.name)) - args = arg_parser.parse_args() - flags = gen_all_feature_flags(os.getcwd(), args.import_dirs) - with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file: - for flag in flags: - output_file.write("%s\n" % flag) +def main(): + """Run the main function.""" + gen_all_feature_flags_file() if __name__ == '__main__': diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index 2d9925db500..356edf0e265 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -1001,7 +1001,7 @@ def _propagate_globals(spec): idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type) -def _parse(stream, error_file_name): +def parse_file(stream, error_file_name): # type: (Any, str) -> syntax.IDLParsedSpec """ Parse a YAML document into an idl.syntax tree. @@ -1105,7 +1105,7 @@ def parse(stream, input_file_name, resolver): """ # pylint: disable=too-many-locals - root_doc = _parse(stream, input_file_name) + root_doc = parse_file(stream, input_file_name) if root_doc.errors: return root_doc @@ -1142,7 +1142,7 @@ def parse(stream, input_file_name, resolver): # Parse imported file with resolver.open(resolved_file_name) as file_stream: - parsed_doc = _parse(file_stream, resolved_file_name) + parsed_doc = parse_file(file_stream, resolved_file_name) # Check for errors if parsed_doc.errors: diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py index 1a30f29428a..64fd3541bc7 100644 --- a/buildscripts/idl/idl_check_compatibility.py +++ b/buildscripts/idl/idl_check_compatibility.py @@ -824,8 +824,13 @@ def check_param_or_type_validator(ctxt: IDLCompatibilityContext, old_field: synt ctxt.add_command_or_param_type_validators_not_equal_error( cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) else: - ctxt.add_command_or_param_type_contains_validator_error( - cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) + new_field_name: str = cmd_name + "-param-" + new_field.name + # In SERVER-77382 we fixed the error handling of creating time-series collections by + # adding a new validator to two 'stable' fields, but it didn't break any stable API + # guarantees. + if new_field_name not in ["create-param-timeField", "create-param-metaField"]: + ctxt.add_command_or_param_type_contains_validator_error( + cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) def get_all_struct_fields(struct: syntax.Struct, idl_file: syntax.IDLParsedSpec, diff --git a/buildscripts/jepsen_report.py b/buildscripts/jepsen_report.py new file mode 100644 index 00000000000..00747f42f09 --- /dev/null +++ b/buildscripts/jepsen_report.py @@ -0,0 +1,226 @@ +"""Generate Evergreen reports from the Jepsen list-append workload.""" +import json +import re +import sys +import os +from datetime import datetime, timezone +from typing import List, Optional, Tuple + +from typing_extensions import TypedDict +import click + +from buildscripts.simple_report import Result, Report + + +class ParserOutput(TypedDict): + """Result of parsing jepsen log file. Each List[str] is a list of test names.""" + + success: List[str] + unknown: List[str] + crashed: List[str] + failed: List[str] + start: int + end: int + elapsed: int + + +_JEPSEN_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" +_JEPSEN_MILLI_RE = re.compile("([0-9]+){(.*)}") +_JEPSEN_TIME_RE = re.compile("[0-9]{4}-[0-8]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]+{.*}") + + +def _time_parse(time: str): + split = time.split(",") + date = datetime.strptime(split[0], _JEPSEN_TIME_FORMAT) + match = _JEPSEN_MILLI_RE.match(split[1]) + microseconds = 0 + if match: + microseconds = int(match[1]) * 1000 + + return date.replace(microsecond=microseconds, tzinfo=timezone.utc) + + +def _calc_time_from_log(log: str) -> Tuple[int, int, int]: + if not log: + return (0, 0, 0) + start_time = None + end_time = None + for line in log.splitlines(): + if _JEPSEN_TIME_RE.match(line): + if start_time is None: + start_time = _time_parse(line) + else: + end_time = _time_parse(line) + + if start_time is None or end_time is None: + return (0, 0, 0) + + elapsed_time = int(end_time.timestamp() - start_time.timestamp()) + + return (int(start_time.timestamp()), int(end_time.timestamp()), elapsed_time) + + +SUCCESS_RE = re.compile("([0-9]+) successes") +CRASH_RE = re.compile("([0-9]+) crashed") +UNKNOWN_RE = re.compile("([0-9]+) unknown") +FAIL_RE = re.compile("([0-9]+) failures") + + +def parse(text: List[str]) -> ParserOutput: + """Given a List of strings representing jepsen log file split by newlines, return the ParserOutput struct.""" + + successful_tests: List[str] = [] + indeterminate_tests: List[str] = [] + crashed_tests: List[str] = [] + failed_tests: List[str] = [] + target = None + table_matches = 0 + for line in text: + if "# Successful tests" in line: + target = successful_tests + continue + elif "# Indeterminate tests" in line: + target = indeterminate_tests + continue + elif "# Crashed tests" in line: + target = crashed_tests + continue + elif "# Failed tests" in line: + target = failed_tests + continue + + # at this point we're parsing this table: + # 29 successes + # 0 unknown + # 1 crashed + # 0 failures + s_match = SUCCESS_RE.match(line) + if s_match: + target = None + assert int(s_match[1]) == len(successful_tests) + table_matches += 1 + + u_match = UNKNOWN_RE.match(line) + if u_match: + target = None + assert int(u_match[1]) == len(indeterminate_tests) + table_matches += 1 + c_match = CRASH_RE.match(line) + if c_match: + target = None + assert int(c_match[1]) == len(crashed_tests) + table_matches += 1 + f_match = FAIL_RE.match(line) + if f_match: + target = None + assert int(f_match[1]) == len(failed_tests) + table_matches += 1 + + if target is not None and line.strip(): + target.append(line) + + assert table_matches == 4, f"Failed to parse summary table. Expected 4, found {table_matches}" + return ParserOutput({ + 'success': successful_tests, + 'unknown': indeterminate_tests, + 'crashed': crashed_tests, + 'failed': failed_tests, + }) + + +def _try_find_log_file(store: Optional[str], test_name) -> str: + if store is None: + return "" + + try: + with open(os.path.join(store, test_name, "jepsen.log")) as fh: + return fh.read() + + except Exception: # pylint: disable=broad-except + return "" + + +def report(out: ParserOutput, start_time: int, end_time: int, elapsed: int, + store: Optional[str]) -> Report: + """Given ParserOutput, return report.json as a dict.""" + + results = [] + failures = 0 + for test_name in out['success']: + log_raw = _try_find_log_file(store, test_name) + start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) + results.append( + Result(status='pass', exit_code=0, test_file=test_name, start=start_time, end=end_time, + elapsed=elapsed_time, log_raw=log_raw)) + + for test_name in out['failed']: + log_raw = _try_find_log_file(store, test_name) + start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) + failures += 1 + results.append( + Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, + elapsed=elapsed_time, log_raw=log_raw)) + + for test_name in out['crashed']: + log_raw = "Log files are unavailable for crashed tests because Jepsen does not save them separately. You may be able to find the exception and stack trace in the task log" + failures += 1 + results.append( + Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, + elapsed=elapsed, log_raw=log_raw)) + + for test_name in out['unknown']: + log_raw = _try_find_log_file(store, test_name) + start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) + failures += 1 + results.append( + Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, + elapsed=elapsed_time, log_raw=log_raw)) + return Report({ + "failures": failures, + "results": results, + }) + + +def _get_log_lines(filename: str) -> List[str]: + with open(filename) as fh: + return fh.read().splitlines() + + +def _put_report(report_: Report) -> None: + with open("report.json", "w") as fh: + json.dump(report_, fh) + + +@click.command() +@click.option("--start_time", type=int, required=True) +@click.option("--end_time", type=int, required=True) +@click.option("--elapsed", type=int, required=True) +@click.option("--emit_status_files", type=bool, is_flag=True, default=False, + help="If true, emit status files for marking Evergreen tasks as system fails") +@click.option("--store", type=str, default=None, + help="Path to folder containing jepsen 'store' directory") +@click.argument("filename", type=str) +def main(filename: str, start_time: str, end_time: str, elapsed: str, emit_status_files: bool, + store: Optional[str]): + """Generate Evergreen reports from the Jepsen list-append workload.""" + + out = parse(_get_log_lines(filename)) + _put_report(report(out, start_time, end_time, elapsed, store)) + + exit_code = 255 + if out['crashed']: + exit_code = 2 + if emit_status_files: + with open("jepsen_system_fail.txt", "w") as fh: + fh.write(str(exit_code)) + else: + if out['unknown'] or out['failed']: + exit_code = 1 + else: + exit_code = 0 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py index b31a9dbf8db..b4d0bba0490 100644 --- a/buildscripts/moduleconfig.py +++ b/buildscripts/moduleconfig.py @@ -33,16 +33,26 @@ import os def discover_modules(module_root, allowed_modules): + # pylint: disable=too-many-branches """Scan module_root for subdirectories that look like MongoDB modules. Return a list of imported build.py module objects. """ found_modules = [] + found_module_names = [] if allowed_modules is not None: allowed_modules = allowed_modules.split(',') + # When `--modules=` is passed, the split on empty string is represented + # in memory as [''] + if allowed_modules == ['']: + allowed_modules = [] if not os.path.isdir(module_root): + if allowed_modules: + raise RuntimeError( + f"Requested the following modules: {allowed_modules}, but the module root '{module_root}' could not be found. Check the module root, or remove the module from the scons invocation." + ) return found_modules for name in os.listdir(module_root): @@ -66,11 +76,17 @@ def discover_modules(module_root, allowed_modules): if getattr(module, "name", None) is None: module.name = name found_modules.append(module) + found_module_names.append(name) finally: fp.close() except (FileNotFoundError, IOError): pass + if allowed_modules is not None: + missing_modules = set(allowed_modules) - set(found_module_names) + if missing_modules: + raise RuntimeError(f"Failed to locate all modules. Could not find: {missing_modules}") + return found_modules diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 7dc529e39d0..035b00f3a07 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Script and library for symbolizing MongoDB stack traces. To use as a script, paste the JSON object on the line after ----- BEGIN BACKTRACE ----- into the @@ -24,20 +23,42 @@ import signal import subprocess import sys import time +from abc import ABC, abstractmethod from collections import OrderedDict +from datetime import timedelta from pathlib import Path -from typing import Dict +from typing import Dict, List, Any, Union, Optional import requests # pylint: disable=wrong-import-position # pylint: disable=too-many-branches +from tenacity import wait_fixed, stop_after_delay, retry_if_result, Retrying + sys.path.append(str(Path(os.getcwd(), __file__).parent.parent)) -from buildscripts.util.oauth import Configs, get_oauth_credentials +from buildscripts.util.oauth import Configs, get_oauth_credentials, get_client_cred_oauth_credentials from buildscripts.build_system_options import PathOptions +SYMBOLIZER_PATH_ENV = "MONGOSYMB_SYMBOLIZER_PATH" +# since older versions may have issues with symbolizing, we are setting the toolchain version to v4 +DEFAULT_SYMBOLIZER_PATH = "/opt/mongodbtoolchain/v4/bin/llvm-symbolizer" + + +class DbgFileResolver(ABC): + """Base gdb path resolver class.""" + + @abstractmethod + def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]: + """ + To get path for given build info. + + :param soinfo: soinfo as dict + :return: path as string or None (if path not found) + """ + raise NotImplementedError + -class PathDbgFileResolver(object): +class PathDbgFileResolver(DbgFileResolver): """PathDbgFileResolver class.""" def __init__(self, bin_path_guess): @@ -54,7 +75,7 @@ class PathDbgFileResolver(object): return path if path else self._bin_path_guess -class S3BuildidDbgFileResolver(object): +class S3BuildidDbgFileResolver(DbgFileResolver): """S3BuildidDbgFileResolver class.""" def __init__(self, cache_dir, s3_bucket): @@ -73,7 +94,7 @@ class S3BuildidDbgFileResolver(object): if not os.path.exists(build_id_path): try: self._get_from_s3(build_id) - except Exception: # pylint: disable=broad-except + except Exception: # noqa pylint: disable=broad-except ex = sys.exc_info()[0] sys.stderr.write("Failed to find debug symbols for {} in s3: {}\n".format( build_id, ex)) @@ -147,7 +168,7 @@ class CachedResults(object): return self._cached_results.get(key) -class PathResolver(object): +class PathResolver(DbgFileResolver): """ Class to find path for given buildId. @@ -163,16 +184,17 @@ class PathResolver(object): # This amount of attributes are necessary. # the main (API) sever that we'll be sending requests to - default_host = 'https://symbolizer-service.server-tig.prod.corp.mongodb.com' - default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbolizer_downloads_cache') - default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') + default_host = "https://symbolizer-service.server-tig.prod.corp.mongodb.com" + default_cache_dir = os.path.join(os.getcwd(), "build", "symbolizer_downloads_cache") + default_creds_file_path = os.path.join(os.getcwd(), ".symbolizer_credentials.json") default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" + download_timeout_secs = timedelta(minutes=4).total_seconds() def __init__(self, host: str = None, cache_size: int = 0, cache_dir: str = None, client_credentials_scope: str = None, client_credentials_user_name: str = None, - client_id: str = None, redirect_port: int = None, scope: str = None, - auth_domain: str = None): + client_id: str = None, client_secret: str = None, redirect_port: int = None, + scope: str = None, auth_domain: str = None): """ Initialize instance. @@ -187,6 +209,7 @@ class PathResolver(object): self.client_credentials_scope = client_credentials_scope or self.default_client_credentials_scope self.client_credentials_user_name = client_credentials_user_name or self.default_client_credentials_user_name self.client_id = client_id + self.client_secret = client_secret self.redirect_port = redirect_port self.scope = scope self.auth_domain = auth_domain @@ -212,11 +235,17 @@ class PathResolver(object): data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get("expire_time") if time.time() < expire_time: - # credentials hasn't expired yet + # credentials not expired yet self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) return - credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True) + if self.client_id and self.client_secret: + # auth using secrets + credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret, + self.configs) + else: + # since we don't have access to secrets, ask user to auth manually + credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True) self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) # write credentials to local file for further useage @@ -265,7 +294,7 @@ class PathResolver(object): :param url: download URL :return: full name for local file """ - return url.split('/')[-1] + return url.split("/")[-1] @staticmethod def unpack(path: str) -> str: @@ -273,9 +302,9 @@ class PathResolver(object): Use to utar/unzip files. :param path: full path of file - :return: full path of directory of unpacked file + :return: full path to directory of unpacked file """ - out_dir = path.replace('.tgz', '', 1) + out_dir = path.replace(".tgz", "", 1) if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -296,12 +325,25 @@ class PathResolver(object): filename = self.url_to_filename(url) path = os.path.join(self.cache_dir, filename) if not os.path.exists(path): - subprocess.check_call(['wget', url], cwd=self.cache_dir) + print("Downloading the file...") + self.get_file_from_service(url, path) else: - print('File aready exists in cache') + print("File already exists in cache") exists_locally = True return path, exists_locally + def get_file_from_service(self, url: str, local_path: str) -> None: + """ + Get file from URL and write to a local file. + + :param url: URL string + :param local_path: full name for local file + """ + with requests.get(url, stream=True, timeout=self.download_timeout_secs) as response: + with open(local_path, "wb") as file: + for chunk in response.iter_content(chunk_size=2 * 1024 * 1024): + file.write(chunk) + def get_dbg_file(self, soinfo: dict) -> str or None: """ To get path for given buildId. @@ -310,32 +352,26 @@ class PathResolver(object): :return: path as string or None (if path not found) """ build_id = soinfo.get("buildId", "").lower() - binary_name = 'mongo' + version = soinfo.get("version") + binary_name = "mongo" # search from cached results path = self.get_from_cache(build_id) if not path: # path does not exist in cache, so we send request to server try: - response = self.http_client.get(f'{self.host}/find_by_id', - params={'build_id': build_id}) + search_parameters = {"build_id": build_id} + if version: + search_parameters["version"] = version + print(f"Getting data from service... Search parameters: {search_parameters}") + response = self.http_client.get(f"{self.host}/find_by_id", params=search_parameters) if response.status_code != 200: - # if we could not find the path of binary, that might be system library. - # we can try using frame's own `path` data. - # symbolization can succeed only if that binary exists on local - # machine (more specifically: in the given path). - system_path = soinfo.get('path') - if system_path: - sys.stdout.write( - f"Could not find path of binary from symbolizer web service. Trying to use the " - f"provided path: {system_path}\n") - return system_path sys.stderr.write( f"Server returned unsuccessful status: {response.status_code}, " f"response body: {response.text}\n") return None else: - data = response.json().get('data', {}) - path, binary_name = data.get('debug_symbols_url'), data.get('file_name') + data = response.json().get("data", {}) + path, binary_name = data.get("debug_symbols_url"), data.get("file_name") except Exception as err: # noqa pylint: disable=broad-except sys.stderr.write(f"Error occurred while trying to get response from server " f"for buildId({build_id}): {err}\n") @@ -352,7 +388,7 @@ class PathResolver(object): try: dl_path, exists_locally = self.download(path) if exists_locally: - path = dl_path.replace('.tgz', '', 1) + path = dl_path.replace(".tgz", "", 1) else: print("Downloaded, now unpacking...") path = self.unpack(dl_path) @@ -362,8 +398,8 @@ class PathResolver(object): # if file has extension, it is good. if not, we should append .debug, because those without extension are # from release builds, and their debug symbol files contain .debug extension. # we need to map those 2 different file names ('<name>' becomes '<name>.debug'). - if not binary_name.endswith('.debug') and not binary_name.endswith('.so'): - binary_name = f'{binary_name}.debug' + if not binary_name.endswith(".debug"): + binary_name = f"{binary_name}.debug" inner_folder_name = self.path_options.get_binary_folder_name(binary_name) @@ -381,6 +417,7 @@ def parse_input(trace_doc, dbg_path_resolver): return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry} base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"]) + version = get_version(trace_doc) frames = [] for frame in trace_doc["backtrace"]: @@ -390,6 +427,8 @@ def parse_input(trace_doc, dbg_path_resolver): ) continue soinfo = base_addr_map.get(frame["b"], {}) + if version: + soinfo["version"] = version elf_type = soinfo.get("elfType", 0) if elf_type == 3: addr_base = "0" @@ -399,7 +438,7 @@ def parse_input(trace_doc, dbg_path_resolver): addr_base = soinfo.get("vmaddr", "0") addr = int(addr_base, 16) + int(frame["o"], 16) # addr currently points to the return address which is the one *after* the call. x86 is - # variable length so going backwards is difficult. However llvm-symbolizer seems to do the + # variable length so going backwards is difficult. However, llvm-symbolizer seems to do the # right thing if we just subtract 1 byte here. This has the downside of also adjusting the # address of instructions that cause signals (such as segfaults and divide-by-zero) which # are already correct, but there doesn't seem to be a reliable way to detect that case. @@ -411,6 +450,16 @@ def parse_input(trace_doc, dbg_path_resolver): return frames +def get_version(trace_doc: Dict[str, Any]) -> Optional[str]: + """ + Get version from trace doc. + + :param trace_doc: Traceback dict. + :return: Version string or None. + """ + return trace_doc.get("processInfo", {}).get("mongodbVersion") + + def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format, **kwargs): """Return a list of symbolized stack frames from a trace_doc in MongoDB stack dump format.""" @@ -418,24 +467,23 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i # Keep frames in kwargs to avoid changing the function signature. frames = kwargs.get("frames") if frames is None: - frames = preprocess_frames(dbg_path_resolver, trace_doc, input_format) + total_seconds_for_retries = kwargs.get("total_seconds_for_retries", 0) + frames = preprocess_frames_with_retries(dbg_path_resolver, trace_doc, input_format, + total_seconds_for_retries) if not symbolizer_path: - symbolizer_path_env = "MONGOSYMB_SYMBOLIZER_PATH" - default_symbolizer_path = "llvm-symbolizer" - symbolizer_path = os.environ.get(symbolizer_path_env) + symbolizer_path = os.environ.get(SYMBOLIZER_PATH_ENV) if not symbolizer_path: - print( - f"Env value for '{symbolizer_path_env}' not found, using '{default_symbolizer_path}' " - f"as a defualt executable path.") - symbolizer_path = default_symbolizer_path + print(f"Env value for '{SYMBOLIZER_PATH_ENV}' not found, using" + f" '{DEFAULT_SYMBOLIZER_PATH}' as a default executable path.") + symbolizer_path = DEFAULT_SYMBOLIZER_PATH symbolizer_args = [symbolizer_path] for dh in dsym_hint: symbolizer_args.append("-dsym-hint={}".format(dh)) symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=open("/dev/null")) + stderr=sys.stdout) def extract_symbols(stdin): """Extract symbol information from the output of llvm-symbolizer. @@ -467,7 +515,7 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i for frame in frames: if frame["path"] is None: - print("Path not found in frame:", frame) + print(f"Path not found in frame: {frame}") continue symbol_line = "CODE {path:} {addr:}\n".format(**frame) symbolizer_process.stdin.write(symbol_line.encode()) @@ -478,8 +526,17 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i return frames -def preprocess_frames(dbg_path_resolver, trace_doc, input_format): - """Process the paths in frame objects.""" +def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], + input_format: str) -> List[Dict[str, Any]]: + """ + Process the paths in frame objects. + + :param dbg_path_resolver: debug symbols file path resolver + :param trace_doc: traceback object + :param input_format: format of input + :return: the list of traceback frames + """ + if input_format == "classic": frames = parse_input(trace_doc, dbg_path_resolver) elif input_format == "thin": @@ -488,9 +545,43 @@ def preprocess_frames(dbg_path_resolver, trace_doc, input_format): frame["path"] = dbg_path_resolver.get_dbg_file(frame) else: raise ValueError('Unknown input format "{}"'.format(input_format)) + return frames +def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool: + """ + Check whether not found paths in frames ratio is higher than 0.5. + + :param frames: the list of traceback frames + :return: True if ratio is higher than 0.5 + """ + not_found = [1 for f in frames if f.get("path") is None] + not_found_ratio = len(not_found) / (len(frames) or 1) + return not_found_ratio >= 0.5 + + +def preprocess_frames_with_retries(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], + input_format: str, + total_seconds_for_retries: int = 0) -> List[Dict[str, Any]]: + """ + Process the paths in frame objects. + + :param dbg_path_resolver: debug symbols file path resolver + :param trace_doc: traceback object + :param input_format: format of input + :param total_seconds_for_retries: max wait time for retries in seconds + :return: the list of traceback frames + """ + + retrying = Retrying( + retry=retry_if_result(has_high_not_found_paths_ratio), wait=wait_fixed(60), + stop=stop_after_delay(total_seconds_for_retries), + retry_error_callback=lambda retry_state: retry_state.outcome.result()) + + return retrying(preprocess_frames, dbg_path_resolver, trace_doc, input_format) + + def classic_output(frames, outfile, **kwargs): # pylint: disable=unused-argument """Provide classic output.""" for frame in frames: @@ -516,6 +607,12 @@ def make_argument_parser(parser=None, **kwargs): parser.add_argument('--debug-file-resolver', choices=['path', 's3', 'pr'], default='pr') parser.add_argument('--src-dir-to-move', action="store", type=str, default=None, help="Specify a src dir to move to /data/mci/{original_buildid}/src") + parser.add_argument( + '--total-seconds-for-retries', default=0, type=int, + help="If web service fails to find path for given build id, it could be because mapping " + "process was not finished yet. We can wait for it to finish and retry again. Each retry" + " adds 2 minutes to previous wait time. It is guaranteed that total wait time does not exceed this " + "specified amount.") parser.add_argument('--live', action='store_true') s3_group = parser.add_argument_group( @@ -531,6 +628,8 @@ def make_argument_parser(parser=None, **kwargs): help='URL of web service running the API to get debug symbol URL') pr_group.add_argument('--pr-cache-dir', default='', help='Full path to a directory to store cache/files') + pr_group.add_argument('--client-secret', default='', help='Secret key for Okta Oauth') + pr_group.add_argument('--client-id', default='', help='Client id for Okta Oauth') # caching mechanism is currently not fully developed and needs more advanced cleaning techniques, we add an option # to enable it after completing the implementation @@ -565,11 +664,17 @@ def substitute_stdin(options, resolver): if not trace_doc["backtrace"]: print("Trace is empty, skipping...") continue - frames = symbolize_frames(trace_doc, resolver, options.symbolizer_path, [], - options.output_format) + frames = symbolize_frames( + trace_doc, + resolver, + options.symbolizer_path, + [], + options.output_format, + ) print(prefix) print("Symbolizing...") classic_output(frames, sys.stdout, indent=2) + print("Completed, waiting for input...") else: print(line) @@ -583,7 +688,8 @@ def main(options): elif options.debug_file_resolver == 's3': resolver = S3BuildidDbgFileResolver(options.s3_cache_dir, options.s3_bucket) elif options.debug_file_resolver == 'pr': - resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir) + resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir, + client_secret=options.client_secret, client_id=options.client_id) if options.live: print("Entering live mode") @@ -597,7 +703,7 @@ def main(options): if not trace_doc or not trace_doc.strip(): print("Please provide the backtrace through stdin for symbolization;" - "e.g. `your/symbolization/command < /file/with/stacktrace`") + " e.g. `your/symbolization/command < /file/with/stacktrace`") # Search the trace_doc for an object having "backtrace" and "processInfo" keys. def bt_search(obj): @@ -624,8 +730,8 @@ def main(options): except json.JSONDecodeError: pass else: - print("could not find json backtrace object in input", file=sys.stderr) - exit(1) + sys.stderr.write("could not find json backtrace object in input\n") + sys.exit(1) output_fn = None if options.output_format == 'json': @@ -633,7 +739,8 @@ def main(options): if options.output_format == 'classic': output_fn = classic_output - frames = preprocess_frames(resolver, trace_doc, options.input_format) + frames = preprocess_frames_with_retries(resolver, trace_doc, options.input_format, + options.total_seconds_for_retries) if options.src_dir_to_move and resolver.mci_build_dir is not None: try: diff --git a/buildscripts/package_test/files/sources.list.debian8 b/buildscripts/package_test/files/sources.list.debian8 deleted file mode 100644 index 0ad1174270a..00000000000 --- a/buildscripts/package_test/files/sources.list.debian8 +++ /dev/null @@ -1,4 +0,0 @@ -deb http://archive.debian.org/debian jessie main -deb-src http://archive.debian.org/debian jessie main -deb http://security.debian.org/ jessie/updates main -deb-src http://security.debian.org/ jessie/updates main diff --git a/buildscripts/package_test/kitchen.legacy.yml b/buildscripts/package_test/kitchen.legacy.yml deleted file mode 100644 index c44c9a92d5e..00000000000 --- a/buildscripts/package_test/kitchen.legacy.yml +++ /dev/null @@ -1,122 +0,0 @@ ---- -driver: - name: ec2 - region: us-east-1 - vpc_mode: true - vpc_id: <%= ENV['KITCHEN_VPC'] %> - subnet_id: <%= ENV['KITCHEN_SUBNET'] %> - security_group_ids: - - <%= ENV['KITCHEN_SECURITY_GROUP'] %> - aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> - interface: private - associate_public_ip: true - tags: - name: "server package test" - owner: "build" - expire-on: "<%= ENV['KITCHEN_EXPIRE'] %>" - -verifier: - name: inspec - sudo: true - -provisioner: - name: chef_zero - log_level: info - require_chef_omnibus: 12 - -platforms: - - name: amazon-x86-64 - driver: - image_id: ami-0080e4c5bc078760e - transport: - username: ec2-user - - name: amazon2-x86-64 - driver: - image_id: ami-428aa838 - transport: - username: ec2-user - - name: amazon2-arm64 - driver: - image_id: ami-0c582118883b46f4f - instance_type: c6g.medium - transport: - username: ec2-user - - name: debian10-x86-64 - driver: - image_id: ami-0dedf6a6502877301 - transport: - username: admin - - name: rhel70-x86-64 - driver: - image_id: ami-2051294a - transport: - username: root - - name: rhel80-x86-64 - driver: - image_id: ami-0c322300a1dd5dc79 - transport: - username: ec2-user - - name: rhel82-arm64 - driver: - image_id: ami-029ba835ddd43c34f - instance_type: m6g.medium - transport: - username: ec2-user - - name: suse11-x86-64 - driver: - image_id: ami-7f2e6015 - transport: - username: ec2-user - - name: suse12-x86-64 - driver: - image_id: ami-043eebeabcc4e3d35 - transport: - username: ec2-user - - name: suse15-x86-64 - driver: - image_id: ami-06ea7729e394412c8 - transport: - username: ec2-user - - name: ubuntu1204-x86-64 - driver: - image_id: ami-3fec7956 - transport: - username: ubuntu - - name: ubuntu1404-x86-64 - driver: - image_id: ami-1d8c9574 - transport: - username: ubuntu - - name: ubuntu1604-x86-64 - driver: - image_id: ami-64140d0e - transport: - username: ubuntu - - name: ubuntu1804-x86-64 - driver: - image_id: ami-7ad76705 - transport: - username: ubuntu - - name: ubuntu1804-arm64 - driver: - image_id: ami-01ac7d9c1179d7b74 - instance_type: m6g.medium - - name: ubuntu2004-x86-64 - driver: - image_id: ami-068663a3c619dd892 - transport: - username: ubuntu - - name: ubuntu2004-arm64 - driver: - image_id: ami-00579fbb15b954340 - instance_type: m6g.medium - -transport: - ssh_key: ~/.ssh/kitchen.pem - -suites: - - name: service - run_list: - - recipe[package_test::install_mongodb] - attributes: - artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/kitchen.yml b/buildscripts/package_test/kitchen.yml deleted file mode 100644 index f27ab0bfd26..00000000000 --- a/buildscripts/package_test/kitchen.yml +++ /dev/null @@ -1,139 +0,0 @@ ---- -driver: - name: ec2 - region: us-east-1 - vpc_mode: true - vpc_id: <%= ENV['KITCHEN_VPC'] %> - subnet_id: <%= ENV['KITCHEN_SUBNET'] %> - security_group_ids: - - <%= ENV['KITCHEN_SECURITY_GROUP'] %> - aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> - interface: private - associate_public_ip: true - tags: - name: "server package test" - owner: "build" - expire-on: "<%= ENV['KITCHEN_EXPIRE'] %>" - -verifier: - name: inspec - sudo: true - -provisioner: - name: chef_zero - log_level: info - product_name: cinc - product_version: 17.9.52 - download_url: https://omnitruck.cinc.sh/install.sh - -platforms: - - name: amazon-x86-64 - driver: - image_id: ami-0080e4c5bc078760e - transport: - username: ec2-user - - name: amazon2-x86-64 - driver: - image_id: ami-02013ed1a71752ea7 - transport: - username: ec2-user - - name: amazon2-arm64 - driver: - image_id: ami-0c582118883b46f4f - instance_type: c6g.medium - transport: - username: ec2-user - - name: debian71-x86-64 - driver: - image_id: ami-4b124a22 - transport: - username: admin - - name: debian81-x86-64 - driver: - image_id: ami-896d85e2 - transport: - username: admin - - name: debian10-x86-64 - driver: - image_id: ami-0dedf6a6502877301 - transport: - username: admin - - name: debian11-x86-64 - driver: - image_id: ami-06a80441f25333895 - transport: - username: admin - - name: rhel70-x86-64 - driver: - image_id: ami-0051b1b2c5a166c8c - transport: - username: root - - name: rhel80-x86-64 - driver: - image_id: ami-0c322300a1dd5dc79 - transport: - username: ec2-user - - name: rhel82-arm64 - driver: - image_id: ami-029ba835ddd43c34f - instance_type: m6g.medium - transport: - username: ec2-user - - name: suse11-x86-64 - driver: - image_id: ami-7f2e6015 - transport: - username: ec2-user - - name: suse12-x86-64 - driver: - image_id: ami-043eebeabcc4e3d35 - transport: - username: ec2-user - - name: suse15-x86-64 - driver: - image_id: ami-06ea7729e394412c8 - transport: - username: ec2-user - - name: ubuntu1204-x86-64 - driver: - image_id: ami-3fec7956 - transport: - username: ubuntu - - name: ubuntu1404-x86-64 - driver: - image_id: ami-1d8c9574 - transport: - username: ubuntu - - name: ubuntu1604-x86-64 - driver: - image_id: ami-64140d0e - transport: - username: ubuntu - - name: ubuntu1804-x86-64 - driver: - image_id: ami-7ad76705 - transport: - username: ubuntu - - name: ubuntu1804-arm64 - driver: - image_id: ami-01ac7d9c1179d7b74 - instance_type: m6g.medium - - name: ubuntu2004-x86-64 - driver: - image_id: ami-068663a3c619dd892 - transport: - username: ubuntu - - name: ubuntu2004-arm64 - driver: - image_id: ami-00579fbb15b954340 - instance_type: m6g.medium - -transport: - ssh_key: ~/.ssh/kitchen.pem - -suites: - - name: service - run_list: - - recipe[package_test::install_mongodb] - attributes: - artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb deleted file mode 100644 index 3bbfb147acc..00000000000 --- a/buildscripts/package_test/metadata.rb +++ /dev/null @@ -1,2 +0,0 @@ -name 'package_test' -version '0.1.0' diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb deleted file mode 100644 index d57546ce6b9..00000000000 --- a/buildscripts/package_test/recipes/install_mongodb.rb +++ /dev/null @@ -1,146 +0,0 @@ -# This Chef task installs MongoDB in a new EC2 instance spun up by Kitchen in -# preparation for running some basic server functionality tests. - -artifacts_tarball = 'artifacts.tgz' -homedir = "/tmp" - -ruby_block 'allow sudo over tty' do - block do - file = Chef::Util::FileEdit.new('/etc/sudoers') - file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty') - file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw') - file.write_file - end -end - -# This file limits processes to 1024. It therefore interfereres with `ulimit -u` when present. -if platform_family? 'rhel' or platform_family? 'amazon' - file '/etc/security/limits.d/90-nproc.conf' do - action :delete - end -end - -remote_file "#{homedir}/#{artifacts_tarball}" do - source node['artifacts_url'] -end - -execute 'extract artifacts' do - command "tar xzvf #{artifacts_tarball}" - live_stream true - cwd homedir -end - -if platform_family? 'debian' - - # SERVER-40491 Debian 8 sources.list need to point to archive url - if node['platform'] == 'debian' and node['platform_version'] == '8.1' - cookbook_file '/etc/apt/sources.list' do - source 'sources.list.debian8' - owner 'root' - group 'root' - mode '0644' - action :create - end - end - - execute 'apt update' do - command 'apt update' - live_stream true - end - - ENV['DEBIAN_FRONTEND'] = 'noninteractive' - package 'openssl' - - # the ubuntu image does not have some dependencies installed by default - # and it is required for the install_compass script - if node['platform'] == 'ubuntu' and node['platform_version'] == '20.04' - execute 'install dependencies ubuntu 20.04' do - command 'apt-get install -y python3 libsasl2-modules-gssapi-mit' - live_stream true - end - link '/usr/bin/python' do - to '/usr/bin/python3' - end - else - execute 'install dependencies' do - command 'apt-get install -y python libsasl2-modules-gssapi-mit' - live_stream true - end - end - - # dpkg returns 1 if dependencies are not satisfied, which they will not be - # for enterprise builds. We install dependencies in the next block. - execute 'install mongod' do - command 'dpkg -i `find . -name "*server*.deb"`' - live_stream true - cwd homedir - returns [0, 1] - end - - # install the tools so we can test install_compass - execute 'install mongo tools' do - command 'dpkg -i `find . -name "*tools-extra*.deb"`' - live_stream true - cwd homedir - returns [0, 1] - end - - # yum and zypper fetch dependencies automatically, but dpkg does not. - # Installing the dependencies explicitly is fragile, so we reply on apt-get - # to install dependencies after the fact. - execute 'update and fix broken dependencies' do - command 'apt update && apt -y -f install' - live_stream true - end -end - -if platform_family? 'rhel' or platform_family? 'amazon' - bash 'wait for yum updates if they are running' do - code <<-EOH - sleep 120 - EOH - end - execute 'install mongod' do - command 'yum install -y `find . -name "*server*.rpm"`' - live_stream true - cwd homedir - end - - # install the tools so we can test install_compass - execute 'install mongo tools' do - command 'yum install -y `find . -name "*tools-extra*.rpm"`' - live_stream true - cwd homedir - end -end - -if platform_family? 'suse' - bash 'wait for zypper lock to be released' do - code <<-EOD - retry_counter=0 - # We also need to make sure another instance of zypper isn't running while - # we do our install, so just run zypper refresh until it doesn't fail. - # Waiting for 2 minutes is copied from an internal project where we do this. - until [ "$retry_counter" -ge "12" ]; do - zypper refresh && exit 0 - retry_counter=$(($retry_counter + 1)) - [ "$retry_counter" = "12" ] && break - sleep 10 - done - exit 1 - EOD - flags "-x" - end - - execute 'install mongod' do - command 'zypper --no-gpg-checks -n install `find . -name "*server*.rpm"`' - live_stream true - cwd homedir - end - - execute 'install mongo tools' do - command 'zypper --no-gpg-checks -n install `find . -name "*tools-extra*.rpm"`' - live_stream true - cwd homedir - end -end diff --git a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb deleted file mode 100644 index 821f55f1a57..00000000000 --- a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb +++ /dev/null @@ -1,219 +0,0 @@ -############################################################ -# This section verifies start, stop, and restart after -# installation within a new EC2 instance spun up by Kitchen. -# -# - stop mongod so that we begin testing from a stopped state -# - verify start, stop, and restart -############################################################ - -# service is not in path for commands with sudo on suse -service = os[:name] == 'suse' ? '/sbin/service' : 'service' - -describe command("#{service} mongod stop") do - its('exit_status') { should eq 0 } -end - -describe command("#{service} mongod start") do - its('exit_status') { should eq 0 } -end - -# Inspec treats all amazon linux as upstart, we explicitly make it use -# systemd_service https://github.com/chef/inspec/issues/2639 -if (os[:name] == 'amazon' and os[:release] == '2.0') - describe systemd_service('mongod') do - it { should be_running } - end -else - describe service('mongod') do - it { should be_running } - end -end - -describe command("#{service} mongod stop") do - its('exit_status') { should eq 0 } -end - -describe command("#{service} mongod restart") do - its('exit_status') { should eq 0 } -end - -if (os[:name] == 'amazon' and os[:release] == '2.0') - describe systemd_service('mongod') do - it { should be_running } - end -else - describe service('mongod') do - it { should be_running } - end -end - -if os[:arch] == 'x86_64' - # install_compass does not run Amazon Linux but *does* run on Amazon Linux 2, - # but the 'redhat' family includes both, apparently. We need to specifically - # exclude Amazon Linux from the set of allowed distributions here because the - # version strings would otherwise pass it. - if ((os[:family] == 'redhat' and os[:name] != "amazon" and os[:release].split('.')[0].to_i >= 7) or - (os[:name] == 'ubuntu' and os[:release].split('.')[0].to_i >= 16) or - (os[:name] == 'debian' and os[:release].split('.')[0].to_i >= 9) or - (os[:name] == 'amazon' and os[:release].split('.')[0].to_i == 2)) - describe command("install_compass") do - its('exit_status') { should eq 0 } - its('stderr') { should eq '' } - end - elsif os[:name] == 'suse' - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /You are using an unsupported platform/ } - end - else - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /You are using an unsupported Linux distribution/ } - end - end -else - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /Sorry, MongoDB Compass is only supported on 64-bit Intel platforms/ } - end -end - -############################################################ -# This section verifies files, directories, and users -# - files and directories exist and have correct attributes -# - mongod user exists and has correct attributes -############################################################ - -# convenience variables for init system and package type -upstart = (os[:name] == 'ubuntu' && os[:release][0..1] == '14') || - (os[:name] == 'amazon') -sysvinit = if (os[:name] == 'debian' && os[:release][0] == '7') || - (os[:name] == 'redhat' && os[:release][0] == '6') || - (os[:name] == 'suse' && os[:release][0..1] == '11') || - (os[:name] == 'ubuntu' && os[:release][0..1] == '12') - true - else - false - end -systemd = !(upstart || sysvinit) -rpm = if os[:name] == 'amazon' || os[:name] == 'redhat' || os[:name] == 'suse' - true - else - false - end -deb = !rpm - -# these files should exist on all systems -%w( - /etc/mongod.conf - /usr/bin/mongod - /var/log/mongodb/mongod.log -).each do |filename| - describe file(filename) do - it { should be_file } - end -end - -if sysvinit - describe file('/etc/init.d/mongod') do - it { should be_file } - it { should be_executable } - end -end - -if systemd - unit_file_prefix = '' - if os[:name] == 'suse' - # Putting systemd unit files in /usr, which may be a separate partition - # and therefore not available during isolated startups, is bad practice. - # But it's what SUSE has chosen to do, so we have to deal with it. - unit_file_prefix = '/usr' - end - describe file("#{unit_file_prefix}/lib/systemd/system/mongod.service") do - it { should be_file } - end -end - -if rpm - %w( - /var/lib/mongo - /var/run/mongodb - ).each do |filename| - describe file(filename) do - it { should be_directory } - end - end - - describe user('mongod') do - it { should exist } - its('groups') { should include 'mongod' } - its('home') { should eq '/var/lib/mongo' } - its('shell') { should eq '/bin/false' } - end -end - -if deb - describe file('/var/lib/mongodb') do - it { should be_directory } - end - - describe user('mongodb') do - it { should exist } - its('groups') { should include 'mongodb' } - # All versions of Debian 10 will use /usr/sbin/nologin for service - # account shells - its('shell') { - if ((os[:name] == 'debian' and os[:release].split('.')[0] >= '10') or - (os[:name] == 'ubuntu' and os[:release] == '18.04') or - (os[:name] == 'ubuntu' and os[:release] == '20.04')) - should eq '/usr/sbin/nologin' - else - should eq '/bin/false' - end - } - end -end - -############################################################ -# This section verifies ulimits. -############################################################ - -ulimits = { - 'Max file size' => 'unlimited', - 'Max cpu time' => 'unlimited', - 'Max address space' => 'unlimited', - 'Max open files' => '64000', - 'Max resident set' => 'unlimited', - 'Max processes' => '64000' -} -ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits' - -ulimits.each do |limit, value| - describe command("#{ulimits_cmd} | grep \"#{limit}\"") do - its('stdout') { should match(/#{limit}\s+#{value}/) } - end -end - -############################################################ -# This section verifies uninstall. -############################################################ - -if rpm - describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do - its('exit_status') { should eq 0 } - end -elsif deb - describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do - its('exit_status') { should eq 0 } - end -end - -# make sure we cleaned up -%w( - /lib/systemd/system/mongod.service - /usr/bin/mongod -).each do |filename| - describe file(filename) do - it { should_not exist } - end -end diff --git a/buildscripts/packager.py b/buildscripts/packager.py index d90f0c80489..cf464bdfbfc 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -302,6 +302,8 @@ class Distro(object): return "bionic" elif build_os == 'ubuntu2004': return "focal" + elif build_os == 'ubuntu2204': + return "jammy" else: raise Exception("unsupported build_os: %s" % build_os) elif self.dname == 'debian': @@ -341,7 +343,17 @@ class Distro(object): if re.search("(suse)", self.dname): return ["suse11", "suse12", "suse15"] elif re.search("(redhat|fedora|centos)", self.dname): - return ["rhel82", "rhel80", "rhel70", "rhel71", "rhel72", "rhel62", "rhel55", "rhel67"] + return [ + "rhel90", + "rhel82", + "rhel80", + "rhel70", + "rhel71", + "rhel72", + "rhel62", + "rhel55", + "rhel67", + ] elif self.dname in ['amazon', 'amazon2']: return [self.dname] elif self.dname == 'ubuntu': @@ -351,6 +363,7 @@ class Distro(object): "ubuntu1604", "ubuntu1804", "ubuntu2004", + "ubuntu2204", ] elif self.dname == 'debian': return ["debian81", "debian92", "debian10", "debian11"] diff --git a/buildscripts/packager_enterprise.py b/buildscripts/packager_enterprise.py index fcc260c86be..6d5a876d4db 100755 --- a/buildscripts/packager_enterprise.py +++ b/buildscripts/packager_enterprise.py @@ -119,7 +119,7 @@ class EnterpriseDistro(packager.Distro): def build_os(self, arch): # pylint: disable=too-many-branches """Return the build os label in the binary package to download. - The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80" are for redhat, + The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80", "rhel90" are for redhat, the others are delegated to the super class. """ # pylint: disable=too-many-return-statements @@ -139,16 +139,16 @@ class EnterpriseDistro(packager.Distro): return [] if arch == "arm64": if self.dname == 'ubuntu': - return ["ubuntu1804", "ubuntu2004"] + return ["ubuntu1804", "ubuntu2004", "ubuntu2204"] if arch == "aarch64": if self.dname == 'redhat': - return ["rhel82"] + return ["rhel82", "rhel90"] if self.dname == 'amazon2': return ["amazon2"] return [] if re.search("(redhat|fedora|centos)", self.dname): - return ["rhel80", "rhel70", "rhel62", "rhel57"] + return ["rhel90", "rhel80", "rhel70", "rhel62", "rhel57"] return super(EnterpriseDistro, self).build_os(arch) # pylint: enable=too-many-return-statements diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py index 44af0fca70b..0d7fee88496 100644 --- a/buildscripts/patch_builds/change_data.py +++ b/buildscripts/patch_builds/change_data.py @@ -87,7 +87,7 @@ def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) - work_tree_files = _modified_files_for_diff(diff, LOGGER.bind(diff="working tree diff")) commit = repo.index - diff = commit.diff(revision_map.get(repo.git_dir, repo.head.commit)) + diff = commit.diff(revision_map.get(repo.git_dir, repo.head.commit), R=True) index_files = _modified_files_for_diff(diff, LOGGER.bind(diff="index diff")) untracked_files = set(repo.untracked_files) diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py deleted file mode 100644 index d8d45872685..00000000000 --- a/buildscripts/promote_silent_failures.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -"""Convert silent test failures into non-silent failures. - -Any test files with at least 2 executions in the report.json file that have a "silentfail" status, -this script will change the outputted report to have a "fail" status instead. -""" - -import collections -import json -import optparse -import os -import sys - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from buildscripts.resmokelib.testing import report - - -def read_json_file(json_file): - """Return contents of a JSON file.""" - with open(json_file) as json_data: - return json.load(json_data) - - -def main(): - """Execute Main program.""" - - usage = "usage: %prog [options] report.json" - parser = optparse.OptionParser(usage=usage) - parser.add_option( - "-o", "--output-file", dest="outfile", default="-", - help=("If '-', then the report file is written to stdout." - " Any other value is treated as the output file name. By default," - " output is written to stdout.")) - - (options, args) = parser.parse_args() - - if len(args) != 1: - parser.error("Requires a single report.json file.") - - report_file_json = read_json_file(args[0]) - test_report = report.TestReport.from_dict(report_file_json) - - # Count number of "silentfail" per test file. - status_dict = collections.defaultdict(int) - for test_info in test_report.test_infos: - if test_info.evergreen_status == "silentfail": - status_dict[test_info.test_id] += 1 - - # For test files with more than 1 "silentfail", convert status to "fail". - for test_info in test_report.test_infos: - if status_dict[test_info.test_id] >= 2: - test_info.evergreen_status = "fail" - - result_report = test_report.as_dict() - if options.outfile != "-": - with open(options.outfile, "w") as fp: - json.dump(result_report, fp) - else: - print(json.dumps(result_report)) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/resmoke_tests_runtime_validate.py b/buildscripts/resmoke_tests_runtime_validate.py index cffcbf48e9d..5c17d1ee5f8 100644 --- a/buildscripts/resmoke_tests_runtime_validate.py +++ b/buildscripts/resmoke_tests_runtime_validate.py @@ -3,7 +3,6 @@ import json import sys from collections import namedtuple -from datetime import datetime, timedelta from statistics import mean from typing import Dict, List @@ -13,7 +12,8 @@ import structlog from buildscripts.resmokelib.testing.report import TestInfo, TestReport from buildscripts.resmokelib.utils import get_task_name_without_suffix from buildscripts.util.cmdutils import enable_logging -from evergreen import RetryingEvergreenApi, TestStats + +from buildscripts.util.teststats import HistoricTaskData, HistoricalTestInformation LOGGER = structlog.get_logger("buildscripts.resmoke_tests_runtime_validate") @@ -34,17 +34,12 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]: return [test_info for test_info in test_report.test_infos if "jstests" in test_info.test_file] -def get_historic_stats(evg_api_config: str, project_id: str, test_files: List[str], task_name: str, - build_variant: str) -> List[TestStats]: +def get_historic_stats(project_id: str, task_name: str, + build_variant: str) -> List[HistoricalTestInformation]: """Get historic test stats.""" - evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) - before_date = datetime.today() - after_date = before_date - timedelta(days=LOOK_BACK_NUM_DAYS) base_task_name = get_task_name_without_suffix(task_name, build_variant).replace( BURN_IN_PREFIX, "") - return evg_api.test_stats_by_project(project_id=project_id, after_date=after_date, - before_date=before_date, tests=test_files, - tasks=[base_task_name], variants=[build_variant]) + return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant) def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]: @@ -63,13 +58,10 @@ def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]: @click.command() @click.option("--resmoke-report-file", type=str, required=True, help="Location of resmoke's report JSON file.") -@click.option("--evg-api-config", type=str, required=True, - help="Location of evergreen api configuration.") @click.option("--project-id", type=str, required=True, help="Evergreen project id.") @click.option("--build-variant", type=str, required=True, help="Evergreen build variant name.") @click.option("--task-name", type=str, required=True, help="Evergreen task name.") -def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_variant: str, - task_name: str) -> None: +def main(resmoke_report_file: str, project_id: str, build_variant: str, task_name: str) -> None: """Compare resmoke tests runtime with historic stats.""" enable_logging(verbose=False) @@ -79,10 +71,9 @@ def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_v for test_info in current_test_infos ]) - historic_stats = get_historic_stats(evg_api_config, project_id, list(current_stats_map.keys()), - task_name, build_variant) + historic_stats = get_historic_stats(project_id, task_name, build_variant) historic_stats_map = make_stats_map([ - _TestData(test_stats.test_file, test_stats.avg_duration_pass) + _TestData(test_stats.test_name, test_stats.avg_duration_pass) for test_stats in historic_stats ]) diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml index e25ee213769..6265d19b9e9 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml @@ -272,23 +272,43 @@ - "multiversion.replica_sets_multiversion_testdata_last_continuous" -- suite_name: generational_fuzzer_last_lts +- suite_name: aggregation_multiversion_fuzzer_last_lts base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" -- suite_name: generational_fuzzer_last_continuous +- suite_name: aggregation_multiversion_fuzzer_last_continuous + base_suite: generational_fuzzer + overrides: + - "multiversion.replica_sets_multiversion_testdata_last_continuous" + +- suite_name: aggregation_expression_multiversion_fuzzer_last_lts + base_suite: generational_fuzzer + overrides: + - "multiversion.replica_sets_multiversion_testdata_last_lts" + +- suite_name: aggregation_expression_multiversion_fuzzer_last_continuous + base_suite: generational_fuzzer + overrides: + - "multiversion.replica_sets_multiversion_testdata_last_continuous" + +- suite_name: update_fuzzer_last_lts + base_suite: generational_fuzzer + overrides: + - "multiversion.replica_sets_multiversion_testdata_last_lts" + +- suite_name: update_fuzzer_last_continuous base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" -- suite_name: generational_fuzzer_replication_last_lts +- suite_name: update_fuzzer_replication_last_lts base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" -- suite_name: generational_fuzzer_replication_last_continuous +- suite_name: update_fuzzer_replication_last_continuous base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml index 2ec2f4fe333..6590d78359d 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml @@ -21,6 +21,7 @@ - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js @@ -53,6 +54,8 @@ # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary. - jstests/core/geo_s2ordering.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/**/*.js - name: reconfig_kill_primary_jscore_passthrough_exclude_files @@ -81,6 +84,7 @@ - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # These test run commands using legacy queries, which are not supported on sessions. - jstests/core/comment_field.js @@ -121,6 +125,8 @@ # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary. - jstests/core/geo_s2ordering.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/**/*.js - name: kill_primary_jscore_passthrough_exclude_with_any_tags diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml index 51fc059280b..607a68dc431 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml @@ -3,8 +3,6 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Don't run these tests as transactions can only run on primaries. - jstests/core/txns/**/*.js @@ -47,6 +45,7 @@ selector: - jstests/core/tailable_cursor_invalidation.js - jstests/core/tailable_getmore_batch_size.js - jstests/core/tailable_skip_limit.js + - jstests/core/timeseries/timeseries_lastpoint_top.js - jstests/core/constructors.js - jstests/core/views/views_all_commands.js - jstests/core/or4.js diff --git a/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml b/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml index cb9f747aebb..d2a33b08945 100644 --- a/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml @@ -65,6 +65,8 @@ selector: # TODO (SERVER-61259): $text not supported: "No query solutions" - jstests/core/fts6.js - jstests/core/fts_projection.js + # Assumes there is one collection that is not clustered. + - jstests/core/find_with_resume_after_param.js exclude_with_any_tags: - assumes_standalone_mongod diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml index 4fedf2ca12b..8b84c71d5ec 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml @@ -31,6 +31,8 @@ selector: - uses_write_concern # Sharding is not supported for live exports and imports. - requires_sharding + # Workloads that kill random sessions may interrupt the export/import commands. + - kills_random_sessions executor: config: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml index 83467144c4c..7e4e48ed066 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml @@ -9,9 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml index 3bfbf5175bd..ff7501c54de 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml @@ -9,9 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index 18452678c0c..f2e6a544dea 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -9,9 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml index a904a527bc6..7f0ce597620 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml @@ -13,9 +13,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml index b629f4d1971..ca3e87dde52 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml @@ -13,9 +13,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml index 0d711ca421c..66a8a47b4d6 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml @@ -13,9 +13,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml index e3280f20994..aa1599a3d1e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml @@ -18,9 +18,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml index 17bf8eb2ccf..cc53c9d8b91 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml @@ -18,9 +18,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml index 84d7c942484..1920a225dd0 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml @@ -13,9 +13,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml index 88a29447a8e..609de156c2e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml @@ -18,9 +18,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml index f2e8e9a06b1..02d2c081cfb 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml @@ -9,10 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - - jstests/concurrency/fsm_workloads/map_reduce_drop.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml index 0d104100f9d..7fede223baa 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml @@ -9,10 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - - jstests/concurrency/fsm_workloads/map_reduce_drop.js - # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index 41e0539bbc3..a276ebf9d67 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -9,9 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index fe148c981c7..f075f53951b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -9,10 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - - jstests/concurrency/fsm_workloads/map_reduce_drop.js - # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index f4e9fad96fc..58a638ba149 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -9,10 +9,6 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js - # SERVER-17397 Drops of sharded namespaces may not fully succeed - - jstests/concurrency/fsm_workloads/create_database.js - - jstests/concurrency/fsm_workloads/map_reduce_drop.js - # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml index e6125107252..38bad5b9d73 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml @@ -32,13 +32,6 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js - # This workload kills random sessions and a different FSM workload wouldn't be able to handle - # the error response from the op being killed. - - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js - - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js - # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -56,8 +49,13 @@ selector: - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js + # This workload involves upgrading and downgrading the FCV, and can cause failure on other + # FSM workloads which depend on features which are only available in the latest FCV. + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + exclude_with_any_tags: - requires_sharding + - kills_random_sessions group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml index 179d44f40ad..04a5a6fe2da 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml @@ -32,13 +32,6 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js - # This workload kills random sessions and a different FSM workload wouldn't be able to handle - # the error response from the op being killed. - - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js - - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js - # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -61,8 +54,13 @@ selector: - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js + # This workload involves upgrading and downgrading the FCV, and can cause failure on other + # FSM workloads which depend on features which are only available in the latest FCV. + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + exclude_with_any_tags: - requires_sharding + - kills_random_sessions group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml index 3bb3060f1cd..23859ef8d98 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml @@ -32,13 +32,6 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js - # This workload kills random sessions and a different FSM workload wouldn't be able to handle - # the error response from the op being killed. - - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js - - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js - # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -62,8 +55,13 @@ selector: - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js + # This workload involves upgrading and downgrading the FCV, and can cause failure on other + # FSM workloads which depend on features which are only available in the latest FCV. + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + exclude_with_any_tags: - requires_sharding + - kills_random_sessions group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml index 09b0a6f7ece..b40e2783d5c 100755 --- a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml @@ -256,9 +256,6 @@ selector: - jstests/core/index_check6.js - jstests/core/index_check7.js - jstests/core/index_decimal.js - - jstests/core/index_elemmatch1.js - - jstests/core/index_elemmatch2.js - - jstests/core/index_elemmatch2.js - jstests/core/index_filter_commands.js - jstests/core/index_filter_on_hidden_index.js - jstests/core/index_multiple_compatibility.js @@ -508,8 +505,6 @@ selector: - jstests/core/in7.js - jstests/core/index13.js - jstests/core/index_check2.js - - jstests/core/index_elemmatch1.js - - jstests/core/index_elemmatch2.js - jstests/core/indexl.js - jstests/core/json_schema/misc_validation.js - jstests/core/ne_array.js @@ -758,7 +753,6 @@ selector: - jstests/core/geonear_key.js - jstests/core/getmore_invalidated_documents.js - jstests/core/hidden_index.js - - jstests/core/index_elemmatch2.js - jstests/core/index_partial_2dsphere.js - jstests/core/json_schema/misc_validation.js - jstests/core/list_collections_filter.js diff --git a/buildscripts/resmokeconfig/suites/free_monitoring.yml b/buildscripts/resmokeconfig/suites/free_monitoring.yml deleted file mode 100644 index 98fb2867c51..00000000000 --- a/buildscripts/resmokeconfig/suites/free_monitoring.yml +++ /dev/null @@ -1,10 +0,0 @@ -test_kind: js_test - -selector: - roots: - - jstests/free_mon/*.js - -executor: - config: - shell_options: - nodb: '' diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index bc58d35542c..d885199b493 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -223,6 +223,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Does not support tojson of command objects. - jstests/core/SERVER-23626.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index daff4e411c8..1b27ca1e5c3 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -221,6 +221,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Does not support tojson of command objects. - jstests/core/SERVER-23626.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml index d292c3ae69d..769a7daf339 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml @@ -7,8 +7,6 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -33,6 +31,7 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # These test run commands using legacy queries, which are not supported on sessions. - jstests/core/comment_field.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml index 1288ffb0a86..7f91860f3b8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml @@ -3,8 +3,6 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -24,6 +22,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml index b3d3d510dae..0ef725251e6 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml @@ -24,6 +24,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml index 3f8a241c4bc..dd0743c6a23 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml @@ -30,6 +30,7 @@ selector: - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js + - jstests/core/project_with_collation.js # These tests rely on the assumption that an update command is run only once. - jstests/core/find_and_modify_metrics.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml index 985f095650d..b382dde715a 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml @@ -3,8 +3,6 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -25,6 +23,7 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/search.yml b/buildscripts/resmokeconfig/suites/search.yml index 61ba50f7715..4c003db89d1 100644 --- a/buildscripts/resmokeconfig/suites/search.yml +++ b/buildscripts/resmokeconfig/suites/search.yml @@ -3,6 +3,9 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/search/*.js + - src/mongo/db/modules/*/jstests/mongot/*.js + exclude_with_any_tags: + - requires_auth executor: config: diff --git a/buildscripts/resmokeconfig/suites/search_auth.yml b/buildscripts/resmokeconfig/suites/search_auth.yml index edf79042a40..0afb113e42a 100644 --- a/buildscripts/resmokeconfig/suites/search_auth.yml +++ b/buildscripts/resmokeconfig/suites/search_auth.yml @@ -7,6 +7,7 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/search/*.js + - src/mongo/db/modules/*/jstests/mongot/*.js executor: config: diff --git a/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml b/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml new file mode 100644 index 00000000000..8bfb2fef169 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml @@ -0,0 +1,24 @@ +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/search/*.js + - src/mongo/db/modules/*/jstests/mongot/*.js + +executor: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-256 + keyFile: *keyFile + keyFileData: *keyFileData + roleGraphInvalidationIsFatal: true + setParameters: + pinTaskExecCursorConns: true + nodb: '' diff --git a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml index 6419782211b..3fab3de86db 100644 --- a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml +++ b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml @@ -24,6 +24,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/sharding_auth.yml b/buildscripts/resmokeconfig/suites/sharding_auth.yml index 964d03ce993..02e731e6352 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth.yml @@ -32,6 +32,7 @@ selector: - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 # Runs with auth enabled. - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js + - jstests/sharding/cluster_time_across_add_shard.js # Skip because this suite implicitly authenticates as __system, which allows bypassing user write # blocking. - jstests/sharding/set_user_write_block_mode.js diff --git a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml index 630863957be..a6c193f0e91 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml @@ -32,6 +32,7 @@ selector: - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 # Runs with auth enabled. - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js + - jstests/sharding/cluster_time_across_add_shard.js # Skip because this suite implicitly authenticates as __system, which allows bypassing user write # blocking. - jstests/sharding/set_user_write_block_mode.js diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 20749832032..4e5b4b96ae4 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -10,6 +10,8 @@ selector: - jstests/sharding/*[aA]uth*.js - jstests/sharding/query/*[aA]uth*.js - jstests/sharding/change_streams/*[aA]uth*.js + - jstests/sharding/cluster_time_across_add_shard.js + - jstests/sharding/internal_txns/internal_client_restrictions.js - jstests/sharding/internal_txns/non_retryable_writes_during_migration.js - jstests/sharding/internal_txns/retry_on_transient_error_validation.js @@ -29,7 +31,6 @@ selector: - jstests/sharding/addshard1.js - jstests/sharding/addshard2.js - jstests/sharding/autosplit.js - - jstests/sharding/auto_rebalance_parallel.js - jstests/sharding/basic_merge.js - jstests/sharding/count1.js - jstests/sharding/count2.js @@ -51,12 +52,10 @@ selector: - jstests/sharding/limit_push.js - jstests/sharding/merge_with_drop_shard.js - jstests/sharding/merge_with_move_primary.js - - jstests/sharding/migrateBig_balancer.js - jstests/sharding/move_chunk_basic.js - jstests/sharding/movePrimary1.js - jstests/sharding/names.js - jstests/sharding/prefix_shard_key.js - - jstests/sharding/presplit.js - jstests/sharding/query_config.js - jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js - jstests/sharding/remove1.js @@ -64,11 +63,8 @@ selector: - jstests/sharding/shard2.js - jstests/sharding/shard3.js - jstests/sharding/shard_collection_basic.js - - jstests/sharding/shard_existing_coll_chunk_count.js - - jstests/sharding/sharding_balance1.js - jstests/sharding/sharding_balance2.js - jstests/sharding/sharding_balance3.js - - jstests/sharding/sharding_balance4.js - jstests/sharding/sharding_migrate_cursor1.js - jstests/sharding/tag_range.js - jstests/sharding/top_chunk_autosplit.js diff --git a/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml b/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml index 312f8249245..790d06a5f0c 100644 --- a/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml +++ b/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml @@ -30,6 +30,7 @@ executor: config: {} hooks: - class: SimulateCrash + - class: CleanupConcurrencyWorkloads fixture: class: ReplicaSetFixture mongod_options: diff --git a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml index 7a1f8d9f47a..3bc42f1f487 100644 --- a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml @@ -34,6 +34,7 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js + - jstests/core/project_with_collation.js - jstests/core/bench_test*.js # benchRun() used for writes - jstests/core/benchrun_pipeline_updates.js # benchRun() used for writes diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml index 36433f7ceca..c6a2225f44f 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml @@ -43,6 +43,7 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml index fc02fb23677..6fd164a5025 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml @@ -43,6 +43,7 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml index 94d46e70994..6e139307b27 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml @@ -43,6 +43,7 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js + - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/vector_search.yml b/buildscripts/resmokeconfig/suites/vector_search.yml new file mode 100644 index 00000000000..effd621be8f --- /dev/null +++ b/buildscripts/resmokeconfig/suites/vector_search.yml @@ -0,0 +1,12 @@ +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/vector_search/*.js + exclude_with_any_tags: + - requires_auth + +executor: + config: + shell_options: + nodb: '' diff --git a/buildscripts/resmokeconfig/suites/vector_search_auth.yml b/buildscripts/resmokeconfig/suites/vector_search_auth.yml new file mode 100644 index 00000000000..c025afe3619 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/vector_search_auth.yml @@ -0,0 +1,21 @@ +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/vector_search/*.js + +executor: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-256 + keyFile: *keyFile + keyFileData: *keyFileData + roleGraphInvalidationIsFatal: true + nodb: '' diff --git a/buildscripts/resmokeconfig/suites/vector_search_ssl.yml b/buildscripts/resmokeconfig/suites/vector_search_ssl.yml new file mode 100644 index 00000000000..e234ea490fd --- /dev/null +++ b/buildscripts/resmokeconfig/suites/vector_search_ssl.yml @@ -0,0 +1,15 @@ +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/vector_search/ssl/*.js + +executor: + config: + shell_options: + nodb: '' + ssl: '' + tlsAllowInvalidHostnames: '' + tlsAllowInvalidCertificates: '' + tlsCAFile: jstests/libs/ca.pem + tlsCertificateKeyFile: jstests/libs/client.pem diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py index 2cb85721e9e..3dc6a0a522d 100644 --- a/buildscripts/resmokelib/cli.py +++ b/buildscripts/resmokelib/cli.py @@ -18,6 +18,6 @@ def main(argv): usage="Resmoke is MongoDB's correctness testing orchestrator.\n" "For more information, see the help message for each subcommand.\n" "For example: resmoke.py run -h\n" - "Note: bisect and setup-multiversion subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" + "Note: bisect, setup-multiversion and symbolize subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" ) subcommand.execute() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index c1cc1048585..4b46d11f4df 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -55,7 +55,7 @@ DEFAULTS = { "archive_limit_tests": 10, "base_port": 20000, "backup_on_restart_dir": None, - "buildlogger_url": "https://logkeeper.mongodb.org", + "buildlogger_url": "https://logkeeper2.build.10gen.cc", "continue_on_failure": False, "dbpath_prefix": None, "dbtest_executable": None, @@ -88,11 +88,11 @@ DEFAULTS = { "repeat_tests_min": None, "repeat_tests_secs": None, "replay_file": None, - "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, - "run_all_feature_flags_no_tests": False, + "run_no_feature_flag_tests": False, "additional_feature_flags": None, + "additional_feature_flags_file": None, "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, @@ -166,7 +166,6 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "num_repeat_tests_max", "num_repeat_tests_min", "time_repeat_tests_secs", - "report_failure_status", ]) @@ -233,7 +232,6 @@ class SuiteOptions(_SuiteOptions): REPEAT_TESTS_MAX, REPEAT_TESTS_MIN, REPEAT_TESTS_SECS, - REPORT_FAILURE_STATUS, ]))) options = self._asdict() @@ -375,8 +373,11 @@ INSTALL_DIR = None # Whether to run tests for feature flags. RUN_ALL_FEATURE_FLAG_TESTS = None -# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true. -RUN_ALL_FEATURE_FLAGS = None +# Whether to run the tests with enabled feature flags +RUN_NO_FEATURE_FLAG_TESTS = None + +# the path to a file containing feature flags +ADDITIONAL_FEATURE_FLAGS_FILE = None # List of enabled feature flags. ENABLED_FEATURE_FLAGS = [] @@ -433,9 +434,6 @@ REPEAT_TESTS_MIN = None # If set, then each test is repeated the specified time (seconds) inside the suites. REPEAT_TESTS_SECS = None -# Controls if the test failure status should be reported as failed or be silently ignored. -REPORT_FAILURE_STATUS = None - # If set, then resmoke.py will write out a report file with the status of each test that ran. REPORT_FILE = None @@ -576,3 +574,12 @@ SHORTEN_LOGGER_NAME_CONFIG: dict = {} # that get loaded for multiversion tests can behave correctly on master and on v5.0; the latter # case runs 5.0 and 4.4 binaries and has this value set to True. Can be removed after 6.0. USE_LEGACY_MULTIVERSION = True + +# Expansions file location +# in CI, the expansions file is located in the ${workdir}, one dir up +# from src, the checkout directory +EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else None + +# Symbolizer secrets +SYMBOLIZER_CLIENT_SECRET = None +SYMBOLIZER_CLIENT_ID = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index db1319b8343..23c7ec69597 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -15,6 +15,7 @@ import shlex import pymongo.uri_parser +from buildscripts.idl import gen_all_feature_flag_list from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config @@ -27,6 +28,7 @@ def validate_and_update_config(parser, args): """Validate inputs and update config module.""" _validate_options(parser, args) _update_config_vars(args) + _update_symbolizer_secrets() _validate_config(parser) _set_logging_config() @@ -51,14 +53,9 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) - if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests: - if not os.path.isfile(ALL_FEATURE_FLAG_FILE): - parser.error( - "To run tests with all feature flags, the %s file must exist and be placed in" - " your working directory. The file can be downloaded from the artifacts tarball" - " in Evergreen. Alternatively, if you know which feature flags you want to enable," - " you can use the --additionalFeatureFlags command line argument" % - ALL_FEATURE_FLAG_FILE) + if args.additional_feature_flags_file and not os.path.isfile( + args.additional_feature_flags_file): + parser.error("The specified additional feature flags file does not exist.") def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) @@ -184,28 +181,36 @@ be invoked as either: - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""") raise RuntimeError(err) + def process_feature_flag_file(path): + with open(path) as fd: + return fd.read().split() + def setup_feature_flags(): _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") - _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests") + _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests") + _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") - # Running all feature flag tests implies running the fixtures with feature flags. if _config.RUN_ALL_FEATURE_FLAG_TESTS: - _config.RUN_ALL_FEATURE_FLAGS = True + print("Generating: ", ALL_FEATURE_FLAG_FILE) + gen_all_feature_flag_list.gen_all_feature_flags_file() all_ff = [] enabled_feature_flags = [] try: - with open(ALL_FEATURE_FLAG_FILE) as fd: - all_ff = fd.read().split() + all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE) except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS: raise - if _config.RUN_ALL_FEATURE_FLAGS: + if _config.RUN_ALL_FEATURE_FLAG_TESTS: enabled_feature_flags = all_ff[:] + if _config.ADDITIONAL_FEATURE_FLAGS_FILE: + enabled_feature_flags.extend( + process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE)) + # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags")) @@ -229,7 +234,7 @@ be invoked as either: _config.EXCLUDE_WITH_ANY_TAGS.extend( utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS: + if _config.RUN_NO_FEATURE_FLAG_TESTS: # Don't run any feature flag tests. _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags) else: @@ -302,6 +307,8 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG, _config.WT_COLL_CONFIG, \ _config.WT_INDEX_CONFIG = mongod_fuzzer_configs.fuzz_set_parameters( _config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS) + _config.EXCLUDE_WITH_ANY_TAGS.extend(["uses_compact"]) + _config.EXCLUDE_WITH_ANY_TAGS.extend(["requires_emptycapped"]) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) mongos_set_parameters = config.pop("mongos_set_parameters") @@ -322,7 +329,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") - _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config") @@ -502,3 +508,13 @@ def _tags_from_list(tags_list): tags.extend([t for t in tag.split(",") if t != ""]) return tags return None + + +def _update_symbolizer_secrets(): + """Open `expansions.yml`, get values for symbolizer secrets and update their values inside config.py .""" + if not _config.EVERGREEN_TASK_ID: + # not running on Evergreen + return + yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) + _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") + _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 19354d4c314..2a22d177a29 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -251,6 +251,11 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');") + if config.FUZZ_MONGOD_CONFIGS is not None and config.FUZZ_MONGOD_CONFIGS is not False: + # Prevent commands from running with the config fuzzer. + eval_sb.append( + "load('jstests/libs/override_methods/config_fuzzer_incompatible_commands.js');") + # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');") diff --git a/buildscripts/resmokelib/flags.py b/buildscripts/resmokelib/flags.py new file mode 100644 index 00000000000..6aff6961666 --- /dev/null +++ b/buildscripts/resmokelib/flags.py @@ -0,0 +1,5 @@ +"""Global flags used by resmoke.""" + +import threading + +HANG_ANALYZER_CALLED = threading.Event() diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index e5112b870b3..2c4d4b761d7 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -458,7 +458,15 @@ class GDBDumper(Dumper): cmds = self._prefix() + self._process_specific(pinfo, take_dump, logger) + self._postfix() - call([dbg, "--quiet", "--nx"] + list( + # gcore is both a command within GDB and a script packaged alongside gdb. The gcore script + # invokes the gdb binary with --readnever to avoid spending time loading the debug symbols + # prior to taking the core dump. The debug symbols are unneeded to generate the core dump. + # + # For reference + # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/gcore.in;h=34860de630cf0ee766e102eb82f7a3fddba6b368#l101 + skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] + + call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, diff --git a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py index 44d0b23e54e..ba259955444 100755 --- a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py +++ b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py @@ -68,7 +68,13 @@ class HangAnalyzer(Subcommand): self._configure_processes() self._setup_logging(logger) - def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements + def kill_rogue_processes(self): + """Kill any processes that are currently being analyzed.""" + processes = process_list.get_processes(self.process_ids, self.interesting_processes, + self.options.process_match, self.root_logger) + process.teardown_processes(self.root_logger, processes, dump_pids={}) + + def execute(self): # pylint: disable=too-many-branches """ Execute hang analysis. diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index 6325ecabb89..fbd3f1a9bfc 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -298,6 +298,7 @@ class BuildloggerServer(object): "builder": builder, "buildnum": build_num, "task_id": _config.EVERGREEN_TASK_ID, + "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -316,6 +317,7 @@ class BuildloggerServer(object): "command": test_command, "phase": self.config.get("build_phase", "unknown"), "task_id": _config.EVERGREEN_TASK_ID, + "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongod_fuzzer_configs.py index ec84d6c5a4e..874bc7764cf 100644 --- a/buildscripts/resmokelib/mongod_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongod_fuzzer_configs.py @@ -11,13 +11,16 @@ def generate_eviction_configs(rng): eviction_trigger = rng.randint(eviction_target + 1, 99) # Fuzz eviction_dirty_target and trigger both as relative and absolute values - target_bytes_min = 10 * 1024 * 1024 # 10MB + target_bytes_min = 50 * 1024 * 1024 # 50MB # 5% of 1GB default cache size on Evergreen target_bytes_max = 256 * 1024 * 1024 # 256MB # 1GB default cache size on Evergreen eviction_dirty_target = rng.choice( [rng.randint(5, 50), rng.randint(target_bytes_min, target_bytes_max)]) trigger_max = 75 if eviction_dirty_target <= 50 else target_bytes_max eviction_dirty_trigger = rng.randint(eviction_dirty_target + 1, trigger_max) + assert eviction_dirty_trigger > eviction_dirty_target + assert eviction_dirty_trigger <= trigger_max + close_idle_time_secs = rng.randint(1, 100) close_handle_minimum = rng.randint(0, 1000) close_scan_interval = rng.randint(1, 100) @@ -89,6 +92,8 @@ def generate_independent_parameters(rng): # The old retryable writes format is used by other variants. Weight towards turning on the # new retryable writes format on in this one. ret["storeFindAndModifyImagesInSideCollection"] = True + # TODO (SERVER-75632): Uncomment this to enable passthrough testing. + # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) return ret diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 2c9bae45766..713dedf0f82 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -49,14 +49,14 @@ class MultiversionConfigSubcommand(Subcommand): mongo_version=MongoVersion.from_yaml_file(multiversionconstants.MONGO_VERSION_YAML), mongo_releases=MongoReleases.from_yaml_file(multiversionconstants.RELEASES_YAML), ) - fcv_constants = multiversion_service.calculate_fcv_constants() + version_constants = multiversion_service.calculate_version_constants() return MultiversionConfig( last_versions=multiversionconstants.OLD_VERSIONS, - requires_fcv_tag=fcv_constants.get_fcv_tag_list(), - requires_fcv_tag_lts=fcv_constants.get_lts_fcv_tag_list(), - requires_fcv_tag_continuous=fcv_constants.get_continuous_fcv_tag_list(), - last_lts_fcv=fcv_constants.get_last_lts_fcv(), - last_continuous_fcv=fcv_constants.get_last_continuous_fcv(), + requires_fcv_tag=version_constants.get_fcv_tag_list(), + requires_fcv_tag_lts=version_constants.get_lts_fcv_tag_list(), + requires_fcv_tag_continuous=version_constants.get_continuous_fcv_tag_list(), + last_lts_fcv=version_constants.get_last_lts_fcv(), + last_continuous_fcv=version_constants.get_last_continuous_fcv(), ) diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 3597cc2177f..621d5ade9ae 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -25,9 +25,9 @@ def version_str(version: Version) -> str: return f"{version.major}.{version.minor}" -class FcvConstantValues(NamedTuple): +class VersionConstantValues(NamedTuple): """ - Object to hold the calculated FCV constants. + Object to hold the calculated Version constants. * latest: Latest FCV. * last_continuous: Last continuous FCV. @@ -36,6 +36,7 @@ class FcvConstantValues(NamedTuple): * requires_fcv_tag_list_continuous: List of FCVs that we need to generate a tag for against continuous versions. * fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0. + * eols: List of stable MongoDB versions since v2.0 that have been EOL'd. """ latest: Version @@ -44,6 +45,7 @@ class FcvConstantValues(NamedTuple): requires_fcv_tag_list: List[Version] requires_fcv_tag_list_continuous: List[Version] fcvs_less_than_latest: List[Version] + eols: List[Version] def get_fcv_tag_list(self) -> str: """Get a comma joined string of all the fcv tags.""" @@ -96,6 +98,10 @@ class FcvConstantValues(NamedTuple): last_continuous = self.get_last_continuous_fcv() return f"{base_name}-{last_continuous}" + def get_eols(self) -> List[str]: + """Get EOL'd versions as list of strings.""" + return [version_str(eol) for eol in self.eols] + class MongoVersion(BaseModel): """ @@ -132,12 +138,14 @@ class MongoReleases(BaseModel): * feature_compatibility_version: All FCVs starting with 4.0. * long_term_support_releases: All LTS releases starting with 4.0. + * eol_versions: List of stable MongoDB versions since 2.0 that have been EOL'd. * generate_fcv_lower_bound_override: Extend FCV generation down to the previous value of last LTS. """ feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions") long_term_support_releases: List[str] = Field(alias="longTermSupportReleases") + eol_versions: List[str] = Field(alias="eolVersions") generate_fcv_lower_bound_override: Optional[str] = Field(None, alias="generateFCVLowerBoundOverride") @@ -159,7 +167,11 @@ class MongoReleases(BaseModel): def get_lts_versions(self) -> List[Version]: """Get the Version representation of the lts versions.""" - return [Version(fcv) for fcv in self.long_term_support_releases] + return [Version(lts) for lts in self.long_term_support_releases] + + def get_eol_versions(self) -> List[Version]: + """Get the Version representation of the EOL versions.""" + return [Version(eol) for eol in self.eol_versions] class MultiversionService: @@ -175,11 +187,12 @@ class MultiversionService: self.mongo_version = mongo_version self.mongo_releases = mongo_releases - def calculate_fcv_constants(self) -> FcvConstantValues: + def calculate_version_constants(self) -> VersionConstantValues: """Calculate multiversion constants from data files.""" latest = self.mongo_version.get_version() fcvs = self.mongo_releases.get_fcv_versions() lts = self.mongo_releases.get_lts_versions() + eols = self.mongo_releases.get_eol_versions() lower_bound_override = self.mongo_releases.generate_fcv_lower_bound_override # Highest release less than latest. @@ -200,7 +213,12 @@ class MultiversionService: # All FCVs less than latest. fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)] - return FcvConstantValues(latest=latest, last_continuous=last_continuous, last_lts=last_lts, - requires_fcv_tag_list=requires_fcv_tag_list, - requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, - fcvs_less_than_latest=fcvs_less_than_latest) + return VersionConstantValues( + latest=latest, + last_continuous=last_continuous, + last_lts=last_lts, + requires_fcv_tag_list=requires_fcv_tag_list, + requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, + fcvs_less_than_latest=fcvs_less_than_latest, + eols=eols, + ) diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 0cac4fbc95b..3646d139b1f 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,25 +2,24 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output +import requests import structlog -try: - # when running resmoke - from buildscripts.resmokelib.multiversion.multiversion_service import ( - MongoReleases, MongoVersion, MultiversionService) - from buildscripts.resmokelib.multiversionsetupconstants import \ - USE_EXISTING_RELEASES_FILE -except ImportError: - # when running db-contrib-tool - from multiversion.multiversion_service import (MongoReleases, MongoVersion, MultiversionService) - from multiversionsetupconstants import USE_EXISTING_RELEASES_FILE +from buildscripts.resmokelib.multiversion.multiversion_service import ( + MongoReleases, MongoVersion, MultiversionService, MONGO_VERSION_YAML, RELEASES_YAML) +from buildscripts.resmokelib.multiversionsetupconstants import \ + USE_EXISTING_RELEASES_FILE -LOGGER = structlog.getLogger(__name__) +LAST_LTS = "last_lts" +LAST_CONTINUOUS = "last_continuous" + +# We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date +# with the latest EOL versions. If a "last-continuous" version is EOL, we don't include +# it in the multiversion config and therefore don't test against it. +MASTER_RELEASES_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" -# These values must match the include paths for artifacts.tgz in evergreen.yml. -MONGO_VERSION_YAML = ".resmoke_mongo_version.yml" -RELEASES_YAML = ".resmoke_mongo_release_values.yml" +LOGGER = structlog.getLogger(__name__) def generate_mongo_version_file(): @@ -40,14 +39,8 @@ def generate_mongo_version_file(): def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. - releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml") - if not os.path.isfile(releases_yaml_path): - LOGGER.info( - 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.' - .format(releases_yaml_path)) - return - - shutil.copyfile(releases_yaml_path, RELEASES_YAML) + with open(RELEASES_YAML, "wb") as file: + file.write(requests.get(MASTER_RELEASES_FILE).content) def in_git_root_dir(): @@ -86,33 +79,34 @@ multiversion_service = MultiversionService( mongo_releases=MongoReleases.from_yaml_file(RELEASES_YAML), ) -fcv_constants = multiversion_service.calculate_fcv_constants() +version_constants = multiversion_service.calculate_version_constants() -LAST_LTS_BIN_VERSION = fcv_constants.get_last_lts_fcv() -LAST_CONTINUOUS_BIN_VERSION = fcv_constants.get_last_continuous_fcv() +LAST_LTS_BIN_VERSION = version_constants.get_last_lts_fcv() +LAST_CONTINUOUS_BIN_VERSION = version_constants.get_last_continuous_fcv() -LAST_LTS_FCV = fcv_constants.get_last_lts_fcv() -LAST_CONTINUOUS_FCV = fcv_constants.get_last_continuous_fcv() -LATEST_FCV = fcv_constants.get_latest_fcv() +LAST_LTS_FCV = version_constants.get_last_lts_fcv() +LAST_CONTINUOUS_FCV = version_constants.get_last_continuous_fcv() +LATEST_FCV = version_constants.get_latest_fcv() -LAST_CONTINUOUS_MONGO_BINARY = fcv_constants.build_last_continuous_binary("mongo") -LAST_CONTINUOUS_MONGOD_BINARY = fcv_constants.build_last_continuous_binary("mongod") -LAST_CONTINUOUS_MONGOS_BINARY = fcv_constants.build_last_continuous_binary("mongos") +LAST_CONTINUOUS_MONGO_BINARY = version_constants.build_last_continuous_binary("mongo") +LAST_CONTINUOUS_MONGOD_BINARY = version_constants.build_last_continuous_binary("mongod") +LAST_CONTINUOUS_MONGOS_BINARY = version_constants.build_last_continuous_binary("mongos") -LAST_LTS_MONGO_BINARY = fcv_constants.build_last_lts_binary("mongo") -LAST_LTS_MONGOD_BINARY = fcv_constants.build_last_lts_binary("mongod") -LAST_LTS_MONGOS_BINARY = fcv_constants.build_last_lts_binary("mongos") +LAST_LTS_MONGO_BINARY = version_constants.build_last_lts_binary("mongo") +LAST_LTS_MONGOD_BINARY = version_constants.build_last_lts_binary("mongod") +LAST_LTS_MONGOS_BINARY = version_constants.build_last_lts_binary("mongos") -REQUIRES_FCV_TAG_LATEST = fcv_constants.get_latest_tag() +REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() # Generate tags for all FCVS in (lastLTS, latest], or (lowerBoundOverride, latest] if requested. # All multiversion tests should be run with these tags excluded. -REQUIRES_FCV_TAG = fcv_constants.get_fcv_tag_list() +REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] -EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest]) +EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) -OLD_VERSIONS = ["last_lts"] -if LAST_LTS_FCV != LAST_CONTINUOUS_FCV: - OLD_VERSIONS.append("last_continuous") +OLD_VERSIONS = [ + LAST_LTS +] if LAST_CONTINUOUS_FCV == LAST_LTS_FCV or LAST_CONTINUOUS_FCV in version_constants.get_eols( +) else [LAST_LTS, LAST_CONTINUOUS] diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 5f5bb7f3765..e450540f293 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -11,7 +11,6 @@ from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.multiversion import MultiversionPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin -from buildscripts.resmokelib.symbolizer import SymbolizerPlugin from buildscripts.resmokelib.undodb import UndoDbPlugin _PLUGINS = [ @@ -19,7 +18,6 @@ _PLUGINS = [ HangAnalyzerPlugin(), UndoDbPlugin(), PowercyclePlugin(), - SymbolizerPlugin(), GenerateFCVConstantsPlugin(), DiscoveryPlugin(), MultiversionPlugin(), diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 20ff72c2c79..420825bfec4 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -385,25 +385,12 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ - UNRELIABLE_TAG = _TagInfo( - tag_name="unreliable", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - report_failure_status="silentfail")) - RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) - RETRY_ON_FAILURE_TAG = _TagInfo( - tag_name="retry_on_failure", - evergreen_aware=True, - suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, - report_failure_status="silentfail")) - @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -438,29 +425,8 @@ class TestRunnerEvg(TestRunner): combinations = [] - if config.EVERGREEN_PATCH_BUILD: - combinations.append(("unreliable and resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("unreliable and not resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) - combinations.append(("reliable and resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("reliable and not resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - else: - combinations.append(("retry on failure and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("retry on failure and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - combinations.append(("run once and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("run once and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) + combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) + combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) return combinations @@ -740,15 +706,19 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", - action="store_true", help= - "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" - ) + "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", + help=("Do not run any tests tagged with enabled feature flags." + " This argument has precedence over --runAllFeatureFlagTests" + "; used for multiversion suites")) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") + parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", + action="store", metavar="FILE", + help="The path to a file with feature flags, delimited by newlines.") + parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -879,13 +849,6 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( - "--reportFailureStatus", action="store", dest="report_failure_status", - choices=("fail", "silentfail"), metavar="STATUS", - help="Controls if the test failure status should be reported as failed" - " or be silently ignored (STATUS=silentfail). Dynamic test failures will" - " never be silently ignored. Defaults to STATUS=%(default)s.") - - internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") @@ -1096,15 +1059,6 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) - # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on - # all_feature_flags.txt which may not exist in the local dev environment. - run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) - if run_all_feature_flag_tests is not None: - setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) - del parsed_args.run_all_feature_flag_tests - - del parsed_args.run_all_feature_flags_no_tests - # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py index 5df67812d06..609ea8a6e3e 100644 --- a/buildscripts/resmokelib/sighandler.py +++ b/buildscripts/resmokelib/sighandler.py @@ -10,10 +10,10 @@ import traceback import psutil +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib import reportfile from buildscripts.resmokelib import testing from buildscripts.resmokelib import config -from buildscripts.resmokelib.hang_analyzer import hang_analyzer from buildscripts.resmokelib import parser _IS_WINDOWS = (sys.platform == "win32") @@ -32,8 +32,8 @@ def register(logger, suites, start_time): log suite summaries. """ + HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to SIGUSR1 signal" - _dump_and_log(header_msg) def _handle_set_event(event_handle): @@ -53,6 +53,7 @@ def register(logger, suites, start_time): except win32event.error as err: logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) else: + HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to signal from win32event.SetEvent" _dump_and_log(header_msg) @@ -159,4 +160,26 @@ def _analyze_pids(logger, pids): if not os.getenv('ASAN_OPTIONS'): hang_analyzer_args.append('-c') _hang_analyzer = parser.parse_command_line(hang_analyzer_args, logger=logger) - _hang_analyzer.execute() + + # Evergreen has a 15 minute timeout for task timeout commands + # Limit the hang analyzer to 12 minutes so there is time for other tasks. + hang_analyzer_hard_timeout = None + if config.EVERGREEN_TASK_ID: + hang_analyzer_hard_timeout = 60 * 12 + logger.info( + "Limit the resmoke invoked hang analyzer to 12 minutes so there is time for resmoke to finish up." + ) + + hang_analyzer_thread = threading.Thread(target=_hang_analyzer.execute, daemon=True) + hang_analyzer_thread.start() + hang_analyzer_thread.join(hang_analyzer_hard_timeout) + + if hang_analyzer_thread.is_alive(): + logger.warning( + "Resmoke invoked hang analyzer thread did not finish, but will continue running in the background. The thread may be disruputed and may show extraneous output." + ) + logger.warning("Cleaning up resmoke child processes so that resmoke can fail gracefully.") + _hang_analyzer.kill_rogue_processes() + + else: + logger.info("Done running resmoke invoked hang analyzer thread.") diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 394c025d510..55c23923ecc 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -4,6 +4,8 @@ import os import os.path import time import shutil +import uuid + import yaml import pymongo @@ -23,6 +25,9 @@ class MongoDFixture(interface.Fixture): self.mongod_options = self.fixturelib.make_historic( self.fixturelib.default_if_none(mongod_options, {})) + if "set_parameters" not in self.mongod_options: + self.mongod_options["set_parameters"] = {} + if add_feature_flags: for ff in self.config.ENABLED_FEATURE_FLAGS: self.mongod_options["set_parameters"][ff] = "true" @@ -52,6 +57,11 @@ class MongoDFixture(interface.Fixture): self.port = port or fixturelib.get_next_port(job_num) self.mongod_options["port"] = self.port + # Always log backtraces to a file in the dbpath in our testing. + backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(), + uuid.uuid4().hex + ".stacktrace") + self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name + def setup(self): """Set up the mongod.""" if not self.preserve_dbpath and os.path.lexists(self._dbpath): @@ -61,7 +71,8 @@ class MongoDFixture(interface.Fixture): launcher = MongodLauncher(self.fixturelib) # Second return val is the port, which we ignore because we explicitly created the port above. - # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 + # The port is used to set other mongod_option's here: + # https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num, executable=self.mongod_executable, mongod_options=self.mongod_options) @@ -168,7 +179,7 @@ class MongoDFixture(interface.Fixture): def get_driver_connection_url(self): """Return the driver connection URL.""" - return "mongodb://" + self.get_internal_connection_string() + return "mongodb://" + self.get_internal_connection_string() + "/?directConnection=true" # The below parameters define the default 'logComponentVerbosity' object passed to mongod processes @@ -180,15 +191,16 @@ class MongoDFixture(interface.Fixture): # The default verbosity setting for any tests that are not started with an Evergreen task id. This # will apply to any tests run locally. DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"rollback": 2}, "sharding": {"migration": 2}, "transaction": 4, - "tenantMigration": 4 + "replication": {"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, + "transaction": 4, "tenantMigration": 4 } # The default verbosity setting for any mongod processes running in Evergreen i.e. started with an # Evergreen task id. DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, "rollback": 2}, - "sharding": {"migration": 2}, "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 + "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, + "rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, + "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 } @@ -242,6 +254,15 @@ class MongodLauncher(object): if "shardsvr" in mongod_options and "orphanCleanupDelaySecs" not in suite_set_parameters: suite_set_parameters["orphanCleanupDelaySecs"] = 1 + # receiveChunkWaitForRangeDeleterTimeoutMS controls the amount of time an incoming migration + # will wait for an intersecting range with data in it to be cleared up before failing. The + # default is 10 seconds, but in some slower variants this is not enough time for the range + # deleter to finish so we increase it here to 90 seconds. Setting a value for this parameter + # in the .yml file overrides this. + if (("shardsvr" in mongod_options or "configsvr" in mongod_options) + and "receiveChunkWaitForRangeDeleterTimeoutMS" not in suite_set_parameters): + suite_set_parameters["receiveChunkWaitForRangeDeleterTimeoutMS"] = 90000 + # The LogicalSessionCache does automatic background refreshes in the server. This is # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off # background refreshing for tests. Set in the .yml file to override this. @@ -292,6 +313,13 @@ class MongodLauncher(object): "mode": "alwaysOn", "data": {"numTickets": self.config.FLOW_CONTROL_TICKETS} } + # The internalQueryForceClassicEngine parameter was renamed to + # internalQueryFrameworkControl starting in version 6.1. + if "internalQueryFrameworkControl" in suite_set_parameters and suite_set_parameters[ + "internalQueryFrameworkControl"] == "forceClassicEngine": + del suite_set_parameters["internalQueryFrameworkControl"] + suite_set_parameters["internalQueryForceClassicEngine"] = True + _add_testing_set_parameters(suite_set_parameters) shortcut_opts = { @@ -368,3 +396,7 @@ def _add_testing_set_parameters(suite_set_parameters): """ suite_set_parameters.setdefault("testingDiagnosticsEnabled", True) suite_set_parameters.setdefault("enableTestCommands", True) + # The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up. + # Set it to true for now as a placeholder that will error if no further processing is done. + # The placeholder is needed so older versions don't have this option won't have this value set. + suite_set_parameters.setdefault("backtraceLogFile", True) diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index 38909056ce3..23ebfda8de3 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -6,6 +6,7 @@ import threading from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib import utils +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib.utils import globstar @@ -105,5 +106,9 @@ class HookTestArchival(object): else: logger.info("Archive succeeded for %s: %s", test_name, message) - if not manager.setup_fixture(logger): + if HANG_ANALYZER_CALLED.is_set(): + logger.info("Hang Analyzer has been called. Fixtures will not be restarted.") + raise errors.StopExecution( + "Hang analyzer has been called. Stopping further execution of tests.") + elif not manager.setup_fixture(logger): raise errors.StopExecution("Error while restarting test fixtures after archiving.") diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index b0ff858c5dc..5278819da8f 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -276,16 +276,15 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): secondary.await_ready() client = secondary.mongo_client() - minvalid_doc = client.local["replset.minvalid"].find_one() oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one() recovery_timestamp_res = client.admin.command("replSetTest", getLastStableRecoveryTimestamp=True) latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural", pymongo.DESCENDING)]) - self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {}," + self.logger.info("Checking replication invariants. oplogTruncateAfterPoint: {}," " stable recovery timestamp: {}, latest oplog doc: {}".format( - minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res, + oplog_truncate_after_doc, recovery_timestamp_res, latest_oplog_doc)) null_ts = bson.Timestamp(0, 0) @@ -299,13 +298,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): raise errors.ServerFailure( "Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc)) - # The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default - # it to null. - oplog_truncate_after_ts = null_ts - if oplog_truncate_after_doc is not None: - oplog_truncate_after_ts = oplog_truncate_after_doc.get( - "oplogTruncateAfterPoint", null_ts) - # The "lastStableRecoveryTimestamp" field is present if the storage engine supports # "recover to a timestamp". If it's a null timestamp on a durable storage engine, that # means we do not yet have a stable checkpoint timestamp and must be restarting at the @@ -328,94 +320,6 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): recovery_timestamp, latest_oplog_entry_ts, recovery_timestamp_res, latest_oplog_doc)) - if minvalid_doc is not None: - applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts) - minvalid_ts = minvalid_doc.get("ts", null_ts) - - # The "appliedThrough" value should always equal the "last stable recovery - # timestamp", AKA the stable checkpoint for durable engines, on server restart. - # - # The written "appliedThrough" time is updated with the latest timestamp at the end - # of each batch application, and batch boundaries are the only valid stable - # timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must - # equal the checkpoint timestamp, because any stable timestamp that the checkpoint - # could use includes an equal persisted appliedThrough timestamp. - if (recovery_timestamp != null_ts and applied_through_ts != null_ts - and (not recovery_timestamp == applied_through_ts)): - raise errors.ServerFailure( - "The condition last stable recovery timestamp ({}) == appliedThrough ({})" - " doesn't hold: minValid document={}," - " getLastStableRecoveryTimestamp result={}, last oplog entry={}".format( - recovery_timestamp, applied_through_ts, minvalid_doc, - recovery_timestamp_res, latest_oplog_doc)) - - if applied_through_ts == null_ts: - # We clear "appliedThrough" to represent having applied through the top of the - # oplog in PRIMARY state or immediately after "rollback via refetch". - # If we are using a storage engine that supports "recover to a timestamp," - # then we will have a "last stable recovery timestamp" and we should use that - # as our "appliedThrough" (similarly to why we assert their equality above). - # If both are null, then we are in PRIMARY state on a storage engine that does - # not support "recover to a timestamp" or in RECOVERING immediately after - # "rollback via refetch". Since we do not update "minValid" in PRIMARY state, - # we leave "appliedThrough" as null so that the invariants below hold, rather - # than substituting the latest oplog entry for the "appliedThrough" value. - applied_through_ts = recovery_timestamp - - if minvalid_ts == null_ts: - # The server treats the "ts" field in the minValid document as missing when its - # value is the null timestamp. - minvalid_ts = applied_through_ts - - if latest_oplog_entry_ts == null_ts: - # If the oplog is empty, we treat the "minValid" as the latest oplog entry. - latest_oplog_entry_ts = minvalid_ts - - if oplog_truncate_after_ts == null_ts: - # The server treats the "oplogTruncateAfterPoint" field as missing when its - # value is the null timestamp. When it is null, the oplog is complete and - # should not be truncated, so it is effectively the top of the oplog. - oplog_truncate_after_ts = latest_oplog_entry_ts - - # Check the ordering invariants before the secondary has reconciled the end of - # its oplog. - # The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of - # oplog entries before they are written to the oplog. Thus, it can be ahead - # of the top of the oplog before any oplog entries are written, and behind it - # after some are written. Thus, we cannot compare it to the top of the oplog. - - # appliedThrough <= minValid - # appliedThrough represents the end of the previous batch, so it is always the - # earliest. - if applied_through_ts > minvalid_ts: - raise errors.ServerFailure( - "The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid" - " document={}, latest oplog entry={}".format( - applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc)) - - # minValid <= oplogTruncateAfterPoint - # This is true because this hook is never run after a rollback. Thus, we only - # move "minValid" to the end of each batch after the batch is written to the oplog. - # We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from - # the end of the previous batch to the end of the current batch. Thus "minValid" - # must be less than or equal to the "oplogTruncateAfterPoint". - if minvalid_ts > oplog_truncate_after_ts: - raise errors.ServerFailure( - "The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't" - " hold: minValid document={}, oplogTruncateAfterPoint document={}," - " latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts, - minvalid_doc, oplog_truncate_after_doc, - latest_oplog_doc)) - - # minvalid <= latest oplog entry - # "minValid" is set to the end of a batch after the batch is written to the oplog. - # Thus it is always less than or equal to the top of the oplog. - if minvalid_ts > latest_oplog_entry_ts: - raise errors.ServerFailure( - "The condition minValid <= top of oplog ({} <= {}) doesn't" - " hold: minValid document={}, latest oplog entry={}".format( - minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc)) - try: secondary.teardown() except errors.ServerFailure: diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index c09602baee8..8fb565cd1d8 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -363,7 +363,7 @@ class FileBasedStepdownLifecycle(object): # We remove the "permitted" file to revoke permission for the stepdown thread to continue # performing stepdowns. - os.remove(self.__stepdown_files.permitted) + fs.remove_if_exists(self.__stepdown_files.permitted) class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-attributes @@ -627,6 +627,10 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at break except pymongo.errors.NotMasterError: pass + except pymongo.errors.OperationFailure as ex: + if ex.code == 166: # CommandNotSupportedOnView + # listCollections return also views and collStats is not supported on views + break retarget_time = time.time() - start_time if retarget_time >= 60: raise RuntimeError( diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index 38fe409b656..0aa87390e59 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -10,6 +10,7 @@ import unittest from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import logging +from buildscripts.resmokelib.testing.symbolizer_service import ResmokeSymbolizer # pylint: disable=attribute-defined-outside-init @@ -54,7 +55,9 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for report in reports: if not isinstance(report, TestReport): - raise TypeError("reports must be a list of TestReport instances") + raise TypeError( + f"reports must be a list of TestReport instances, current report is {type(report)}" + ) with report._lock: # pylint: disable=protected-access for test_info in report.test_infos: @@ -136,28 +139,37 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr def stopTest(self, test): # pylint: disable=invalid-name """Call after 'test' has run.""" - unittest.TestResult.stopTest(self, test) - - with self._lock: - test_info = self.find_test_info(test) - test_info.end_time = time.time() - test_status = "no failures detected" if test_info.status == "pass" else "failed" - - time_taken = test_info.end_time - test_info.start_time - self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, - test_status) - - # Asynchronously closes the buildlogger test handler to avoid having too many threads open - # on 32-bit systems. - for handler in test.logger.handlers: - # We ignore the cancellation token returned by close_later() since we always want the - # logs to eventually get flushed. - logging.flush.close_later(handler) - - # Restore the original logger for the test. - test.reset_logger() - - def addError(self, test, err): # pylint: disable=invalid-name + try: + # check if there are stacktrace files, if so, invoke the symbolizer here. + # log symbolized output to test.logger.info() + symbolizer = ResmokeSymbolizer() + symbolizer.symbolize_test_logs(test) + + unittest.TestResult.stopTest(self, test) + + with self._lock: + test_info = self.find_test_info(test) + test_info.end_time = time.time() + test_status = "no failures detected" if test_info.status == "pass" else "failed" + + time_taken = test_info.end_time - test_info.start_time + self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, + test_status) + + finally: + # This is a failsafe. In the event that 'stopTest' fails, + # any rogue logger handlers will be removed from this test. + # If not cleaned up, these will trigger 'setup failures' -- + # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE. + for handler in test.logger.handlers: + # We ignore the cancellation token returned by close_later() since we always want the + # logs to eventually get flushed. + logging.flush.close_later(handler) + + # Restore the original logger for the test. + test.reset_logger() + + def addError(self, test, err): """Call when a non-failureException was raised during the execution of 'test'.""" unittest.TestResult.addError(self, test, err) @@ -201,12 +213,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = self.find_test_info(test) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = test.return_code def setFailure(self, test, return_code=1): # pylint: disable=invalid-name @@ -218,12 +225,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr raise ValueError("stopTest was not called on %s" % (test.basename())) test_info.status = "fail" - if test_info.dynamic: - # Dynamic tests are used for data consistency checks, so the failures are never - # silenced. - test_info.evergreen_status = "fail" - else: - test_info.evergreen_status = self.suite_options.report_failure_status + test_info.evergreen_status = "fail" test_info.return_code = return_code # Recompute number of success, failures, and errors. diff --git a/buildscripts/resmokelib/testing/symbolizer_service.py b/buildscripts/resmokelib/testing/symbolizer_service.py new file mode 100644 index 00000000000..9e78b0dd6b7 --- /dev/null +++ b/buildscripts/resmokelib/testing/symbolizer_service.py @@ -0,0 +1,294 @@ +"""Symbolize stacktraces inside test logs.""" +from __future__ import annotations + +import os +import subprocess +import sys +import time +from datetime import timedelta +from threading import Lock + +from typing import List, Optional, NamedTuple + +from buildscripts.resmokelib import config as _config +from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED +from buildscripts.resmokelib.testing.testcases.interface import TestCase + +# This lock prevents different resmoke jobs from symbolizing stacktraces concurrently, +# which includes downloading the debug symbols, that can be reused by other resmoke jobs +_lock = Lock() + +STACKTRACE_FILE_EXTENSION = ".stacktrace" +SYMBOLIZE_RETRY_TIMEOUT_SECS = timedelta(minutes=4).total_seconds() + + +class ResmokeSymbolizerConfig(NamedTuple): + """ + Resmoke symbolizer config. + + * evg_task_id: evergreen task ID resmoke runs on + * client_id: symbolizer client ID + * client_secret: symbolizer client secret + """ + + evg_task_id: Optional[str] + client_id: Optional[str] + client_secret: Optional[str] + + @classmethod + def from_resmoke_config(cls) -> ResmokeSymbolizerConfig: + """ + Make resmoke symbolizer config from a global resmoke config. + + :return: resmoke symbolizer config + """ + return cls( + evg_task_id=_config.EVERGREEN_TASK_ID, + client_id=_config.SYMBOLIZER_CLIENT_ID, + client_secret=_config.SYMBOLIZER_CLIENT_SECRET, + ) + + @staticmethod + def is_windows() -> bool: + """ + Whether we are on Windows. + + :return: True if on Windows + """ + return sys.platform == "win32" or sys.platform == "cygwin" + + @staticmethod + def is_macos() -> bool: + """ + Whether we are on MacOS. + + :return: True if on MacOS. + """ + return sys.platform == "darwin" + + +class ResmokeSymbolizer: + """Symbolize stacktraces inside test logs.""" + + def __init__(self, config: Optional[ResmokeSymbolizerConfig] = None, + symbolizer_service: Optional[SymbolizerService] = None, + file_service: Optional[FileService] = None): + """Initialize instance.""" + + self.config = config if config is not None else ResmokeSymbolizerConfig.from_resmoke_config( + ) + self.symbolizer_service = symbolizer_service if symbolizer_service is not None else SymbolizerService( + ) + self.file_service = file_service if file_service is not None else FileService() + + def symbolize_test_logs(self, test: TestCase, + symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS) -> None: + """ + Perform all necessary actions to symbolize and write output to test logs. + + :param test: resmoke test case + :param symbolize_retry_timeout: the timeout for symbolizer retries + """ + if not self.should_symbolize(test): + return + + dbpath = self.get_stacktrace_dir(test) + if dbpath is None: + return + + test.logger.info("Looking for stacktrace files in '%s'", dbpath) + files = self.collect_stacktrace_files(dbpath) + if not files: + test.logger.info("No failure logs/stacktrace files found, skipping symbolization") + return + + with _lock: + test.logger.info("Found stacktrace files. \nBEGIN Symbolization") + test.logger.info("Stacktrace files: %s", files) + + start_time = time.perf_counter() + for file_path in files: + test.logger.info("Working on: %s", file_path) + symbolizer_script_timeout = int(symbolize_retry_timeout - + (time.perf_counter() - start_time)) + symbolized_out = self.symbolizer_service.run_symbolizer_script( + file_path, symbolizer_script_timeout) + test.logger.info(symbolized_out) + if time.perf_counter() - start_time > symbolize_retry_timeout: + break + + # To avoid performing the same actions on these files again, we remove them + self.file_service.remove_all(files) + + test.logger.info("\nEND Symbolization \nSymbolization process completed. ") + + def should_symbolize(self, test: TestCase) -> bool: + """ + Check whether we should perform symbolization process. + + :param test: resmoke test case + :return: whether we should symbolize + """ + if self.config.evg_task_id is None: + test.logger.info("Not running in Evergreen, skipping symbolization") + return False + + if self.config.client_id is None or self.config.client_secret is None: + test.logger.info("Symbolizer client secret and/or client ID are absent," + " skipping symbolization") + return False + + if self.config.is_windows(): + test.logger.info("Running on Windows, skipping symbolization") + return False + + if self.config.is_macos(): + test.logger.info("Running on MacOS, skipping symbolization") + return False + + if HANG_ANALYZER_CALLED.is_set(): + test.logger.info( + "Hang analyzer has been called, skipping symbolization to meet timeout constraints." + ) + return False + + return True + + def get_stacktrace_dir(self, test: TestCase) -> Optional[str]: + """ + Get dbpath from test case. + + :param test: resmoke test case + :return: dbpath or None + """ + if not hasattr(test, "fixture") or test.fixture is None: + test.logger.info("Test fixture is not available, could not get dbpath") + return None + + dbpath = test.fixture.get_dbpath_prefix() + if not self.file_service.check_path_exists(dbpath): + test.logger.info("dbpath '%s' directory not found", dbpath) + return None + + return dbpath + + def collect_stacktrace_files(self, dir_path: str) -> List[str]: + """ + Collect all stacktrace files which are not empty and return their full paths. + + :param dir_path: directory to look into + :return: list of stacktrace files paths + """ + + files = self.file_service.find_all_children_recursively(dir_path) + files = self.file_service.filter_by_extension(files, STACKTRACE_FILE_EXTENSION) + self.file_service.remove_empty(files) + files = self.file_service.filter_out_non_files(files) + + return files + + +class FileService: + """A service for working with files.""" + + @staticmethod + def find_all_children_recursively(dir_path: str) -> List[str]: + """ + Find all children files in directory recursively. + + :param dir_path: directory path + :return: list of all children files + """ + children_in_dir = [] + for parent, _, children in os.walk(dir_path): + children_in_dir.extend(os.path.join(parent, child) for child in children) + return children_in_dir + + @staticmethod + def filter_by_extension(files: List[str], extension: str) -> List[str]: + """ + Filter files by extension. + + :param files: list of file paths + :param extension: file extension + :return: filtered list of file paths + """ + return [f for f in files if f.endswith(extension)] + + @staticmethod + def filter_out_non_files(files: List[str]) -> List[str]: + """ + Filter out non files. + + :param files: list of paths + :return: filtered list of file paths + """ + return [f for f in files if os.path.isfile(f)] + + @staticmethod + def remove_empty(files: List[str]) -> None: + """ + Delete files that are empty. + + :param files: list of paths + """ + for file in [f for f in files if os.stat(f).st_size == 0]: + os.remove(file) + + @staticmethod + def remove_all(files: List[str]) -> None: + """ + Delete all files. + + :param files: list of paths + """ + for file in files: + os.remove(file) + + @staticmethod + def check_path_exists(path: str) -> bool: + """ + Check that file or directory exists. + + :param path: file or directory path + :return: whether path exists + """ + return os.path.exists(path) + + +class SymbolizerService: + """Wrapper around symbolizer script.""" + + @staticmethod + def run_symbolizer_script(full_file_path: str, retry_timeout_secs: int) -> str: + """ + Symbolize given file and return symbolized output as string. + + :param full_file_path: stacktrace file path + :param retry_timeout_secs: the timeout for symbolizer to retry + :return: symbolized output as string + """ + + symbolizer_args = [ + "db-contrib-tool", + "symbolize", + "--client-secret", + _config.SYMBOLIZER_CLIENT_SECRET, + "--client-id", + _config.SYMBOLIZER_CLIENT_ID, + "--total-seconds-for-retries", + str(retry_timeout_secs), + ] + + with open(full_file_path) as file_obj: + symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, + stdin=file_obj, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + try: + output, _ = symbolizer_process.communicate(timeout=retry_timeout_secs) + except subprocess.TimeoutExpired: + symbolizer_process.kill() + output, _ = symbolizer_process.communicate() + + return output.strip().decode() diff --git a/buildscripts/resmokelib/utils/check_has_tag.py b/buildscripts/resmokelib/utils/check_has_tag.py index aebdd5644fe..18ae19bc21b 100755 --- a/buildscripts/resmokelib/utils/check_has_tag.py +++ b/buildscripts/resmokelib/utils/check_has_tag.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """CLI interface for jscomment.""" +import re import sys import jscomment @@ -16,11 +17,21 @@ try: print(' 3 if any error happened during check') print('Usage:') print(' check_has_tag.py <jsfile> <tag>') + print('Notice: <tag> is a regex, not search string') sys.exit(2) else: tags = jscomment.get_tags(sys.argv[1]) print(sys.argv[1], "has tags:", tags) - sys.exit(0 if sys.argv[2] in tags else 1) + + prog = re.compile(sys.argv[2]) + for tag in tags: + if prog.match(tag): + print("tag matches:", tag) + sys.exit(0) + + print("no tags match", sys.argv[2]) + sys.exit(1) + except Exception as err: # pylint: disable=W0703 print(err) sys.exit(3) diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py index 5857870e627..6153349b6af 100644 --- a/buildscripts/resmokelib/utils/globstar.py +++ b/buildscripts/resmokelib/utils/globstar.py @@ -1,11 +1,9 @@ """Filename globbing utility.""" import glob as _glob -import os import os.path import re -_GLOBSTAR = "**" _CONTAINS_GLOB_PATTERN = re.compile("[*?[]") @@ -35,155 +33,6 @@ def iglob(globbed_pathname): expanded to match zero or more subdirectories. """ - parts = _split_path(globbed_pathname) - parts = _canonicalize(parts) - - index = _find_globstar(parts) - if index == -1: - for pathname in _glob.iglob(globbed_pathname): - # Normalize 'pathname' so exact string comparison can be used later. - yield os.path.normpath(pathname) - return - - # **, **/, or **/a - if index == 0: - expand = _expand_curdir - - # a/** or a/**/ or a/**/b - else: - expand = _expand - - prefix_parts = parts[:index] - suffix_parts = parts[index + 1:] - - prefix = os.path.join(*prefix_parts) if prefix_parts else os.curdir - suffix = os.path.join(*suffix_parts) if suffix_parts else "" - - for (kind, path) in expand(prefix): - if not suffix_parts: - yield path - - # Avoid following symlinks to avoid an infinite loop - elif suffix_parts and kind == "dir" and not os.path.islink(path): - path = os.path.join(path, suffix) - for pathname in iglob(path): - yield pathname - - -def _split_path(pathname): - """Return 'pathname' as a list of path components.""" - - parts = [] - - while True: - (dirname, basename) = os.path.split(pathname) - parts.append(basename) - if pathname == dirname: - parts.append(dirname) - break - if not dirname: - break - pathname = dirname - - parts.reverse() - return parts - - -def _canonicalize(parts): - """Return a copy of 'parts' with consecutive "**"s coalesced. - - Raise a ValueError for unsupported uses of "**". - """ - - res = [] - - prev_was_globstar = False - for part in parts: - if part == _GLOBSTAR: - # Skip consecutive **'s - if not prev_was_globstar: - prev_was_globstar = True - res.append(part) - elif _GLOBSTAR in part: # a/b**/c or a/**b/c - raise ValueError("Can only specify glob patterns of the form a/**/b") - else: - prev_was_globstar = False - res.append(part) - - return res - - -def _find_globstar(parts): - """Return the index of the first occurrence of "**" in 'parts'. - - Return -1 if "**" is not found in the list. - """ - - for (idx, part) in enumerate(parts): - if part == _GLOBSTAR: - return idx - return -1 - - -def _list_dir(pathname): - """Return a pair of subdirectory names and filenames contained within the 'pathname' directory. - - If 'pathname' does not exist, then None is returned. - """ - - try: - (_root, dirs, files) = next(os.walk(pathname)) - return (dirs, files) - except StopIteration: - return None # 'pathname' directory does not exist - - -def _expand(pathname): - """Emit tuples of the form ("dir", dirname) and ("file", filename). - - The result is for all directories and files contained within the 'pathname' directory. - """ - - res = _list_dir(pathname) - if res is None: - return - - (dirs, files) = res - - # Zero expansion - if os.path.basename(pathname): - yield ("dir", os.path.join(pathname, "")) - - for fname in files: - path = os.path.join(pathname, fname) - yield ("file", path) - - for dname in dirs: - path = os.path.join(pathname, dname) - for xpath in _expand(path): - yield xpath - - -def _expand_curdir(pathname): - """Emit tuples of the form ("dir", dirname) and ("file", filename). - - The result is for all directories and files contained within the 'pathname' directory. - - The returned pathnames omit a "./" prefix. - """ - - res = _list_dir(pathname) - if res is None: - return - - (dirs, files) = res - - # Zero expansion - yield ("dir", "") - - for fname in files: - yield ("file", fname) - - for dname in dirs: - for xdir in _expand(dname): - yield xdir + for pathname in _glob.iglob(globbed_pathname, recursive=True): + # Normalize 'pathname' so exact string comparison can be used later. + yield os.path.normpath(pathname) diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py index 7af28d11ed8..d7c2c295492 100644 --- a/buildscripts/resmokelib/utils/jscomment.py +++ b/buildscripts/resmokelib/utils/jscomment.py @@ -36,6 +36,18 @@ def get_tags(pathname): tags = yaml.safe_load(_strip_jscomments(match.group(1))) if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) + + for tag in tags: + if '//' in tag: + raise ValueError(("Found a JS line comment '%s'. "\ + "Use '#' YAML style comments instead in a tags array %s") + % (tag, pathname)) + + if ' ' in tag: + raise ValueError(("Found an empty space in tag '%s'. "\ + "This is not permitted and may indicate a missing comma in %s") + % (tag, pathname)) + return tags except yaml.YAMLError as err: raise ValueError( diff --git a/buildscripts/scons_metrics/metrics.py b/buildscripts/scons_metrics/metrics.py index 1ba1f7825ad..6e9949f770c 100644 --- a/buildscripts/scons_metrics/metrics.py +++ b/buildscripts/scons_metrics/metrics.py @@ -1,5 +1,6 @@ """SCons metrics.""" import re +import os from typing import Optional, NamedTuple, List, Pattern, AnyStr from buildscripts.util.cedar_report import CedarMetric, CedarTestReport @@ -105,8 +106,11 @@ class SconsMetrics: # pylint: disable=too-many-instance-attributes self.total_command_execution_time = self._parse_float( TOTAL_COMMAND_EXECUTION_TIME_REGEX, self.raw_report) - with open(cache_debug_log_file, "r") as fh: - self.final_cache_hit_ratio = self._parse_float(CACHE_HIT_RATIO_REGEX, fh.read()) + if os.path.exists(cache_debug_log_file): + with open(cache_debug_log_file, "r") as fh: + self.final_cache_hit_ratio = self._parse_float(CACHE_HIT_RATIO_REGEX, fh.read()) + else: + self.final_cache_hit_ratio = 0.0 def make_cedar_report(self) -> List[dict]: """Format the data to look like a cedar report json.""" diff --git a/buildscripts/scons_metrics/report.py b/buildscripts/scons_metrics/report.py index 61b60ccc186..c016492c966 100644 --- a/buildscripts/scons_metrics/report.py +++ b/buildscripts/scons_metrics/report.py @@ -31,10 +31,6 @@ def main(scons_stdout_log_file: str, scons_cache_debug_log_file: str, print(f"Could not find SCons stdout log file '{scons_stdout_log_file}'.") sys.exit(1) - if not os.path.exists(scons_cache_debug_log_file): - print(f"Could not find SCons cache debug log file '{scons_cache_debug_log_file}'.") - sys.exit(1) - scons_metrics = SconsMetrics(scons_stdout_log_file, scons_cache_debug_log_file) if not scons_metrics.raw_report: print( diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py new file mode 100644 index 00000000000..da12326b562 --- /dev/null +++ b/buildscripts/simple_report.py @@ -0,0 +1,105 @@ +"""Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" +import json +import pathlib +import os +from typing import List, Dict, Optional +from typing_extensions import TypedDict +import click + + +class Result(TypedDict, total=False): + """Evergreen test result.""" + + status: str + exit_code: int + test_file: str + start: float + end: float + elapsed: float + log_raw: str + + +class Report(TypedDict): + """Evergreen report.""" + + failures: int + results: List[Result] + + +def _open_and_truncate_log_lines(log_file: pathlib.Path) -> List[str]: + with open(log_file) as fh: + lines = fh.read().splitlines() + for i, line in enumerate(lines): + if line == "scons: done reading SConscript files.": + offset = i + # if possible, also shave off the current and next line + # as they contain: + # scons: done reading SConscript files. + # scons: Building targets ... + # which is superfluous. + if len(lines) > i + 2: + offset = i + 2 + return lines[offset:] + + return lines + + +def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str: + lines = _open_and_truncate_log_lines(log_file) + if dedup_lines: + lines = _dedup_lines(lines) + return os.linesep.join(lines) + + +def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: # noqa: D103 + # pylint: disable=missing-function-docstring + status = "pass" if exit_code == 0 else "fail" + return Report({ + 'failures': + 0 if exit_code == 0 else 1, "results": [ + Result({ + "status": status, "exit_code": exit_code, "test_file": test_name, + "log_raw": log_file_contents + }) + ] + }) + + +def try_combine_reports(out: Report): # noqa: D103 + # pylint: disable=missing-function-docstring + try: + with open("report.json") as fh: + report = json.load(fh) + out["results"] += report["results"] + out["failures"] += report["failures"] + except NameError: + pass + except IOError: + pass + + +def _dedup_lines(lines: List[str]) -> List[str]: + return list(set(lines)) + + +def put_report(out: Report): # noqa: D103 + # pylint: disable=missing-function-docstring + with open("report.json", "w") as fh: + json.dump(out, fh) + + +@click.command() +@click.option("--test-name", required=True, type=str) +@click.option("--log-file", required=True, type=pathlib.Path) +@click.option("--exit-code", required=True, type=int) +@click.option("--dedup-lines", is_flag=True) +def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool): + """Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" + log_file_contents = _clean_log_file(log_file, dedup_lines) + report = make_report(test_name, log_file_contents, exit_code) + try_combine_reports(report) + put_report(report) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/task_generation/suite_split.py b/buildscripts/task_generation/suite_split.py index 5e1e9d32115..a65fcae68f0 100644 --- a/buildscripts/task_generation/suite_split.py +++ b/buildscripts/task_generation/suite_split.py @@ -201,25 +201,15 @@ class SuiteSplitService: if self.config.default_to_fallback: return self.calculate_fallback_suites(params) - try: - evg_stats = HistoricTaskData.from_evg(self.evg_api, self.config.evg_project, - self.config.start_date, self.config.end_date, - params.task_name, params.build_variant) - if not evg_stats: - LOGGER.debug("No test history, using fallback suites") - # This is probably a new suite, since there is no test history, just use the - # fallback values. - return self.calculate_fallback_suites(params) + evg_stats = HistoricTaskData.from_s3(self.config.evg_project, params.task_name, + params.build_variant) + + if evg_stats: return self.calculate_suites_from_evg_stats(evg_stats, params) - except requests.HTTPError as err: - if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE: - # Evergreen may return a 503 when the service is degraded. - # We fall back to splitting the tests into a fixed number of suites. - LOGGER.warning("Received 503 from Evergreen, " - "dividing the tests evenly among suites") - return self.calculate_fallback_suites(params) - else: - raise + + LOGGER.debug("No test history, using fallback suites") + # Since there is no test history this is probably a new suite, just use the fallback values. + return self.calculate_fallback_suites(params) def calculate_fallback_suites(self, params: SuiteSplitParameters) -> GeneratedSuite: """Divide tests into a fixed number of suites.""" diff --git a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp index 834d2052a98..24b0b42fed3 100644 --- a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp +++ b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp @@ -27,4 +27,9 @@ LOGV2_ERROR(25, "more words"); LOGV2_ERROR(26, "words", - "comma, more words words words words words words words words words words words words " + "comma, more words words words words words words words words words words words words "); +iassert(27, "words"); +iasserted(28, "words"); +iassertNoTrace(29, "words"); +iassertedNoTrace(30, "words"); +MONGO_UNREACHABLE_TASSERT(31); diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 82434ee25c4..f3473d32096 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -171,7 +171,7 @@ class TestTimeout(_ResmokeSelftest): ] self.execute_resmoke(resmoke_args) - archival_dirs_to_expect = 4 # 2 tests * 2 nodes + archival_dirs_to_expect = 8 # 2 tests * 2 nodes self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) @@ -189,9 +189,11 @@ class TestTimeout(_ResmokeSelftest): self.execute_resmoke(resmoke_args, sleep_secs=25) - archival_dirs_to_expect = 2 # 2 tests * 2 nodes / 2 data_file directories + archival_dirs_to_expect = 4 self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) - self.assert_dir_file_count(self.test_dir_inner, self.archival_file, archival_dirs_to_expect) + archival_inner_dirs_to_expect = 2 + self.assert_dir_file_count(self.test_dir_inner, self.archival_file, + archival_inner_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) self.assert_dir_file_count(self.test_dir, self.analysis_file, analysis_pids_to_expect) diff --git a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py index 2f685f36c97..274b7e5ffbe 100644 --- a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py +++ b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py @@ -38,6 +38,8 @@ class TestCalculateFcvConstants(TestCase): "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0"], + "eolVersions": + ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -45,15 +47,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - fcv_constants = multiversion_service.calculate_fcv_constants() + version_constants = multiversion_service.calculate_version_constants() - self.assertEqual(fcv_constants.latest, Version("6.0")) - self.assertEqual(fcv_constants.last_continuous, Version("5.3")) - self.assertEqual(fcv_constants.last_lts, Version("5.0")) - self.assertEqual(fcv_constants.requires_fcv_tag_list, + self.assertEqual(version_constants.latest, Version("6.0")) + self.assertEqual(version_constants.last_continuous, Version("5.3")) + self.assertEqual(version_constants.last_lts, Version("5.0")) + self.assertEqual(version_constants.requires_fcv_tag_list, [Version(v) for v in ["5.1", "5.2", "5.3", "6.0"]]) - self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) - self.assertEqual(fcv_constants.fcvs_less_than_latest, [ + self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) + self.assertEqual(version_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3"] ]) @@ -67,6 +69,8 @@ class TestCalculateFcvConstants(TestCase): "6.1", "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0", "6.0"], + "eolVersions": + ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -74,15 +78,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - fcv_constants = multiversion_service.calculate_fcv_constants() + version_constants = multiversion_service.calculate_version_constants() - self.assertEqual(fcv_constants.latest, Version("100.0")) - self.assertEqual(fcv_constants.last_continuous, Version("6.1")) - self.assertEqual(fcv_constants.last_lts, Version("6.0")) - self.assertEqual(fcv_constants.requires_fcv_tag_list, + self.assertEqual(version_constants.latest, Version("100.0")) + self.assertEqual(version_constants.last_continuous, Version("6.1")) + self.assertEqual(version_constants.last_lts, Version("6.0")) + self.assertEqual(version_constants.requires_fcv_tag_list, [Version(v) for v in ["6.1", "100.0"]]) - self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) - self.assertEqual(fcv_constants.fcvs_less_than_latest, [ + self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) + self.assertEqual(version_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1"] ]) diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py index b9fc74f088b..c1ba97c6056 100644 --- a/buildscripts/tests/resmokelib/test_parser.py +++ b/buildscripts/tests/resmokelib/test_parser.py @@ -248,7 +248,6 @@ class TestLocalCommandLine(unittest.TestCase): cmdline = to_local_args([ "run", "--suites=my_suite", - "--reportFailureStatus=fail", "--reportFile=report.json", "--perfReportFile=perf.json", "--storageEngine=my_storage_engine", diff --git a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py b/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py new file mode 100644 index 00000000000..caeb81e1e4b --- /dev/null +++ b/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py @@ -0,0 +1,250 @@ +"""Unit tests for buildscripts/resmokelib/testing/symbolizer_service.py.""" +# pylint: disable=missing-docstring +import os +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import MagicMock + +from buildscripts.resmokelib.testing import symbolizer_service as under_test + + +def mock_resmoke_symbolizer_config(): + config_mock: under_test.ResmokeSymbolizerConfig = MagicMock( + spec_set=under_test.ResmokeSymbolizerConfig) + config_mock.evg_task_id = "evg_task_id" + config_mock.client_id = "client_id" + config_mock.client_secret = "client_secret" + config_mock.is_windows.return_value = False + config_mock.is_macos.return_value = False + return config_mock + + +class TestResmokeSymbolizer(unittest.TestCase): + def setUp(self) -> None: + self.config_mock = mock_resmoke_symbolizer_config() + self.symbolizer_service_mock: under_test.SymbolizerService = MagicMock( + spec_set=under_test.SymbolizerService) + self.file_service_mock: under_test.FileService = MagicMock(spec_set=under_test.FileService) + self.resmoke_symbolizer = under_test.ResmokeSymbolizer( + self.config_mock, self.symbolizer_service_mock, self.file_service_mock) + + def test_symbolize_test_logs_process_all_files(self): + stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] + self.file_service_mock.filter_out_non_files.return_value = stacktrace_files + + self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) + + self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 5) + for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): + self.assertEqual(call.args[0], f"file{i}.stacktrace") + self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) + + def test_symbolize_test_logs_hit_timeout(self): + stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] + self.file_service_mock.filter_out_non_files.return_value = stacktrace_files + + self.resmoke_symbolizer.symbolize_test_logs(MagicMock(), 0) + + self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 1) + for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): + self.assertEqual(call.args[0], f"file{i}.stacktrace") + self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) + + def test_symbolize_test_logs_should_not_symbolize(self): + self.config_mock.is_windows.return_value = True + + self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) + self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() + + def test_symbolize_test_logs_could_not_get_dbpath(self): + self.file_service_mock.check_path_exists.return_value = False + + self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) + self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() + + def test_symbolize_test_logs_did_not_find_files(self): + self.file_service_mock.find_all_children_recursively.return_value = [] + + self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) + self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() + + def test_should_not_symbolize_if_not_in_evergreen(self): + self.config_mock.evg_task_id = None + ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) + self.assertFalse(ret) + + def test_should_not_symbolize_if_secrets_are_absent(self): + self.config_mock.client_id = None + self.config_mock.client_secret = None + ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) + self.assertFalse(ret) + + def test_should_not_symbolize_if_on_windows(self): + self.config_mock.is_windows.return_value = True + ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) + self.assertFalse(ret) + + def test_should_not_symbolize_if_on_macos(self): + self.config_mock.is_macos.return_value = True + ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) + self.assertFalse(ret) + + def test_should_symbolize_return_true(self): + ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) + self.assertTrue(ret) + + def test_get_stacktrace_dir_returns_dir(self): + dbpath = "dbpath" + test = MagicMock(fixture=MagicMock(get_dbpath_prefix=MagicMock(return_value=dbpath))) + self.file_service_mock.check_path_exists.return_value = True + + ret = self.resmoke_symbolizer.get_stacktrace_dir(test) + self.assertEqual(ret, dbpath) + + def test_get_stacktrace_dir_if_dir_does_not_exist(self): + test = MagicMock() + self.file_service_mock.check_path_exists.return_value = False + + ret = self.resmoke_symbolizer.get_stacktrace_dir(test) + self.assertEqual(ret, None) + + def test_get_stacktrace_dir_if_fixture_is_not_available(self): + test = MagicMock(fixture=None) + + ret = self.resmoke_symbolizer.get_stacktrace_dir(test) + self.assertEqual(ret, None) + + +class TestFileService(unittest.TestCase): + def setUp(self) -> None: + self.file_service = under_test.FileService() + self.relative_dir_paths = [ + os.path.join("dir_1"), + os.path.join("dir_2", "dir_2_1"), + os.path.join("dir_2", "dir_2_2"), + os.path.join("dir_3"), + ] + self.relative_file_paths = [ + os.path.join("dir_1", "file_1.stacktrace"), + os.path.join("dir_2", "file_2.stacktrace"), + os.path.join("dir_2", "dir_2_1", "file_3.stacktrace"), + os.path.join("dir_2", "dir_2_2", "file_4.stacktrace"), + ] + + def test_find_all_children_recursively_returns_files(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + for file in abs_file_paths: + Path(file).touch() + + ret = self.file_service.find_all_children_recursively(tmpdir) + self.assertListEqual(sorted(ret), sorted(abs_file_paths)) + + def test_find_all_children_recursively_no_files(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + + ret = self.file_service.find_all_children_recursively(tmpdir) + self.assertListEqual(ret, []) + + def test_find_all_children_recursively_no_dirs(self): + with TemporaryDirectory() as tmpdir: + ret = self.file_service.find_all_children_recursively(tmpdir) + self.assertListEqual(ret, []) + + def test_filter_out_non_files_if_all_files_absent(self): + with TemporaryDirectory() as tmpdir: + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + ret = self.file_service.filter_out_non_files(abs_file_paths) + self.assertListEqual(ret, []) + + def test_filter_out_non_files_if_some_files_present(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + Path(abs_file_paths[1]).touch() + Path(abs_file_paths[3]).touch() + + ret = self.file_service.filter_out_non_files(abs_file_paths) + self.assertListEqual(ret, [abs_file_paths[1], abs_file_paths[3]]) + + def test_filter_out_non_files_if_all_files_present(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + for file in abs_file_paths: + Path(file).touch() + + ret = self.file_service.filter_out_non_files(abs_file_paths) + self.assertListEqual(ret, abs_file_paths) + + def test_remove_empty_files_if_no_empty(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + for file in abs_file_paths: + with open(file, "w") as fstream: + fstream.write("stacktrace") + + self.file_service.remove_empty(abs_file_paths) + for file in abs_file_paths: + self.assertTrue(os.path.exists(file)) + + def test_remove_empty_files_if_partly_empty(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + with open(abs_file_paths[0], "w") as fstream: + fstream.write("stacktrace") + Path(abs_file_paths[1]).touch() + with open(abs_file_paths[2], "w") as fstream: + fstream.write("stacktrace") + Path(abs_file_paths[3]).touch() + + self.file_service.remove_empty(abs_file_paths) + + self.assertTrue(os.path.exists(abs_file_paths[0])) + self.assertFalse(os.path.exists(abs_file_paths[1])) + self.assertTrue(os.path.exists(abs_file_paths[2])) + self.assertFalse(os.path.exists(abs_file_paths[3])) + + def test_remove_empty_files_if_all_empty(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + for file in abs_file_paths: + Path(file).touch() + + self.file_service.remove_empty(abs_file_paths) + for file in abs_file_paths: + self.assertFalse(os.path.exists(file)) + + def test_remove_all_files(self): + with TemporaryDirectory() as tmpdir: + abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] + abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] + for dir_ in abs_dir_paths: + Path(dir_).mkdir(parents=True) + for file in abs_file_paths: + with open(file, "w") as fstream: + fstream.write("stacktrace") + + self.file_service.remove_all(abs_file_paths) + for file in abs_file_paths: + self.assertFalse(os.path.exists(file)) diff --git a/buildscripts/tests/task_generation/test_suite_split.py b/buildscripts/tests/task_generation/test_suite_split.py index 7c54ed489aa..b2b197ed7c5 100644 --- a/buildscripts/tests/task_generation/test_suite_split.py +++ b/buildscripts/tests/task_generation/test_suite_split.py @@ -8,7 +8,7 @@ import requests import buildscripts.task_generation.suite_split as under_test from buildscripts.task_generation.suite_split_strategies import greedy_division, \ round_robin_fallback -from buildscripts.util.teststats import TestRuntime +from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access @@ -31,7 +31,12 @@ def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None): def tst_stat_mock(file, duration, pass_count): - return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) + return HistoricalTestInformation( + test_name=file, + num_pass=pass_count, + num_fail=0, + avg_duration_pass=duration, + ) def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None): @@ -115,15 +120,16 @@ class TestGeneratedSuite(unittest.TestCase): class TestSplitSuite(unittest.TestCase): - def test_calculate_suites(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_calculate_suites(self, get_stats_from_s3_mock): mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats + get_stats_from_s3_mock.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_file for stat in mock_test_stats + stat.test_name for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} @@ -137,32 +143,15 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(10, len(sub_suite.test_list)) - def test_calculate_suites_fallback_on_error(self): - n_tests = 100 - max_sub_suites = 4 - split_config = build_mock_split_config(max_sub_suites=max_sub_suites) - split_params = build_mock_split_params() - - suite_split_service = build_mock_service(split_config=split_config) - mock_evg_error(suite_split_service.evg_api) - suite_split_service.resmoke_proxy.list_tests.return_value = [ - f"test_{i}.js" for i in range(n_tests) - ] - - suite = suite_split_service.split_suite(split_params) - - self.assertEqual(max_sub_suites, len(suite)) - for sub_suite in suite.sub_suites: - self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - - def test_calculate_suites_uses_fallback_on_no_results(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock): n_tests = 100 max_sub_suites = 5 split_config = build_mock_split_config(max_sub_suites=max_sub_suites) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - suite_split_service.evg_api.test_stats_by_project.return_value = [] + get_stats_from_s3_mock.return_value = [] suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -173,7 +162,9 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_calculate_suites_uses_fallback_if_only_results_are_filtered( + self, get_stats_from_s3_mock): n_tests = 100 max_sub_suites = 10 mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] @@ -182,7 +173,7 @@ class TestSplitSuite(unittest.TestCase): split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats + get_stats_from_s3_mock.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -198,31 +189,17 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - def test_calculate_suites_fail_on_unexpected_error(self): - n_tests = 100 - max_sub_suites = 4 - split_config = build_mock_split_config(max_sub_suites=max_sub_suites) - split_params = build_mock_split_params() - - suite_split_service = build_mock_service(split_config=split_config) - mock_evg_error(suite_split_service.evg_api, error_code=requests.codes.INTERNAL_SERVER_ERROR) - suite_split_service.resmoke_proxy.list_tests.return_value = [ - f"test_{i}.js" for i in range(n_tests) - ] - - with self.assertRaises(requests.HTTPError): - suite_split_service.split_suite(split_params) - - def test_calculate_suites_will_filter_specified_tests(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock): mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params( test_filter=lambda t: t in {"test_1.js", "test_2.js"}) suite_split_service = build_mock_service(split_config=split_config) - suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats + get_stats_from_s3_mock.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_file for stat in mock_test_stats + stat.test_name for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py index ec53a02d161..c0e234ea3dd 100644 --- a/buildscripts/tests/test_burn_in_tags.py +++ b/buildscripts/tests/test_burn_in_tags.py @@ -14,8 +14,9 @@ from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns from buildscripts.ciconfig.evergreen import EvergreenProjectConfig import buildscripts.burn_in_tags as under_test +from buildscripts.util.teststats import HistoricalTestInformation -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,too-many-arguments EMPTY_PROJECT = { "buildvariants": [], @@ -55,7 +56,7 @@ def get_evergreen_config() -> EvergreenProjectConfig: class TestCreateEvgBuildVariantMap(unittest.TestCase): def test_create_evg_buildvariant_map(self): expansions_file_data = { - "build_variant": "variant1", "burn_in_tag_buildvariants": "variant2 variant3" + "build_variant": "variant1", "burn_in_tag_include_build_variants": "variant2 variant3" } buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data) @@ -105,7 +106,9 @@ class TestGenerateEvgTasks(unittest.TestCase): self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT) @patch(ns("create_tests_by_task")) - def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_generate_evg_tasks_one_test_changed(self, get_stats_from_s3_mock, + create_tests_by_task_mock): evg_conf_mock = get_evergreen_config() create_tests_by_task_mock.return_value = { "aggregation_mongos_passthrough": TaskInfo( @@ -127,8 +130,13 @@ class TestGenerateEvgTasks(unittest.TestCase): shrub_config = ShrubProject.empty() evergreen_api = MagicMock() repo = MagicMock(working_dir=os.getcwd()) - evergreen_api.test_stats_by_project.return_value = [ - MagicMock(test_file="dir/test2.js", avg_duration_pass=10) + get_stats_from_s3_mock.return_value = [ + HistoricalTestInformation( + test_name="dir/test2.js", + num_pass=1, + num_fail=0, + avg_duration_pass=10, + ) ] under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') @@ -219,8 +227,9 @@ class TestAcceptance(unittest.TestCase): @patch(ns("_create_evg_build_variant_map")) @patch(ns("EvergreenFileChangeDetector")) @patch(burn_in_tests_ns("create_test_membership_map")) + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") def test_tests_generated_if_a_file_changed( - self, create_test_membership_map_mock, find_changed_tests_mock, + self, get_stats_from_s3_mock, create_test_membership_map_mock, find_changed_tests_mock, create_evg_build_variant_map_mock, write_to_file_mock): """ Given a git repository with changes, @@ -236,6 +245,7 @@ class TestAcceptance(unittest.TestCase): 'jstests/slow1/large_role_chain.js', 'jstests/aggregation/accumulators/accumulator_js.js' } + get_stats_from_s3_mock.return_value = [] under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin') diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml index d6a651fcde8..37225817bef 100644 --- a/buildscripts/tests/test_burn_in_tags_evergreen.yml +++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml @@ -64,7 +64,7 @@ buildvariants: display_name: "! Enterprise RHEL 8.0" expansions: multiversion_platform: rhel80 - burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem + burn_in_tag_include_build_variants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem tasks: - name: compile_all_run_unittests_TG distros: diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index c51f8c60ecf..369131eff60 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,12 +4,14 @@ from __future__ import absolute_import import collections import datetime +from io import StringIO import os import sys import subprocess import unittest from mock import Mock, patch, MagicMock +import yaml import buildscripts.burn_in_tests as under_test from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask @@ -556,3 +558,19 @@ class TestLocalFileChangeDetector(unittest.TestCase): self.assertIn(file_list[2], found_tests) self.assertNotIn(file_list[1], found_tests) self.assertEqual(2, len(found_tests)) + + +class TestYamlBurnInExecutor(unittest.TestCase): + @patch('sys.stdout', new_callable=StringIO) + def test_found_tasks_should_be_reported_as_yaml(self, stdout): + n_tasks = 5 + n_tests = 3 + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + + yaml_executor = under_test.YamlBurnInExecutor() + yaml_executor.execute(tests_by_task) + + yaml_raw = stdout.getvalue() + results = yaml.safe_load(yaml_raw) + self.assertEqual(n_tasks, len(results["discovered_tasks"])) + self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) diff --git a/buildscripts/tests/test_debugsymb_mapper.py b/buildscripts/tests/test_debugsymb_mapper.py new file mode 100644 index 00000000000..b1b063f5331 --- /dev/null +++ b/buildscripts/tests/test_debugsymb_mapper.py @@ -0,0 +1,115 @@ +"""Unit tests for debugsymb_mapper.py.""" +# pylint: disable=missing-docstring +import unittest +from unittest.mock import MagicMock + +import buildscripts.debugsymb_mapper as under_test + + +def mock_cmd_client(): + cmd_client = MagicMock(spec_set=under_test.CmdClient) + return cmd_client + + +class TestCmdOutputExtractor(unittest.TestCase): + def setUp(self): + self.cmd_client_mock = mock_cmd_client() + self.cmd_output_extractor = under_test.CmdOutputExtractor(self.cmd_client_mock) + + +class TestGetBuildId(TestCmdOutputExtractor): + def test_get_build_id_returns_build_id(self): + readelf_output = ( + "Displaying notes found in: .note.gnu.build-id\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot\n" + " Owner Data size\tDescription\n" + " GA$<version>3h864 0x00000010\tOPEN\n" + " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)\n" + " GA$<version>3h864 0x00000010\tOPEN\n" + " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)") + self.cmd_client_mock.run.return_value = readelf_output + + build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") + self.assertEqual(build_id_output.build_id, "74c2322104428836f3d94af6cd7471ee7cb5c4ee") + self.assertEqual(build_id_output.cmd_output, readelf_output) + + def test_get_build_id_raises_error(self): + readelf_output = ( + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" + " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" + "\n" + "Displaying notes found in: .gnu.build.attributes.hot") + self.cmd_client_mock.run.return_value = readelf_output + + self.assertRaises(ValueError, self.cmd_output_extractor.get_build_id, "path/to/bin") + + def test_get_build_id_returns_none(self): + readelf_output = ( + "Displaying notes found in: .note.gnu.build-id\n" + " Owner Data size\tDescription\n" + " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)") + self.cmd_client_mock.run.return_value = readelf_output + + build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") + self.assertIsNone(build_id_output.build_id) + self.assertEqual(build_id_output.cmd_output, readelf_output) + + +class TestGetBinVersion(TestCmdOutputExtractor): + def test_get_bin_version_returns_version(self): + # Newer versions command output + version_cmd_output = ('db version v4.4.14-25-gb0475e2\n' + 'Build Info: {\n' + ' "version": "4.4.14-25-gb0475e2",\n' + ' "gitVersion": "b0475e2657c3351b25499971d3340f054ea85b98",\n' + ' "openSSLVersion": "OpenSSL 1.1.1 11 Sep 2018",\n' + ' "modules": [\n' + ' "enterprise"\n' + ' ],\n' + ' "allocator": "tcmalloc",\n' + ' "environment": {\n' + ' "distmod": "ubuntu1804",\n' + ' "distarch": "x86_64",\n' + ' "target_arch": "x86_64"\n' + ' }\n' + '}') + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertEqual(bin_version_output.mongodb_version, "4.4.14-25-gb0475e2") + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) + + def test_get_bin_version_unsupported_output(self): + # Versions prior to 5.0 are not supported + version_cmd_output = ('db version v4.2.20-7-g5a81409\n' + 'git version: 5a81409faf16f30f1189af6367eb3ceee50a02b5\n' + 'OpenSSL version: OpenSSL 1.1.1 11 Sep 2018\n' + 'allocator: tcmalloc\n' + 'modules: enterprise \n' + 'build environment:\n' + ' distmod: ubuntu1804\n' + ' distarch: x86_64\n' + ' target_arch: x86_64') + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertIsNone(bin_version_output.mongodb_version) + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) + + def test_get_bin_version_returns_none(self): + version_cmd_output = "error: unrecognized arguments: --version" + self.cmd_client_mock.run.return_value = version_cmd_output + + bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") + self.assertIsNone(bin_version_output.mongodb_version) + self.assertEqual(bin_version_output.cmd_output, version_cmd_output) diff --git a/buildscripts/tests/test_errorcodes.py b/buildscripts/tests/test_errorcodes.py index 2a9c9ce1e3f..17c102454b5 100644 --- a/buildscripts/tests/test_errorcodes.py +++ b/buildscripts/tests/test_errorcodes.py @@ -26,7 +26,7 @@ class TestErrorcodes(unittest.TestCase): captured_error_codes.append(code) errorcodes.parse_source_files(accumulate_files, TESTDATA_DIR + 'regex_matching/') - self.assertEqual(26, len(captured_error_codes)) + self.assertEqual(31, len(captured_error_codes)) def test_dup_checking(self): """Test dup checking.""" diff --git a/buildscripts/tests/test_evergreen_activate_gen_tasks.py b/buildscripts/tests/test_evergreen_activate_gen_tasks.py index dbb64380838..5470828c6c3 100644 --- a/buildscripts/tests/test_evergreen_activate_gen_tasks.py +++ b/buildscripts/tests/test_evergreen_activate_gen_tasks.py @@ -1,42 +1,126 @@ """Unit tests for the generate_resmoke_suite script.""" +# pylint: disable=invalid-name import unittest -from mock import MagicMock +from mock import MagicMock, mock from buildscripts import evergreen_activate_gen_tasks as under_test +from evergreen import Build, EvergreenApi, Task, Version # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access # pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter -def build_mock_task(name, task_id): - mock_task = MagicMock(display_name=name, task_id=task_id) +def build_mock_task(display_name, task_id): + mock_task = MagicMock(spec_set=Task, display_name=display_name, task_id=task_id) return mock_task -def build_mock_evg_api(mock_task_list): - mock_build = MagicMock() - mock_build.get_tasks.return_value = mock_task_list - mock_evg_api = MagicMock() - mock_evg_api.build_by_id.return_value = mock_build +def build_mock_task_list(num_tasks): + return [build_mock_task(f"task_{i}", f"id_{i}") for i in range(num_tasks)] + + +class MockVariantData(): + """An object to help create a mock evg api.""" + + def __init__(self, build_id, variant_name, task_list): + self.build_id = build_id + self.variant_name = variant_name + self.task_list = task_list + + +def build_mock_evg_api(variant_data_list): + class VersionPatchedSpec(Version): + """A patched `Version` with instance properties included for magic mock spec.""" + build_variants_map = MagicMock() + + mock_version = MagicMock(spec_set=VersionPatchedSpec) + mock_version.build_variants_map = { + variant_data.variant_name: variant_data.build_id + for variant_data in variant_data_list + } + + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.version_by_id.return_value = mock_version + + build_id_mapping = { + variant_data.build_id: variant_data.task_list + for variant_data in variant_data_list + } + + def tasks_by_build_side_effect(build_id): + return build_id_mapping[build_id] + + mock_evg_api.tasks_by_build.side_effect = tasks_by_build_side_effect return mock_evg_api class TestActivateTask(unittest.TestCase): def test_task_with_display_name_is_activated(self): - n_tasks = 5 - mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] - mock_evg_api = build_mock_evg_api(mock_task_list) + expansions = under_test.EvgExpansions(**{ + "build_id": "build_id", + "version_id": "version_id", + "task_name": "task_3_gen", + }) + mock_task_list = build_mock_task_list(5) + mock_evg_api = build_mock_evg_api( + [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) - under_test.activate_task("build_id", "task_3", mock_evg_api) + under_test.activate_task(expansions, mock_evg_api) mock_evg_api.configure_task.assert_called_with("id_3", activated=True) def test_task_with_no_matching_name(self): - n_tasks = 5 - mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] - mock_evg_api = build_mock_evg_api(mock_task_list) + expansions = under_test.EvgExpansions(**{ + "build_id": "build_id", + "version_id": "version_id", + "task_name": "not_an_existing_task", + }) + mock_task_list = build_mock_task_list(5) + mock_evg_api = build_mock_evg_api( + [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) - under_test.activate_task("build_id", "not_an_existing_task", mock_evg_api) + under_test.activate_task(expansions, mock_evg_api) mock_evg_api.configure_task.assert_not_called() + + def test_burn_in_tags_tasks_are_activated(self): + expansions = under_test.EvgExpansions(**{ + "build_id": "build_id", + "version_id": "version_id", + "task_name": "burn_in_tags_gen", + }) + mock_task_list_2 = build_mock_task_list(5) + mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) + mock_task_list_3 = build_mock_task_list(5) + mock_task_list_3.append(build_mock_task("burn_in_tests", "burn_in_tests_id_3")) + mock_evg_api = build_mock_evg_api([ + MockVariantData("1", "variant1-generated-by-burn-in-tags", mock_task_list_2), + MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_3) + ]) + + under_test.activate_task(expansions, mock_evg_api) + + mock_evg_api.configure_task.assert_has_calls([ + mock.call("burn_in_tests_id_2", activated=True), + mock.call("burn_in_tests_id_3", activated=True) + ]) + + def test_burn_in_tags_task_skips_non_existing_build_variant(self): + expansions = under_test.EvgExpansions(**{ + "build_id": "build_id", + "version_id": "version_id", + "task_name": "burn_in_tags_gen", + }) + mock_task_list_1 = build_mock_task_list(5) + mock_task_list_1.append(build_mock_task("burn_in_tags_gen", "burn_in_tags_gen_id_1")) + mock_task_list_2 = build_mock_task_list(5) + mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) + mock_evg_api = build_mock_evg_api([ + MockVariantData("1", "variant1-non-burn-in", mock_task_list_1), + MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_2) + ]) + + under_test.activate_task(expansions, mock_evg_api) + + mock_evg_api.configure_task.assert_called_once_with("burn_in_tests_id_2", activated=True) diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py index ee77ced2579..3ea78039930 100644 --- a/buildscripts/tests/test_evergreen_burn_in_tests.py +++ b/buildscripts/tests/test_evergreen_burn_in_tests.py @@ -89,7 +89,8 @@ class TestAcceptance(unittest.TestCase): @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch(ns("write_file")) - def test_tests_generated_if_a_file_changed(self, write_json_mock): + @patch(ns("HistoricTaskData.get_stats_from_s3")) + def test_tests_generated_if_a_file_changed(self, get_stats_from_s3_mock, write_json_mock): """ Given a git repository with changes, When burn_in_tests is run, @@ -108,6 +109,7 @@ class TestAcceptance(unittest.TestCase): ) # yapf: disable mock_evg_conf = get_evergreen_config("etc/evergreen.yml") mock_evg_api = MagicMock() + get_stats_from_s3_mock.return_value = [] under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, mock_evg_conf, repos, "testfile.json", 'install-dir/bin') @@ -241,41 +243,30 @@ class TestGenerateTimeouts(unittest.TestCase): class TestGetTaskRuntimeHistory(unittest.TestCase): - def test_get_task_runtime_history(self): - mock_evg_api = MagicMock() - mock_evg_api.test_stats_by_project.return_value = [ - MagicMock( - test_file="dir/test2.js", - task_name="task1", - variant="variant1", - distro="distro1", - date=datetime.utcnow().date(), + @patch(ns("HistoricTaskData.get_stats_from_s3")) + def test_get_task_runtime_history(self, get_stats_from_s3_mock): + test_stats = [ + teststats_utils.HistoricalTestInformation( + test_name="dir/test2.js", num_pass=1, num_fail=0, avg_duration_pass=10.1, ) ] - analysis_duration = under_test.AVG_TEST_RUNTIME_ANALYSIS_DAYS - end_date = datetime.utcnow().replace(microsecond=0) - start_date = end_date - timedelta(days=analysis_duration) + get_stats_from_s3_mock.return_value = test_stats mock_gen_config = MagicMock(project="project1", build_variant="variant1") - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api, - history_end_date=end_date) + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) result = executor.get_task_runtime_history("task1") self.assertEqual(result, [("dir/test2.js", 10.1)]) - mock_evg_api.test_stats_by_project.assert_called_with( - "project1", after_date=start_date, before_date=end_date, group_by="test", - group_num_days=14, tasks=["task1"], variants=["variant1"]) - def test_get_task_runtime_history_evg_degraded_mode_error(self): - mock_response = MagicMock(status_code=requests.codes.SERVICE_UNAVAILABLE) - mock_evg_api = MagicMock() - mock_evg_api.test_stats_by_project.side_effect = requests.HTTPError(response=mock_response) + @patch(ns("HistoricTaskData.get_stats_from_s3")) + def test_get_task_runtime_history_when_s3_has_no_data(self, get_stats_from_s3_mock): + get_stats_from_s3_mock.return_value = [] mock_gen_config = MagicMock(project="project1", build_variant="variant1") - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api) + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) result = executor.get_task_runtime_history("task1") self.assertEqual(result, []) @@ -321,7 +312,8 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): self.assertEqual(0, len(evg_config_dict["tasks"])) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_one_task_one_test(self): + @patch(ns("HistoricTaskData.get_stats_from_s3")) + def test_one_task_one_test(self, get_stats_from_s3_mock): n_tasks = 1 n_tests = 1 resmoke_options = "options for resmoke" @@ -329,10 +321,10 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): gen_config = MagicMock(run_build_variant="variant", distro=None) repeat_config = MagicMock() repeat_config.generate_resmoke_options.return_value = resmoke_options - mock_evg_api = MagicMock() tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + get_stats_from_s3_mock.return_value = [] - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) executor.generate_tasks_for_variant(tests_by_task, build_variant) shrub_config = ShrubProject.empty().add_build_variant(build_variant) @@ -345,16 +337,17 @@ class TestCreateGenerateTasksConfig(unittest.TestCase): self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"]) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_n_task_m_test(self): + @patch(ns("HistoricTaskData.get_stats_from_s3")) + def test_n_task_m_test(self, get_stats_from_s3_mock): n_tasks = 3 n_tests = 5 build_variant = BuildVariant("build variant") gen_config = MagicMock(run_build_variant="variant", distro=None) repeat_config = MagicMock() tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - mock_evg_api = MagicMock() + get_stats_from_s3_mock.return_value = [] - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) executor.generate_tasks_for_variant(tests_by_task, build_variant) evg_config_dict = build_variant.as_dict() @@ -369,14 +362,12 @@ class TestCreateGenerateTasksFile(unittest.TestCase): gen_config = MagicMock(require_multiversion_setup=False) repeat_config = MagicMock() tests_by_task = MagicMock() - mock_evg_api = MagicMock() validate_mock.return_value = False exit_mock.side_effect = ValueError("exiting") with self.assertRaises(ValueError): - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api, - "gen_file.json") + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, "gen_file.json") executor.execute(tests_by_task) exit_mock.assert_called_once() diff --git a/buildscripts/tests/test_evergreen_task_timeout.py b/buildscripts/tests/test_evergreen_task_timeout.py index b3e4e201d96..294c50d0a17 100644 --- a/buildscripts/tests/test_evergreen_task_timeout.py +++ b/buildscripts/tests/test_evergreen_task_timeout.py @@ -110,7 +110,7 @@ class TestTimeoutOverrides(unittest.TestCase): class TestDetermineExecTimeout(unittest.TestCase): def _validate_exec_timeout(self, idle_timeout, exec_timeout, historic_timeout, evg_alias, - build_variant, timeout_override, expected_timeout): + build_variant, display_name, timeout_override, expected_timeout): task_name = "task_name" variant = build_variant overrides = {} @@ -121,8 +121,9 @@ class TestDetermineExecTimeout(unittest.TestCase): orchestrator = under_test.TaskTimeoutOrchestrator( timeout_service=MagicMock(spec_set=TimeoutService), - timeout_overrides=mock_timeout_overrides, - evg_project_config=MagicMock(spec_set=EvergreenProjectConfig)) + timeout_overrides=mock_timeout_overrides, evg_project_config=MagicMock( + spec_set=EvergreenProjectConfig, + get_variant=MagicMock(return_value=MagicMock(display_name=display_name)))) actual_timeout = orchestrator.determine_exec_timeout( task_name, variant, idle_timeout, exec_timeout, evg_alias, historic_timeout) @@ -132,78 +133,83 @@ class TestDetermineExecTimeout(unittest.TestCase): def test_timeout_used_if_specified(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=42), historic_timeout=None, evg_alias=None, build_variant="variant", - timeout_override=None, expected_timeout=timedelta(seconds=42)) + display_name="not required", timeout_override=None, + expected_timeout=timedelta(seconds=42)) def test_default_is_returned_with_no_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, - evg_alias=None, build_variant="variant", timeout_override=None, + evg_alias=None, build_variant="variant", + display_name="not required", timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_is_returned_with_timeout_at_zero(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - timeout_override=None, + display_name="not required", timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_required_returned_on_required_variants(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - timeout_override=None, + display_name="! required", timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) def test_override_on_required_should_use_override(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - timeout_override=3 * 60, + display_name="! required", timeout_override=3 * 60, expected_timeout=timedelta(minutes=3 * 60)) def test_task_specific_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - timeout_override=60, expected_timeout=timedelta(minutes=60)) + display_name="not required", timeout_override=60, + expected_timeout=timedelta(minutes=60)) def test_commit_queue_items_use_commit_queue_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=under_test.COMMIT_QUEUE_ALIAS, - build_variant="variant", timeout_override=None, + build_variant="variant", display_name="not required", + timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_use_idle_timeout_if_greater_than_exec_timeout(self): self._validate_exec_timeout( idle_timeout=timedelta(hours=2), exec_timeout=timedelta(minutes=10), - historic_timeout=None, evg_alias=None, build_variant="variant", timeout_override=None, - expected_timeout=timedelta(hours=2)) + historic_timeout=None, evg_alias=None, build_variant="variant", + display_name="not required", timeout_override=None, expected_timeout=timedelta(hours=2)) def test_historic_timeout_should_be_used_if_given(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", timeout_override=None, - expected_timeout=timedelta(minutes=15)) + build_variant="variant", display_name="not required", + timeout_override=None, expected_timeout=timedelta(minutes=15)) def test_commit_queue_should_override_historic_timeouts(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), - evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", timeout_override=None, + evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", + display_name="not required", timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_override_should_override_historic_timeouts(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", timeout_override=33, - expected_timeout=timedelta(minutes=33)) + build_variant="variant", display_name="not required", + timeout_override=33, expected_timeout=timedelta(minutes=33)) def test_historic_timeout_should_not_be_overridden_by_required_bv(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant-required", timeout_override=None, - expected_timeout=timedelta(minutes=15)) + build_variant="variant-required", display_name="! required", + timeout_override=None, expected_timeout=timedelta(minutes=15)) def test_historic_timeout_should_not_be_increase_required_bv_timeout(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT + timedelta(minutes=30), - evg_alias=None, build_variant="variant-required", timeout_override=None, - expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) + evg_alias=None, build_variant="variant-required", display_name="! required", + timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) class TestDetermineIdleTimeout(unittest.TestCase): diff --git a/buildscripts/tests/test_jepsen_report.py b/buildscripts/tests/test_jepsen_report.py new file mode 100644 index 00000000000..cdfb095ba5a --- /dev/null +++ b/buildscripts/tests/test_jepsen_report.py @@ -0,0 +1,195 @@ +"""Tests for jepsen report generator.""" +import unittest +import textwrap +import random +import os +from unittest.mock import patch, mock_open, MagicMock +from click.testing import CliRunner + +from buildscripts.jepsen_report import parse, ParserOutput, main + +_CORPUS = textwrap.dedent("""\ + "indeterminate: Command failed with error 251 (NoSuchTransaction): 'Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1' on server n9:27017. The full response is {\"writeConcernError\": {\"code\": 6, \"codeName\": \"HostUnreachable\", \"errmsg\": \"operation was interrupted\", \"errInfo\": {\"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"clientSupplied\"}}}, \"topologyVersion\": {\"processId\": {\"$oid\": \"625f0b0ef9d6a12d9b562ff9\"}, \"counter\": 21}, \"ok\": 0.0, \"errmsg\": \"Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1\", \"code\": 251, \"codeName\": \"NoSuchTransaction\", \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 16}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"operationTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 5}}, \"recoveryToken\": {\"recoveryShardId\": \"rs_shard1\"}}", + :index 1141}})}, + :workload {:valid? true}, + :valid? true} + + +Everything looks good! ヽ(‘ー`)ノ + + + +# Successful tests + +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T163539.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164131.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164724.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165317.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165910.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T170503.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171055.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171648.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172239.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172832.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T173634.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174227.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174820.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T175413.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180006.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180558.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181148.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181946.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T182539.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183131.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183720.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184313.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184906.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T185459.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190052.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190645.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191239.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191833.000Z + +# Crashed tests + +mongodb list-append w:majority r:majority tw:majority tr:snapshot partition +mongodb list-append w:majority r:majority tw:majority tr:snapshot partition + +28 successes +0 unknown +2 crashed +0 failures +""").splitlines() + + +class TestParser(unittest.TestCase): + """TestParser.""" + + @classmethod + def _corpus_generator(cls): + n_pass = random.randint(1, 30) + n_fail = random.randint(1, 30) + n_unknown = random.randint(1, 30) + n_crash = random.randint(1, 30) + + corpus = textwrap.dedent("""\ + :index 2893}})}, + :workload {:valid? true}, + :valid? true} +Everything looks good! ヽ(‘ー`)ノ +""") + successful_tests = [] + if n_pass > 0: + corpus += "\n# Successful tests\n\n" + for i in range(0, n_pass): + corpus += f"test {i}\n" + successful_tests.append(f"test {i}") + + failed_tests = [] + if n_fail > 0: + corpus += "\n# Failed tests\n\n" + for i in range(0, n_fail): + corpus += f"test {i}\n" + failed_tests.append(f"test {i}") + + indeterminate_tests = [] + if n_unknown > 0: + corpus += "\n# Indeterminate tests\n\n" + for i in range(0, n_unknown): + corpus += f"test {i}\n" + indeterminate_tests.append(f"test {i}") + + crashed_tests = [] + if n_crash > 0: + corpus += "\n# Crashed tests\n\n" + for i in range(0, n_crash): + corpus += f"test {i}\n" + crashed_tests.append(f"test {i}") + # note leading newline for this block is required to match the actual + # logs + corpus += textwrap.dedent(f""" +{n_pass} successes +{n_unknown} unknown +{n_crash} crashed +{n_fail} failures +""") + + return { + 'expected': + ParserOutput({ + 'success': successful_tests, + 'unknown': indeterminate_tests, + 'crashed': crashed_tests, + 'failed': failed_tests, + }), 'corpus': + corpus + } + + def test_parser(self): + """Test with embedded corpus.""" + out = parse(_CORPUS) + self.assertEqual(len(out['success']), 28) + self.assertEqual(len(out['unknown']), 0) + self.assertEqual(len(out['crashed']), 2) + self.assertEqual(len(out['failed']), 0) + + def test_parser2(self): + """Test with jepsen.log file.""" + with open(os.path.join(os.path.dirname(__file__), + "test_jepsen_report_corpus.log.txt")) as fh: + corpus = fh.read().splitlines() + out = parse(corpus) + self.assertEqual(len(out['success']), 29) + self.assertEqual(len(out['unknown']), 0) + self.assertEqual(len(out['crashed']), 1) + self.assertEqual(len(out['failed']), 0) + + def test_generated_corpus(self): + """Generate 100 corpuses and test them.""" + for _ in range(0, 100): + self._test_generated_corpus() + + def _test_generated_corpus(self): + gen = self._corpus_generator() + corpus = gen['corpus'].splitlines() + out = parse(corpus) + self.assertDictEqual(out, gen['expected']) + + @patch('buildscripts.jepsen_report._try_find_log_file') + @patch('buildscripts.jepsen_report._get_log_lines') + @patch('buildscripts.jepsen_report._put_report') + def test_main(self, mock_put_report, mock_get_log_lines, mock_try_find_log_file): + """Test main function.""" + gen = self._corpus_generator() + corpus = gen['corpus'].splitlines() + mock_get_log_lines.return_value = corpus + + def _try_find_log_file(_store, _test): + if _try_find_log_file.counter == 0: + _try_find_log_file.counter += 1 + with open( + os.path.join( + os.path.dirname(__file__), "test_jepsen_report_corpus.log.txt")) as fh: + return fh.read() + return "" + + _try_find_log_file.counter = 0 + mock_try_find_log_file.side_effect = _try_find_log_file + + runner = CliRunner() + result = runner.invoke(main, + ["--start_time=0", "--end_time=10", "--elapsed=10", "test.log"]) + num_tests = len(gen['expected']['success']) + len(gen['expected']['unknown']) + len( + gen['expected']['crashed']) + len(gen['expected']['failed']) + num_fails = num_tests - len(gen['expected']['success']) + + callee_dict = mock_put_report.call_args[0][0] + self.assertEqual(callee_dict['failures'], num_fails) + self.assertEqual(len(callee_dict['results']), num_tests) + mock_get_log_lines.assert_called_once_with('test.log') + if gen['expected']['crashed']: + self.assertEqual(result.exit_code, 2) + elif gen['expected']['unknown'] or gen['expected']['failure']: + self.assertEqual(result.exit_code, 1) + else: + self.assertEqual(result.exit_code, 0) diff --git a/buildscripts/tests/test_jepsen_report_corpus.log.txt b/buildscripts/tests/test_jepsen_report_corpus.log.txt new file mode 100644 index 00000000000..89d4e91892b --- /dev/null +++ b/buildscripts/tests/test_jepsen_report_corpus.log.txt @@ -0,0 +1,67 @@ +INFO [2022-04-20 02:46:34,505] jepsen test runner - jepsen.core {:perf + {:latency-graph {:valid? true}, + :rate-graph {:valid? true}, + :valid? true}, + :clock {:valid? true}, + :stats + {:valid? true, + :count 5768, + :ok-count 4930, + :fail-count 0, + :info-count 838, + :by-f + {:txn + {:valid? true, + :count 5768, + :ok-count 4930, + :fail-count 0, + :info-count 838}}}, + :exceptions {:valid? true}, + :workload {:valid? true}, + :valid? true} + + +Everything looks good! ヽ(‘ー`)ノ + + + +# Successful tests + +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235350.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235940.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T000533.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001131.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001724.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002316.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002909.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T003502.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004057.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004651.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005244.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005837.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T010429.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011021.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011617.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012210.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012802.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T013356.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014154.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014746.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015338.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015931.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T020525.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021116.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021708.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022301.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022854.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T023446.000Z +store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T024039.000Z + +# Crashed tests + +mongodb list-append w:majority r:majority tw:majority tr:snapshot partition + +29 successes +0 unknown +1 crashed +0 failures diff --git a/buildscripts/tests/test_mongosymb.py b/buildscripts/tests/test_mongosymb.py new file mode 100644 index 00000000000..188ea385c5f --- /dev/null +++ b/buildscripts/tests/test_mongosymb.py @@ -0,0 +1,68 @@ +"""Unit tests for buildscripts/mongosymb.py.""" +# pylint: disable=missing-docstring +import unittest + +from buildscripts import mongosymb as under_test + + +class TestGetVersion(unittest.TestCase): + def test_get_version_with_patch(self): + trace_doc = { + "processInfo": { + "mongodbVersion": "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169" + } + } + version = under_test.get_version(trace_doc) + self.assertEqual(version, "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169") + + def test_get_version_without_patch(self): + trace_doc = {"processInfo": {"mongodbVersion": "6.1.0-alpha-504-g0c8a142"}} + version = under_test.get_version(trace_doc) + self.assertEqual(version, "6.1.0-alpha-504-g0c8a142") + + def test_get_version_no_mongodb_version(self): + trace_doc = {"processInfo": {}} + version = under_test.get_version(trace_doc) + self.assertEqual(version, None) + + def test_get_version_no_process_info(self): + trace_doc = {} + version = under_test.get_version(trace_doc) + self.assertEqual(version, None) + + +class TestHasHighNotFoundPathsRatio(unittest.TestCase): + def test_not_found_paths_ratio_is_more_than_0_5(self): + frames = [ + {"path": "some/path"}, + {"path": "some/path"}, + {"path": "some/path"}, + {"path": None}, + ] + ret = under_test.has_high_not_found_paths_ratio(frames) + self.assertEqual(ret, False) + + def test_not_found_paths_ratio_is_equal_to_0_5(self): + frames = [ + {"path": "some/path"}, + {"path": "some/path"}, + {"path": None}, + {"path": None}, + ] + ret = under_test.has_high_not_found_paths_ratio(frames) + self.assertEqual(ret, True) + + def test_not_found_paths_ratio_is_less_than_0_5(self): + frames = [ + {"path": "some/path"}, + {"path": None}, + {"path": None}, + {"path": None}, + ] + ret = under_test.has_high_not_found_paths_ratio(frames) + self.assertEqual(ret, True) + + def test_no_frames(self): + frames = [] + ret = under_test.has_high_not_found_paths_ratio(frames) + self.assertEqual(ret, False) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 70689345b69..6a407983a9f 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -24,7 +24,7 @@ from buildscripts.task_generation.task_types.gentask_options import GenTaskOptio from buildscripts.tests.test_burn_in_tests import get_evergreen_config, mock_changed_git_files from buildscripts import selected_tests as under_test -# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter +# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter,too-many-locals NS = "buildscripts.selected_tests" @@ -101,7 +101,9 @@ class TestAcceptance(unittest.TestCase): self.assertEqual(generated_config.file_list[0].file_name, "selected_tests_config.json") @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_when_test_mappings_are_found_for_changed_files(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_when_test_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): + get_stats_from_s3_mock.return_value = [] mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( @@ -149,7 +151,9 @@ class TestAcceptance(unittest.TestCase): self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_when_task_mappings_are_found_for_changed_files(self): + @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") + def test_when_task_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): + get_stats_from_s3_mock.return_value = [] mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py index bb0dd8a0c3e..bb0550659c6 100644 --- a/buildscripts/tests/timeouts/test_timeout_service.py +++ b/buildscripts/tests/timeouts/test_timeout_service.py @@ -1,41 +1,42 @@ """Unit tests for timeout_service.py.""" import random import unittest -from datetime import datetime, timedelta -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch from requests.exceptions import HTTPError -from evergreen import EvergreenApi import buildscripts.timeouts.timeout_service as under_test from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.util.teststats import HistoricTaskData +from buildscripts.util.teststats import HistoricTaskData, HistoricTestInfo # pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access +NS = "buildscripts.timeouts.timeout_service" -def build_mock_service(evg_api=None, resmoke_proxy=None): - end_date = datetime.now() - start_date = end_date - timedelta(weeks=2) - timeout_settings = under_test.TimeoutSettings( - end_date=end_date, - start_date=start_date, - ) + +def ns(relative_name): # pylint: disable=invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + + +def build_mock_service(resmoke_proxy=None): return under_test.TimeoutService( - evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi), - resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService), - timeout_settings=timeout_settings) + resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService)) def tst_stat_mock(file, duration, pass_count): - return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) + return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count, hooks=[]) + + +def tst_runtime_mock(file, duration, pass_count): + return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count) class TestGetTimeoutEstimate(unittest.TestCase): - def test_no_stats_should_return_default_timeout(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_evg_api.test_stats_by_project.return_value = [] - timeout_service = build_mock_service(evg_api=mock_evg_api) + @patch(ns("HistoricTaskData.from_s3")) + def test_no_stats_should_return_default_timeout(self, from_s3_mock: MagicMock): + timeout_service = build_mock_service() + from_s3_mock.return_value = [] timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -48,13 +49,17 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - def test_a_test_with_missing_history_should_cause_a_default_timeout(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] - mock_evg_api.test_stats_by_project.return_value = test_stats + @patch(ns("HistoricTaskData.from_s3")) + def test_a_test_with_missing_history_should_cause_a_default_timeout( + self, from_s3_mock: MagicMock): + test_stats = [ + HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=60, num_pass=1, hooks=[]) + for i in range(30) + ] + from_s3_mock.return_value = HistoricTaskData(test_stats) mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"] - timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) + timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -67,14 +72,19 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] - test_stats.append(tst_stat_mock("zero.js", 0.0, 1)) - mock_evg_api.test_stats_by_project.return_value = test_stats + @patch(ns("HistoricTaskData.from_s3")) + def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout( + self, from_s3_mock: MagicMock): + test_stats = [ + HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=60, num_pass=1, hooks=[]) + for i in range(30) + ] + test_stats.append( + HistoricTestInfo(test_name="zero.js", avg_duration=0.0, num_pass=1, hooks=[])) + from_s3_mock.return_value = HistoricTaskData(test_stats) mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] - timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] + timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -87,15 +97,19 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - def test_all_tests_with_runtime_history_should_use_custom_timeout(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) + @patch(ns("HistoricTaskData.from_s3")) + def test_all_tests_with_runtime_history_should_use_custom_timeout(self, + from_s3_mock: MagicMock): n_tests = 30 test_runtime = 600 - test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)] - mock_evg_api.test_stats_by_project.return_value = test_stats + test_stats = [ + HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=test_runtime, num_pass=1, + hooks=[]) for i in range(n_tests) + ] + from_s3_mock.return_value = HistoricTaskData(test_stats) mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] - timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] + timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -149,10 +163,10 @@ class TestGetTaskHookOverhead(unittest.TestCase): class TestLookupHistoricStats(unittest.TestCase): - def test_no_stats_from_evergreen_should_return_none(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_evg_api.test_stats_by_project.return_value = [] - timeout_service = build_mock_service(evg_api=mock_evg_api) + @patch(ns("HistoricTaskData.from_s3")) + def test_no_stats_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): + from_s3_mock.return_value = None + timeout_service = build_mock_service() timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -165,10 +179,10 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - def test_errors_from_evergreen_should_return_none(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect") - timeout_service = build_mock_service(evg_api=mock_evg_api) + @patch(ns("HistoricTaskData.from_s3")) + def test_errors_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): + from_s3_mock.side_effect = HTTPError("failed to connect") + timeout_service = build_mock_service() timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -181,11 +195,11 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - def test_stats_from_evergreen_should_return_the_stats(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) + @patch(ns("HistoricTaskData.from_s3")) + def test_stats_from_evergreen_should_return_the_stats(self, from_s3_mock: MagicMock): test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] - mock_evg_api.test_stats_by_project.return_value = test_stats - timeout_service = build_mock_service(evg_api=mock_evg_api) + from_s3_mock.return_value = HistoricTaskData(test_stats) + timeout_service = build_mock_service() timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", diff --git a/buildscripts/tests/util/test_teststats.py b/buildscripts/tests/util/test_teststats.py index ebba930d032..9e72d87a737 100644 --- a/buildscripts/tests/util/test_teststats.py +++ b/buildscripts/tests/util/test_teststats.py @@ -2,8 +2,12 @@ import datetime import unittest +from json import JSONDecodeError +from unittest.mock import patch from mock import Mock +from mock.mock import MagicMock +from requests import Session import buildscripts.util.teststats as under_test @@ -21,6 +25,60 @@ class NormalizeTestNameTest(unittest.TestCase): under_test.normalize_test_name("\\home\\user\\test.js")) +class TestHistoricTestInfo(unittest.TestCase): + def test_total_test_runtime_not_passing_test_no_hooks(self): + test_info = under_test.HistoricTestInfo( + test_name='jstests/test.js', + num_pass=0, + avg_duration=0.0, + hooks=[], + ) + + self.assertEqual(0.0, test_info.total_test_runtime()) + + def test_total_test_runtime_not_passing_test_with_hooks(self): + test_info = under_test.HistoricTestInfo( + test_name='jstests/test.js', + num_pass=0, + avg_duration=0.0, + hooks=[ + under_test.HistoricHookInfo( + hook_id='test:hook', + num_pass=10, + avg_duration=5.0, + ), + ], + ) + + self.assertEqual(0.0, test_info.total_test_runtime()) + + def test_total_test_runtime_passing_test_no_hooks(self): + test_info = under_test.HistoricTestInfo( + test_name='jstests/test.js', + num_pass=10, + avg_duration=23.0, + hooks=[], + ) + + self.assertEqual(23.0, test_info.total_test_runtime()) + + def test_total_test_runtime_passing_test_with_hooks(self): + test_info = under_test.HistoricTestInfo( + test_name='jstests/test.js', + num_pass=10, + avg_duration=23.0, + hooks=[ + under_test.HistoricHookInfo( + hook_id='test:hook', + num_pass=10, + avg_duration=5.0, + ), + ], + ) + + self.assertEqual(28.0, test_info.total_test_runtime()) + + class TestHistoricTaskData(unittest.TestCase): def test_no_hooks(self): evg_results = [ @@ -80,7 +138,7 @@ class TestHistoricTaskData(unittest.TestCase): @staticmethod def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0): return Mock( - test_file=test_file, + test_name=test_file, task_name="task1", variant="variant1", distro="distro1", @@ -89,3 +147,53 @@ class TestHistoricTaskData(unittest.TestCase): num_fail=0, avg_duration_pass=duration, ) + + @patch.object(Session, 'get') + def test_get_stats_from_s3_returns_data(self, mock_get): + mock_response = MagicMock() + mock_response.json.return_value = [ + { + "test_name": "jstests/noPassthroughWithMongod/geo_near_random1.js", + "num_pass": 74, + "num_fail": 0, + "avg_duration_pass": 23.16216216216216, + "max_duration_pass": 27.123, + }, + { + "test_name": "shell_advance_cluster_time:ValidateCollections", + "num_pass": 74, + "num_fail": 0, + "avg_duration_pass": 1.662162162162162, + "max_duration_pass": 100.0987, + }, + ] + mock_get.return_value = mock_response + + result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") + + self.assertEqual(result, [ + under_test.HistoricalTestInformation( + test_name="jstests/noPassthroughWithMongod/geo_near_random1.js", + num_pass=74, + num_fail=0, + avg_duration_pass=23.16216216216216, + max_duration_pass=27.123, + ), + under_test.HistoricalTestInformation( + test_name="shell_advance_cluster_time:ValidateCollections", + num_pass=74, + num_fail=0, + avg_duration_pass=1.662162162162162, + max_duration_pass=100.0987, + ), + ]) + + @patch.object(Session, 'get') + def test_get_stats_from_s3_json_decode_error(self, mock_get): + mock_response = MagicMock() + mock_response.json.side_effect = JSONDecodeError("msg", "doc", 0) + mock_get.return_value = mock_response + + result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") + + self.assertEqual(result, []) diff --git a/buildscripts/timeouts/timeout_service.py b/buildscripts/timeouts/timeout_service.py index 8c0d5ad58cd..3df4ecddb74 100644 --- a/buildscripts/timeouts/timeout_service.py +++ b/buildscripts/timeouts/timeout_service.py @@ -1,13 +1,11 @@ """Service for determining task timeouts.""" -from datetime import datetime from typing import Any, Dict, NamedTuple, Optional import inject import structlog from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.timeouts.timeout import TimeoutEstimate -from buildscripts.util.teststats import HistoricTaskData -from evergreen import EvergreenApi +from buildscripts.util.teststats import HistoricTaskData, normalize_test_name LOGGER = structlog.get_logger(__name__) CLEAN_EVERY_N_HOOK = "CleanEveryN" @@ -31,29 +29,17 @@ class TimeoutParams(NamedTuple): is_asan: bool -class TimeoutSettings(NamedTuple): - """Settings for determining timeouts.""" - - start_date: datetime - end_date: datetime - - class TimeoutService: """A service for determining task timeouts.""" @inject.autoparams() - def __init__(self, evg_api: EvergreenApi, resmoke_proxy: ResmokeProxyService, - timeout_settings: TimeoutSettings) -> None: + def __init__(self, resmoke_proxy: ResmokeProxyService) -> None: """ Initialize the service. - :param evg_api: Evergreen API client. :param resmoke_proxy: Proxy to query resmoke. - :param timeout_settings: Settings for how timeouts are calculated. """ - self.evg_api = evg_api self.resmoke_proxy = resmoke_proxy - self.timeout_settings = timeout_settings def get_timeout_estimate(self, timeout_params: TimeoutParams) -> TimeoutEstimate: """ @@ -66,7 +52,10 @@ class TimeoutService: if not historic_stats: return TimeoutEstimate.no_timeouts() - test_set = set(self.resmoke_proxy.list_tests(timeout_params.suite_name)) + test_set = { + normalize_test_name(test) + for test in self.resmoke_proxy.list_tests(timeout_params.suite_name) + } test_runtimes = [ stat for stat in historic_stats.get_tests_runtimes() if stat.test_name in test_set ] @@ -129,7 +118,8 @@ class TimeoutService: return n_expected_runs * avg_clean_every_n_runtime return 0.0 - def lookup_historic_stats(self, timeout_params: TimeoutParams) -> Optional[HistoricTaskData]: + @staticmethod + def lookup_historic_stats(timeout_params: TimeoutParams) -> Optional[HistoricTaskData]: """ Lookup historic test results stats for the given task. @@ -137,13 +127,16 @@ class TimeoutService: :return: Historic test results if they exist. """ try: - evg_stats = HistoricTaskData.from_evg( - self.evg_api, timeout_params.evg_project, self.timeout_settings.start_date, - self.timeout_settings.end_date, timeout_params.task_name, - timeout_params.build_variant) + LOGGER.info( + "Getting historic runtime information", evg_project=timeout_params.evg_project, + build_variant=timeout_params.build_variant, task_name=timeout_params.task_name) + evg_stats = HistoricTaskData.from_s3( + timeout_params.evg_project, timeout_params.task_name, timeout_params.build_variant) if not evg_stats: LOGGER.warning("No historic runtime information available") return None + LOGGER.info("Found historic runtime information", + evg_stats=evg_stats.historic_test_results) return evg_stats except Exception: # pylint: disable=broad-except # If we have any trouble getting the historic runtime information, log the issue, but diff --git a/buildscripts/util/teststats.py b/buildscripts/util/teststats.py index a52fa3c79a4..74ef9c5ab3e 100644 --- a/buildscripts/util/teststats.py +++ b/buildscripts/util/teststats.py @@ -1,15 +1,35 @@ """Utility to support parsing a TestStat.""" from collections import defaultdict from dataclasses import dataclass -from datetime import datetime from itertools import chain -from typing import NamedTuple, List, Callable, Optional +from json import JSONDecodeError -from evergreen import EvergreenApi, TestStats +from typing import NamedTuple, List, Callable, Optional +import requests +from requests.adapters import HTTPAdapter, Retry from buildscripts.util.testname import split_test_hook_name, is_resmoke_hook, get_short_name_from_test_file TASK_LEVEL_HOOKS = {"CleanEveryN"} +TESTS_STATS_S3_LOCATION = "https://mongo-test-stats.s3.amazonaws.com" + + +class HistoricalTestInformation(NamedTuple): + """ + Container for information about the historical runtime of a test. + + test_name: Name of test. + avg_duration_pass: Average of runtime of test that passed. + num_pass: Number of times the test has passed. + num_fail: Number of times the test has failed. + max_duration_pass: Maximum runtime of the test when it passed. + """ + + test_name: str + num_pass: int + num_fail: int + avg_duration_pass: float + max_duration_pass: Optional[float] = None class TestRuntime(NamedTuple): @@ -74,9 +94,9 @@ class HistoricHookInfo(NamedTuple): avg_duration: float @classmethod - def from_test_stats(cls, test_stats: TestStats) -> "HistoricHookInfo": + def from_test_stats(cls, test_stats: HistoricalTestInformation) -> "HistoricHookInfo": """Create an instance from a test_stats object.""" - return cls(hook_id=test_stats.test_file, num_pass=test_stats.num_pass, + return cls(hook_id=test_stats.test_name, num_pass=test_stats.num_pass, avg_duration=test_stats.avg_duration_pass) def test_name(self) -> str: @@ -101,10 +121,10 @@ class HistoricTestInfo(NamedTuple): hooks: List[HistoricHookInfo] @classmethod - def from_test_stats(cls, test_stats: TestStats, + def from_test_stats(cls, test_stats: HistoricalTestInformation, hooks: List[HistoricHookInfo]) -> "HistoricTestInfo": """Create an instance from a test_stats object.""" - return cls(test_name=test_stats.test_file, num_pass=test_stats.num_pass, + return cls(test_name=test_stats.test_name, num_pass=test_stats.num_pass, avg_duration=test_stats.avg_duration_pass, hooks=hooks) def normalized_test_name(self) -> str: @@ -123,7 +143,9 @@ class HistoricTestInfo(NamedTuple): def total_test_runtime(self) -> float: """Get the average runtime of this test and it's non-task level hooks.""" - return self.avg_duration + self.total_hook_runtime(lambda h: not h.is_task_level_hook()) + if self.num_pass > 0: + return self.avg_duration + self.total_hook_runtime(lambda h: not h.is_task_level_hook()) + return 0.0 def get_hook_overhead(self) -> float: """Get the average runtime of this test and it's non-task level hooks.""" @@ -137,46 +159,59 @@ class HistoricTaskData(object): """Initialize the TestStats with raw results from the Evergreen API.""" self.historic_test_results = historic_test_results - # pylint: disable=too-many-arguments + @staticmethod + def get_stats_from_s3(project: str, task: str, variant: str) -> List[HistoricalTestInformation]: + """ + Retrieve test stats from s3 for a given task. + + :param project: Project to query. + :param task: Task to query. + :param variant: Build variant to query. + :return: A list of the Test stats for the specified task. + """ + session = requests.Session() + retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504]) + session.mount('https://', HTTPAdapter(max_retries=retries)) + + response = session.get(f"{TESTS_STATS_S3_LOCATION}/{project}/{variant}/{task}") + + try: + data = response.json() + return [HistoricalTestInformation(**item) for item in data] + except JSONDecodeError: + return [] + @classmethod - def from_evg(cls, evg_api: EvergreenApi, project: str, start_date: datetime, end_date: datetime, - task: str, variant: str) -> "HistoricTaskData": + def from_s3(cls, project: str, task: str, variant: str) -> "HistoricTaskData": """ - Retrieve test stats from evergreen for a given task. + Retrieve test stats from s3 for a given task. - :param evg_api: Evergreen API client. :param project: Project to query. - :param start_date: Start date to query. - :param end_date: End date to query. :param task: Task to query. :param variant: Build variant to query. :return: Test stats for the specified task. """ - days = (end_date - start_date).days - historic_stats = evg_api.test_stats_by_project( - project, after_date=start_date, before_date=end_date, tasks=[task], variants=[variant], - group_by="test", group_num_days=days) - - return cls.from_stats_list(historic_stats) + historical_test_data = cls.get_stats_from_s3(project, task, variant) + return cls.from_stats_list(historical_test_data) @classmethod - def from_stats_list(cls, historic_stats: List[TestStats]) -> "HistoricTaskData": + def from_stats_list( + cls, historical_test_data: List[HistoricalTestInformation]) -> "HistoricTaskData": """ Build historic task data from a list of historic stats. - :param historic_stats: List of historic stats to build from. + :param historical_test_data: A list of information about the runtime of a test. :return: Historic task data from the list of stats. """ - hooks = defaultdict(list) - for hook in [stat for stat in historic_stats if is_resmoke_hook(stat.test_file)]: + for hook in [stat for stat in historical_test_data if is_resmoke_hook(stat.test_name)]: historical_hook = HistoricHookInfo.from_test_stats(hook) hooks[historical_hook.test_name()].append(historical_hook) return cls([ HistoricTestInfo.from_test_stats(stat, - hooks[get_short_name_from_test_file(stat.test_file)]) - for stat in historic_stats if not is_resmoke_hook(stat.test_file) + hooks[get_short_name_from_test_file(stat.test_name)]) + for stat in historical_test_data if not is_resmoke_hook(stat.test_name) ]) def get_tests_runtimes(self) -> List[TestRuntime]: |