diff options
Diffstat (limited to 'buildscripts')
84 files changed, 1998 insertions, 3326 deletions
diff --git a/buildscripts/auto_install_db_contrib_tool.sh b/buildscripts/auto_install_db_contrib_tool.sh index 1ba77c12460..23a86ad0bc4 100755 --- a/buildscripts/auto_install_db_contrib_tool.sh +++ b/buildscripts/auto_install_db_contrib_tool.sh @@ -20,6 +20,7 @@ if [[ -f "$HOME/.zshrc" ]]; then rc_file="$HOME/.zshrc" fi +export PIP_CACHE_DIR=${workdir}/pip_cache if ! command -v db-contrib-tool &> /dev/null; then if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &> /dev/null; then actual_version=$(python3 -c 'import sys; print(sys.version)') diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py deleted file mode 100644 index 3e391fc1b11..00000000000 --- a/buildscripts/burn_in_tags.py +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env python3 -"""Generate burn in tests to run on certain build variants.""" -from collections import namedtuple -import logging -import os -import sys -from typing import Any, Dict, List - -import click - -from git import Repo -from shrub.v2 import ShrubProject, BuildVariant, ExistingTask -from evergreen.api import RetryingEvergreenApi, EvergreenApi - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -from buildscripts.patch_builds.task_generation import validate_task_generation_limit - -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# pylint: disable=wrong-import-position -from buildscripts.evergreen_burn_in_tests import GenerateBurnInExecutor, GenerateConfig, \ - EvergreenFileChangeDetector -from buildscripts.util.fileops import write_file_to_dir -import buildscripts.util.read_config as read_config -from buildscripts.ciconfig import evergreen -from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant -from buildscripts.burn_in_tests import create_tests_by_task, RepeatConfig, DEFAULT_REPO_LOCATIONS -# pylint: enable=wrong-import-position - -EXTERNAL_LOGGERS = { - "evergreen", - "git", - "urllib3", -} -CONFIG_DIRECTORY = "generated_burn_in_tags_config" -CONFIG_FILE = "burn_in_tags_gen.json" -EVERGREEN_FILE = "etc/evergreen.yml" -EVG_CONFIG_FILE = ".evergreen.yml" - -COMPILE_TASK = "compile_and_archive_dist_test_TG" -# Burn in tags requires running on RHEL80 currently. -COMPILE_TASK_DISTRO = "rhel80-large" - -TASK_ID_EXPANSION = "task_id" - -ConfigOptions = namedtuple("ConfigOptions", [ - "build_variant", - "run_build_variant", - "base_commit", - "max_revisions", - "branch", - "check_evergreen", - "distro", - "repeat_tests_secs", - "repeat_tests_min", - "repeat_tests_max", - "project", -]) - - -def _get_config_options(expansions_file_data, build_variant, run_build_variant): - """ - Get the configuration to use. - - :param expansions_file_data: Config data file to use. - :param build_variant: The buildvariant the current patch should be compared against to figure - out which tests have changed. - :param run_build_variant: The buildvariant the generated task should be run on. - :return: ConfigOptions for the generated task to use. - """ - base_commit = expansions_file_data["revision"] - max_revisions = int(expansions_file_data["max_revisions"]) - branch = expansions_file_data["branch_name"] - is_patch = expansions_file_data.get("is_patch", False) - check_evergreen = is_patch != "true" - distro = expansions_file_data["distro_id"] - repeat_tests_min = int(expansions_file_data["repeat_tests_min"]) - repeat_tests_max = int(expansions_file_data["repeat_tests_max"]) - repeat_tests_secs = int(expansions_file_data["repeat_tests_secs"]) - project = expansions_file_data["project"] - - return ConfigOptions(build_variant, run_build_variant, base_commit, max_revisions, branch, - check_evergreen, distro, repeat_tests_secs, repeat_tests_min, - repeat_tests_max, project) - - -def _create_evg_build_variant_map(expansions_file_data): - """ - Generate relationship of base buildvariant to generated buildvariant. - - :param expansions_file_data: Config data file to use. - :return: Map of base buildvariants to their generated buildvariants. - """ - burn_in_tag_build_variants = expansions_file_data["burn_in_tag_include_build_variants"] - - if burn_in_tag_build_variants: - return { - base_variant: f"{base_variant}-required" - for base_variant in burn_in_tag_build_variants.split(" ") - } - - return {} - - -def _generate_evg_build_variant( - source_build_variant: Variant, - run_build_variant: str, - bypass_build_variant: str, -) -> BuildVariant: - """ - Generate a shrub build variant for the given run build variant. - - :param source_build_variant: The build variant to base configuration on. - :param run_build_variant: The build variant to generate. - :param bypass_build_variant: The build variant to get compile artifacts from. - :return: Shrub build variant configuration. - """ - display_name = f"! {source_build_variant.display_name}" - run_on = source_build_variant.run_on - modules = source_build_variant.modules - - expansions = source_build_variant.expansions - expansions["burn_in_bypass"] = bypass_build_variant - - build_variant = BuildVariant(run_build_variant, display_name, expansions=expansions, - modules=modules, run_on=run_on) - build_variant.add_existing_task(ExistingTask(COMPILE_TASK), distros=[COMPILE_TASK_DISTRO]) - return build_variant - - -# pylint: disable=too-many-arguments,too-many-locals -def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, - task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], - repos: List[Repo], evg_conf: EvergreenProjectConfig, - install_dir: str) -> None: - """ - Generate burn in tests tasks for a given shrub config and group of build variants. - - :param evergreen_api: Evergreen.py object. - :param shrub_project: Shrub config object that the build variants will be built upon. - :param task_expansions: Dictionary of expansions for the running task. - :param build_variant_map: Map of base buildvariants to their generated buildvariant. - :param repos: Git repositories. - """ - for build_variant, run_build_variant in build_variant_map.items(): - config_options = _get_config_options(task_expansions, build_variant, run_build_variant) - task_id = task_expansions[TASK_ID_EXPANSION] - change_detector = EvergreenFileChangeDetector(task_id, evergreen_api, os.environ) - changed_tests = change_detector.find_changed_tests(repos) - tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests, install_dir) - if tests_by_task: - shrub_build_variant = _generate_evg_build_variant( - evg_conf.get_variant(build_variant), run_build_variant, - task_expansions["build_variant"]) - gen_config = GenerateConfig(build_variant, config_options.project, run_build_variant, - config_options.distro, - include_gen_task=False).validate(evg_conf) - repeat_config = RepeatConfig(repeat_tests_min=config_options.repeat_tests_min, - repeat_tests_max=config_options.repeat_tests_max, - repeat_tests_secs=config_options.repeat_tests_secs) - - burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config) - burn_in_generator.generate_tasks_for_variant(tests_by_task, shrub_build_variant) - shrub_project.add_build_variant(shrub_build_variant) - - -def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig, - evergreen_api: RetryingEvergreenApi, repos: List[Repo], install_dir: str): - """ - Execute main program. - - :param task_expansions: Dictionary of expansions for the running task. - :param evg_conf: Evergreen configuration. - :param evergreen_api: Evergreen.py object. - :param repos: Git repositories. - :param install_dir: path to bin directory of a testable installation - """ - shrub_project = ShrubProject.empty() - build_variant_map = _create_evg_build_variant_map(task_expansions) - _generate_evg_tasks(evergreen_api, shrub_project, task_expansions, build_variant_map, repos, - evg_conf, install_dir) - - if not validate_task_generation_limit(shrub_project): - sys.exit(1) - write_file_to_dir(CONFIG_DIRECTORY, CONFIG_FILE, shrub_project.json()) - - -def _configure_logging(verbose: bool): - """ - Configure logging for the application. - - :param verbose: If True set log level to DEBUG. - """ - level = logging.DEBUG if verbose else logging.INFO - logging.basicConfig( - format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", - level=level, - stream=sys.stdout, - ) - for log_name in EXTERNAL_LOGGERS: - logging.getLogger(log_name).setLevel(logging.WARNING) - - -@click.command() -@click.option("--expansion-file", "expansion_file", required=True, - help="Location of expansions file generated by evergreen.") -@click.option("--verbose", is_flag=True) -@click.option("--install-dir", "install_dir", required=True, - help="Path to bin directory of a testable installation") -def main(expansion_file: str, verbose: bool, install_dir: str): - """ - Run new or changed tests in repeated mode to validate their stability. - - burn_in_tags detects jstests that are new or changed since the last git command and then - runs those tests in a loop to validate their reliability. - - \f - :param expansion_file: The expansion file containing the configuration params. - """ - _configure_logging(verbose) - evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE) - repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] - expansions_file_data = read_config.read_config_file(expansion_file) - evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) - - burn_in(expansions_file_data, evg_conf, evg_api, repos, install_dir) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 11bb876a5ea..6b3a6bb7bc5 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -10,13 +10,13 @@ import subprocess import sys from abc import ABC, abstractmethod from collections import defaultdict -from typing import Optional, Set, Tuple, List, Dict, NamedTuple +from typing import Dict, List, NamedTuple, Optional, Set, Tuple import click +import structlog import yaml from git import Repo from pydantic import BaseModel -import structlog from structlog.stdlib import LoggerFactory # Get relative imports to work when the package is not installed on the PYTHONPATH. @@ -40,7 +40,7 @@ EXTERNAL_LOGGERS = { "urllib3", } -DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" +DEFAULT_VARIANT = "enterprise-rhel-8-64-bit-dynamic-required" ENTERPRISE_MODULE_PATH = "src/mongo/db/modules/enterprise" DEFAULT_REPO_LOCATIONS = ["."] REPEAT_SUITES = 2 @@ -218,79 +218,72 @@ def _get_task_name(task): return task.name -def _distro_to_run_task_on(task: VariantTask, evg_proj_config: EvergreenProjectConfig, - build_variant: str) -> str: +class SuiteToBurnInInfo(NamedTuple): """ - Determine what distro an task should be run on. - - For normal tasks, the distro will be the default for the build variant unless the task spec - specifies a particular distro to run on. + Information about tests to run under a specific resmoke suite. - For generated tasks, the distro will be the default for the build variant unless (1) the - "use_large_distro" flag is set as a "var" in the "generate resmoke tasks" command of the - task definition and (2) the build variant defines the "large_distro_name" in its expansions. - - :param task: Task being run. - :param evg_proj_config: Evergreen project configuration. - :param build_variant: Build Variant task is being run on. - :return: Distro task should be run on. + name: Name of resmoke.py suite. + resmoke_args: Arguments to provide to resmoke on suite invocation. + tests: List of tests to run as part of suite. """ - task_def = evg_proj_config.get_task(task.name) - if task_def.is_generate_resmoke_task: - resmoke_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) - if "use_large_distro" in resmoke_vars: - evg_build_variant = _get_evg_build_variant_by_name(evg_proj_config, build_variant) - if "large_distro_name" in evg_build_variant.raw["expansions"]: - return evg_build_variant.raw["expansions"]["large_distro_name"] - return task.run_on[0] + name: str + resmoke_args: str + tests: List[str] -class TaskInfo(NamedTuple): +class TaskToBurnInInfo(NamedTuple): """ Information about tests to run under a specific Task. display_task_name: Display name of task. - suite: Name of resmoke.pu suite that runs in this task. - resmoke_args: Arguments to provide to resmoke on task invocation. - tests: List of tests to run as part of task. - require_multiversion_setup: Requires downloading Multiversion binaries. - distro: Evergreen distro task runs on. - build_variant: Evergreen build variant the task runs on. + suites: List of suites with tests to run. """ display_task_name: str - require_multiversion_setup: bool - suite: str - resmoke_args: str - tests: List[str] - distro: str - build_variant: str + suites: List[SuiteToBurnInInfo] @classmethod - def from_task(cls, task: VariantTask, tests_by_suite: Dict[str, List[str]], - evg_proj_config: EvergreenProjectConfig, build_variant: str) -> "TaskInfo": + def from_task( + cls, + task: VariantTask, + tests_by_suite: Dict[str, List[str]], + ) -> "TaskToBurnInInfo": """ Gather the information needed to run the given task. :param task: Task to be run. :param tests_by_suite: Dict of suites. - :param evg_proj_config: Evergreen project configuration. - :param build_variant: Build variant task will be run on. :return: Dictionary of information needed to run task. """ - suite = task.get_suite_name() + suites_to_burn_in = [ + SuiteToBurnInInfo( + name=suite_name, + resmoke_args=resmoke_args, + tests=tests_by_suite[suite_name], + ) for suite_name, resmoke_args in task.combined_suite_to_resmoke_args_map.items() + if len(tests_by_suite[suite_name]) > 0 + ] return cls( - display_task_name=_get_task_name(task), resmoke_args=task.resmoke_args, suite=suite, - tests=tests_by_suite[suite], - require_multiversion_setup=task.require_multiversion_setup(), - distro=_distro_to_run_task_on(task, evg_proj_config, - build_variant), build_variant=build_variant) + display_task_name=_get_task_name(task), + suites=suites_to_burn_in, + ) + + def collect_suite_tests(self) -> List[str]: + """ + Collect all tests that sub suites should run. + + :return: List of tests from sub suites. + """ + test_set = set() + for suite in self.suites: + test_set.update(suite.tests) + return list(test_set) def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str, tests_by_suite: Dict[str, List[str]], - exclude_tasks: [str]) -> Dict[str, TaskInfo]: + exclude_tasks: [str]) -> Dict[str, TaskToBurnInInfo]: """ Find associated tasks for the specified build_variant and suites. @@ -315,8 +308,9 @@ def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str, # Return the list of tasks to run for the specified suite. task_list = { - task_name: TaskInfo.from_task(task, tests_by_suite, evergreen_conf, build_variant) - for task_name, task in all_variant_tasks.items() if task.get_suite_name() in tests_by_suite + task_name: TaskToBurnInInfo.from_task(task, tests_by_suite) + for task_name, task in all_variant_tasks.items() + if any(suite in tests_by_suite for suite in task.get_suite_names()) } log.debug("Found task list", task_list=task_list) @@ -337,7 +331,7 @@ def _set_resmoke_cmd(repeat_config: RepeatConfig, resmoke_args: [str]) -> [str]: def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, evg_conf: EvergreenProjectConfig, exclude_suites: Optional[List] = None, - exclude_tasks: Optional[List] = None) -> Dict[str, TaskInfo]: + exclude_tasks: Optional[List] = None) -> Dict[str, TaskToBurnInInfo]: """ Create a list of tests by task for the given tests. @@ -364,7 +358,7 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, changed_tests: Set[str], - install_dir: Optional[str]) -> Dict[str, TaskInfo]: + install_dir: Optional[str]) -> Dict[str, TaskToBurnInInfo]: """ Create a list of tests by task. @@ -392,7 +386,7 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, return {} -def run_tests(tests_by_task: Dict[str, TaskInfo], resmoke_cmd: [str]) -> None: +def run_tests(tests_by_task: Dict[str, TaskToBurnInInfo], resmoke_cmd: [str]) -> None: """ Run the given tests locally. @@ -402,16 +396,17 @@ def run_tests(tests_by_task: Dict[str, TaskInfo], resmoke_cmd: [str]) -> None: :param resmoke_cmd: Parameter to use when calling resmoke. """ for task in sorted(tests_by_task): - log = LOGGER.bind(task=task) - new_resmoke_cmd = copy.deepcopy(resmoke_cmd) - new_resmoke_cmd.extend(shlex.split(tests_by_task[task].resmoke_args)) - new_resmoke_cmd.extend(tests_by_task[task].tests) - log.debug("starting execution of task") - try: - subprocess.check_call(new_resmoke_cmd, shell=False) - except subprocess.CalledProcessError as err: - log.warning("Resmoke returned an error with task", error=err.returncode) - sys.exit(err.returncode) + for suite in tests_by_task[task].suites: + log = LOGGER.bind(suite=suite.name) + new_resmoke_cmd = copy.deepcopy(resmoke_cmd) + new_resmoke_cmd.extend(shlex.split(suite.resmoke_args)) + new_resmoke_cmd.extend(suite.tests) + log.debug("starting execution of suite") + try: + subprocess.check_call(new_resmoke_cmd, shell=False) + except subprocess.CalledProcessError as err: + log.warning("Resmoke returned an error with suite", error=err.returncode) + sys.exit(err.returncode) def _configure_logging(verbose: bool): @@ -504,7 +499,7 @@ class BurnInExecutor(ABC): """An interface to execute discovered tests.""" @abstractmethod - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -516,7 +511,7 @@ class BurnInExecutor(ABC): class NopBurnInExecutor(BurnInExecutor): """A burn-in executor that displays results, but doesn't execute.""" - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -524,9 +519,11 @@ class NopBurnInExecutor(BurnInExecutor): """ LOGGER.info("Not running tests due to 'no_exec' option.") for task_name, task_info in tests_by_task.items(): - print(task_name) - for test_name in task_info.tests: - print(f"- {test_name}") + print(f"{task_name}:") + for suite in task_info.suites: + print(f" {suite.name}:") + for test_name in suite.tests: + print(f" - {test_name}") class LocalBurnInExecutor(BurnInExecutor): @@ -542,7 +539,7 @@ class LocalBurnInExecutor(BurnInExecutor): self.resmoke_args = resmoke_args self.repeat_config = repeat_config - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: """ Execute the given tests in the given tasks. @@ -553,16 +550,28 @@ class LocalBurnInExecutor(BurnInExecutor): run_tests(tests_by_task, resmoke_cmd) +class DiscoveredSuite(BaseModel): + """ + Model for a discovered suite to run. + + * suite_name: Name of discovered suite. + * test_list: List of tests to run under discovered suite. + """ + + suite_name: str + test_list: List[str] + + class DiscoveredTask(BaseModel): """ Model for a discovered task to run. * task_name: Name of discovered task. - * test_list: List of tests to run under discovered task. + * suites: List of suites to run under discovered task. """ task_name: str - test_list: List[str] + suites: List[DiscoveredSuite] class DiscoveredTaskList(BaseModel): @@ -574,15 +583,20 @@ class DiscoveredTaskList(BaseModel): class YamlBurnInExecutor(BurnInExecutor): """A burn-in executor that outputs discovered tasks as YAML.""" - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + def execute(self, tests_by_task: Dict[str, TaskToBurnInInfo]) -> None: """ Report the given tasks and their tests to stdout. :param tests_by_task: Dictionary of tasks to run with tests to run in each. """ discovered_tasks = DiscoveredTaskList(discovered_tasks=[ - DiscoveredTask(task_name=task_name, test_list=task_info.tests) - for task_name, task_info in tests_by_task.items() + DiscoveredTask( + task_name=task_name, + suites=[ + DiscoveredSuite(suite_name=suite.name, test_list=suite.tests) + for suite in task_info.suites + ], + ) for task_name, task_info in tests_by_task.items() ]) print(yaml.safe_dump(discovered_tasks.dict())) diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 8f331b5e1ee..ed0cf06b6a8 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -7,7 +7,9 @@ from __future__ import annotations import datetime import distutils.spawn # pylint: disable=no-name-in-module -from typing import Set, List, Optional +import os +import subprocess +from typing import Any, Dict, List, Optional, Set import yaml @@ -21,17 +23,22 @@ def parse_evergreen_file(path, evergreen_binary="evergreen"): """Read an Evergreen file and return EvergreenProjectConfig instance.""" if evergreen_binary: if not distutils.spawn.find_executable(evergreen_binary): - raise EnvironmentError( - "Executable '{}' does not exist or is not in the PATH.".format(evergreen_binary)) + default_evergreen_location = os.path.expanduser(os.path.join("~", "evergreen")) + if os.path.exists(default_evergreen_location): + evergreen_binary = default_evergreen_location + elif os.path.exists(f"{default_evergreen_location}.exe"): + evergreen_binary = f"{default_evergreen_location}.exe" + else: + raise EnvironmentError( + "Executable '{}' does not exist or is not in the PATH.".format( + evergreen_binary)) # Call 'evergreen evaluate path' to pre-process the project configuration file. - cmd = runcommand.RunCommand(evergreen_binary) - cmd.add("evaluate") - cmd.add_file(path) - error_code, output = cmd.execute() - if error_code: - raise RuntimeError("Unable to evaluate {}: {}".format(path, output)) - config = yaml.safe_load(output) + cmd = [evergreen_binary, "evaluate", path] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError("Unable to evaluate {}: {}".format(path, result.stdout)) + config = yaml.safe_load(result.stdout) else: with open(path, "r") as fstream: config = yaml.safe_load(fstream) @@ -124,7 +131,7 @@ class Task(object): return None @property - def generate_resmoke_tasks_command(self): + def generate_resmoke_tasks_command(self) -> Optional[Dict[str, Any]]: """Return the 'generate resmoke tasks' command if found, or None.""" return self.find_func_command("generate resmoke tasks") @@ -134,22 +141,27 @@ class Task(object): return self.generate_resmoke_tasks_command is not None @property - def run_tests_command(self): + def run_tests_command(self) -> Optional[Dict[str, Any]]: """Return the 'run tests' command if found, or None.""" return self.find_func_command("run tests") @property - def is_run_tests_task(self): + def is_run_tests_task(self) -> bool: """Return True if 'run_tests' command is found.""" return self.run_tests_command is not None @property - def multiversion_setup_command(self): - """Return the 'do multiversion setup' command if found, or None.""" - return self.find_func_command("do multiversion setup") + def initialize_multiversion_tasks_command(self) -> Optional[Dict[str, Any]]: + """Return the 'initialize multiversion tasks' command if found, or None.""" + return self.find_func_command("initialize multiversion tasks") + + @property + def is_initialize_multiversion_tasks_task(self) -> bool: + """Return True if 'initialize multiversion tasks' command is found.""" + return self.initialize_multiversion_tasks_command is not None @property - def generated_task_name(self): + def generated_task_name(self) -> str: """ Get basename of the tasks generated by this _gen task. @@ -160,45 +172,47 @@ class Task(object): return self.name[:-4] - def get_resmoke_command_vars(self): + def get_resmoke_command_vars(self) -> Dict[str, Any]: """Get the vars for either 'generate resmoke tasks' or 'run tests', both eventually call resmoke.py.""" if self.is_run_tests_task: - return self.run_tests_command.get("vars") - elif self.is_generate_resmoke_task: - return self.generate_resmoke_tasks_command.get("vars") - - return None + return self.run_tests_command.get("vars", {}) + if self.is_generate_resmoke_task: + return self.generate_resmoke_tasks_command.get("vars", {}) - def get_suite_name(self): - """Get the name of the resmoke.py suite; the `suite` expansion overrides the task name.""" + return {} - if self.is_run_tests_task: - suite_name = self.name - elif self.is_generate_resmoke_task: - suite_name = self.generated_task_name - else: - raise ValueError(f"{self.name} task does not run a resmoke.py test suite") + def get_suite_names(self) -> List[str]: + """Get the names of the resmoke.py suites from the task definition.""" command_vars = self.get_resmoke_command_vars() - if command_vars is not None: - suite_name = command_vars.get("suite", suite_name) - return suite_name + if self.is_run_tests_task: + return [command_vars.get("suite", self.name)] + if self.is_generate_resmoke_task and not self.is_initialize_multiversion_tasks_task: + return [command_vars.get("suite", self.generated_task_name)] + if self.is_initialize_multiversion_tasks_task: + return [ + suite + for suite in self.initialize_multiversion_tasks_command.get("vars", {}).keys() + ] + + raise ValueError(f"{self.name} task does not run a resmoke.py test suite") @property - def resmoke_args(self): - """Get the resmoke_args from 'run tests' function if defined, or None.""" - suite_name = self.get_suite_name() - command_vars = self.get_resmoke_command_vars() + def suite_to_resmoke_args_map(self) -> Dict[str, str]: + """Get the resmoke.py arguments from the task definition.""" + output = {} + + for suite_name in self.get_suite_names(): + resmoke_args = f"--suites={suite_name}" - other_args = "" - if command_vars: - other_args = command_vars.get("resmoke_args", other_args) + more_args = self.get_resmoke_command_vars().get("resmoke_args") + if more_args is not None: + resmoke_args = f"{resmoke_args} {more_args}" - if not suite_name and not other_args: - return None + output[suite_name] = resmoke_args - return f"--suites={suite_name} {other_args}" + return output @property def tags(self): @@ -211,22 +225,6 @@ class Task(object): """Check if the task requires running the multiversion setup.""" return "multiversion" in self.tags - def require_multiversion_version_combo(self): - """Check if the task requires generating combinations of multiversion versions.""" - return "multiversion" in self.tags and "no_version_combination" not in self.tags - - def requires_npm(self): - """Check if the task needs to run npm setup.""" - return "require_npm" in self.tags - - def is_test_name_random(self): - """ - Check if the name of the tests are randomly generated. - - Those tests won't have associated test history. - """ - return "random_name" in self.tags - def __str__(self): return self.name @@ -394,16 +392,16 @@ class VariantTask(Task): return f"{self.variant}: {self.name}" @property - def combined_resmoke_args(self): + def combined_suite_to_resmoke_args_map(self) -> Dict[str, str]: """Get the combined resmoke arguments. This results from the concatenation of the task's resmoke_args parameter and the variant's test_flags parameter. """ - resmoke_args = self.resmoke_args - test_flags = self.variant.test_flags - if resmoke_args is None: - return None - elif test_flags is None: - return self.resmoke_args - return "{} {}".format(resmoke_args, test_flags) + variant_test_flags = self.variant.test_flags + if variant_test_flags is not None: + output = {} + for suite_name, task_resmoke_args in self.suite_to_resmoke_args_map.items(): + output[suite_name] = f"{task_resmoke_args} {variant_test_flags}" + return output + return self.suite_to_resmoke_args_map diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py deleted file mode 100644 index bc92a2fef33..00000000000 --- a/buildscripts/evergreen_burn_in_tests.py +++ /dev/null @@ -1,513 +0,0 @@ -#!/usr/bin/env python3 -"""Wrapper around burn_in_tests for evergreen execution.""" -import logging -import os -import sys -from datetime import datetime, timedelta -from math import ceil -from typing import Optional, List, Dict, Set, NamedTuple - -import click -import requests -import structlog -from git import Repo -from shrub.v2 import ShrubProject, BuildVariant, Task, TaskDependency, ExistingTask -from evergreen import RetryingEvergreenApi, EvergreenApi - -from buildscripts.burn_in_tests import RepeatConfig, BurnInExecutor, TaskInfo, FileChangeDetector, \ - DEFAULT_REPO_LOCATIONS, BurnInOrchestrator -from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig -from buildscripts.patch_builds.change_data import RevisionMap -from buildscripts.patch_builds.evg_change_data import generate_revision_map_from_manifest -from buildscripts.patch_builds.task_generation import TimeoutInfo, resmoke_commands, \ - validate_task_generation_limit -from buildscripts.task_generation.constants import CONFIG_FILE, EVERGREEN_FILE, ARCHIVE_DIST_TEST_DEBUG_TASK, \ - BACKPORT_REQUIRED_TAG, RUN_TESTS, EXCLUDES_TAGS_FILE -from buildscripts.task_generation.suite_split import SubSuite, GeneratedSuite -from buildscripts.task_generation.task_types.resmoke_tasks import EXCLUDE_TAGS -from buildscripts.util.fileops import write_file -from buildscripts.util.taskname import name_generated_task -from buildscripts.util.teststats import TestRuntime, HistoricTaskData - -DEFAULT_PROJECT = "mongodb-mongo-master" -DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" -BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen" -BURN_IN_TESTS_TASK = "burn_in_tests" -BURN_IN_ENV_VAR = "BURN_IN_TESTS" -AVG_TEST_SETUP_SEC = 4 * 60 -AVG_TEST_TIME_MULTIPLIER = 3 -MIN_AVG_TEST_OVERFLOW_SEC = float(60) -MIN_AVG_TEST_TIME_SEC = 5 * 60 - -LOGGER = structlog.getLogger(__name__) -EXTERNAL_LOGGERS = { - "evergreen", - "git", - "urllib3", -} - - -def _configure_logging(verbose: bool): - """ - Configure logging for the application. - - :param verbose: If True set log level to DEBUG. - """ - level = logging.DEBUG if verbose else logging.INFO - logging.basicConfig( - format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", - level=level, - stream=sys.stdout, - ) - for log_name in EXTERNAL_LOGGERS: - logging.getLogger(log_name).setLevel(logging.WARNING) - - -class GenerateConfig(object): - """Configuration for how to generate tasks.""" - - def __init__(self, build_variant: str, project: str, run_build_variant: Optional[str] = None, - distro: Optional[str] = None, task_id: Optional[str] = None, - task_prefix: str = "burn_in", include_gen_task: bool = True) -> None: - # pylint: disable=too-many-arguments,too-many-locals - """ - Create a GenerateConfig. - - :param build_variant: Build variant to get tasks from. - :param project: Project to run tasks on. - :param run_build_variant: Build variant to run new tasks on. - :param distro: Distro to run tasks on. - :param task_id: Evergreen task being run under. - :param task_prefix: Prefix to include in generated task names. - :param include_gen_task: Indicates the "_gen" task should be grouped in the display task. - """ - self.build_variant = build_variant - self._run_build_variant = run_build_variant - self.distro = distro - self.project = project - self.task_id = task_id - self.task_prefix = task_prefix - self.include_gen_task = include_gen_task - - @property - def run_build_variant(self): - """Build variant tasks should run against.""" - if self._run_build_variant: - return self._run_build_variant - return self.build_variant - - def validate(self, evg_conf: EvergreenProjectConfig): - """ - Raise an exception if this configuration is invalid. - - :param evg_conf: Evergreen configuration. - :return: self. - """ - self._check_variant(self.build_variant, evg_conf) - return self - - @staticmethod - def _check_variant(build_variant: str, evg_conf: EvergreenProjectConfig): - """ - Check if the build_variant is found in the evergreen file. - - :param build_variant: Build variant to check. - :param evg_conf: Evergreen configuration to check against. - """ - if not evg_conf.get_variant(build_variant): - raise ValueError(f"Build variant '{build_variant}' not found in Evergreen file") - - -def _parse_avg_test_runtime(test: str, - task_avg_test_runtime_stats: List[TestRuntime]) -> Optional[float]: - """ - Parse list of test runtimes to find runtime for particular test. - - :param task_avg_test_runtime_stats: List of average historic runtimes of tests. - :param test: Test name. - :return: Historical average runtime of the test. - """ - for test_stat in task_avg_test_runtime_stats: - if test_stat.test_name == test: - return test_stat.runtime - return None - - -def _calculate_timeout(avg_test_runtime: float) -> int: - """ - Calculate timeout_secs for the Evergreen task. - - :param avg_test_runtime: How long a test has historically taken to run. - :return: The test runtime times AVG_TEST_TIME_MULTIPLIER, or MIN_AVG_TEST_TIME_SEC (whichever - is higher). - """ - return max(MIN_AVG_TEST_TIME_SEC, ceil(avg_test_runtime * AVG_TEST_TIME_MULTIPLIER)) - - -def _calculate_exec_timeout(repeat_config: RepeatConfig, avg_test_runtime: float) -> int: - """ - Calculate exec_timeout_secs for the Evergreen task. - - :param repeat_config: Information about how the test will repeat. - :param avg_test_runtime: How long a test has historically taken to run. - :return: repeat_tests_secs + an amount of padding time so that the test has time to finish on - its final run. - """ - LOGGER.debug("Calculating exec timeout", repeat_config=repeat_config, - avg_test_runtime=avg_test_runtime) - repeat_tests_secs = repeat_config.repeat_tests_secs - if avg_test_runtime > repeat_tests_secs and repeat_config.repeat_tests_min: - # If a single execution of the test takes longer than the repeat time, then we don't - # have to worry about the repeat time at all and can just use the average test runtime - # and minimum number of executions to calculate the exec timeout value. - return ceil(avg_test_runtime * AVG_TEST_TIME_MULTIPLIER * repeat_config.repeat_tests_min) - - test_execution_time_over_limit = avg_test_runtime - (repeat_tests_secs % avg_test_runtime) - test_execution_time_over_limit = max(MIN_AVG_TEST_OVERFLOW_SEC, test_execution_time_over_limit) - return ceil(repeat_tests_secs + (test_execution_time_over_limit * AVG_TEST_TIME_MULTIPLIER) + - AVG_TEST_SETUP_SEC) - - -class BurnInGenTaskParams(NamedTuple): - """Parameters describing how a specific resmoke burn in suite should be generated.""" - - resmoke_args: str - require_multiversion_setup: bool - distro: str - - -class BurnInGenTaskService: - """Class to generate task configurations.""" - - def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig, - task_runtime_stats: List[TestRuntime]) -> None: - """ - Create a new task generator. - - :param generate_config: Generate configuration to use. - :param repeat_config: Repeat configuration to use. - :param task_runtime_stats: Historic runtime of tests associated with task. - """ - self.generate_config = generate_config - self.repeat_config = repeat_config - self.task_runtime_stats = task_runtime_stats - - def generate_timeouts(self, test: str) -> TimeoutInfo: - """ - Add timeout.update command to list of commands for a burn in execution task. - - :param test: Test name. - :return: TimeoutInfo to use. - """ - if self.task_runtime_stats: - avg_test_runtime = _parse_avg_test_runtime(test, self.task_runtime_stats) - if avg_test_runtime: - LOGGER.debug("Avg test runtime", test=test, runtime=avg_test_runtime) - - timeout = _calculate_timeout(avg_test_runtime) - exec_timeout = _calculate_exec_timeout(self.repeat_config, avg_test_runtime) - LOGGER.debug("Using timeout overrides", exec_timeout=exec_timeout, timeout=timeout) - timeout_info = TimeoutInfo.overridden(exec_timeout, timeout) - - LOGGER.debug("Override runtime for test", test=test, timeout=timeout_info) - return timeout_info - - return TimeoutInfo.default_timeout() - - def _generate_run_tests_vars(self, task_name: str, suite_name: str, params: BurnInGenTaskParams, - test_arg: str) -> Dict[str, str]: - run_test_vars = {"suite": suite_name} - - resmoke_args = f"{params.resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_arg}" - - if params.require_multiversion_setup: - run_test_vars["require_multiversion_setup"] = params.require_multiversion_setup - - # TODO: inspect the suite for version instead of doing string parsing on the name. - if "last_continuous" in suite_name: - run_test_vars["multiversion_exclude_tags_version"] = "last_continuous" - resmoke_args += f" --tagFile={EXCLUDES_TAGS_FILE}" - elif "last_lts" in suite_name: - run_test_vars["multiversion_exclude_tags_version"] = "last_lts" - resmoke_args += f" --tagFile={EXCLUDES_TAGS_FILE}" - - resmoke_args += f" --excludeWithAnyTags={EXCLUDE_TAGS},{task_name}_{BACKPORT_REQUIRED_TAG} " - - run_test_vars["resmoke_args"] = resmoke_args - - return run_test_vars - - def _generate_task_name(self, gen_suite: GeneratedSuite, index: int) -> str: - """ - Generate a subtask name. - - :param gen_suite: GeneratedSuite object. - :param index: Index of subtask. - :return: Name to use for generated sub-task. - """ - prefix = self.generate_config.task_prefix - task_name = gen_suite.task_name - - return name_generated_task(f"{prefix}:{task_name}", index, len(gen_suite), - self.generate_config.run_build_variant) - - def generate_tasks(self, gen_suite: GeneratedSuite, params: BurnInGenTaskParams) -> Set[Task]: - """Create the task configuration for the given test using the given index.""" - - tasks = set() - for index, suite in enumerate(gen_suite.sub_suites): - if len(suite.test_list) != 1: - raise ValueError( - f"Can only run one test per suite in burn-in; got {suite.test_list}") - test_name = suite.test_list[0] - test_unix_style = test_name.replace('\\', '/') - run_tests_vars = self._generate_run_tests_vars( - gen_suite.task_name, gen_suite.suite_name, params, test_unix_style) - - timeout_cmd = self.generate_timeouts(test_name) - commands = resmoke_commands(RUN_TESTS, run_tests_vars, timeout_cmd, - params.require_multiversion_setup) - dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)} - - tasks.add(Task(self._generate_task_name(gen_suite, index), commands, dependencies)) - - return tasks - - -class EvergreenFileChangeDetector(FileChangeDetector): - """A file changes detector for detecting test change in evergreen.""" - - def __init__(self, task_id: str, evg_api: EvergreenApi, env_map: Dict[str, str]) -> None: - """ - Create a new evergreen file change detector. - - :param task_id: Id of task being run under. - :param evg_api: Evergreen API client. - :param env_map: Map of environment variables. - """ - self.task_id = task_id - self.evg_api = evg_api - self.env_map = env_map - - def create_revision_map(self, repos: List[Repo]) -> RevisionMap: - """ - Create a map of the repos and the given revisions to diff against. - - :param repos: List of repos being tracked. - :return: Map of repositories and revisions to diff against. - """ - return generate_revision_map_from_manifest(repos, self.task_id, self.evg_api) - - def find_changed_tests(self, repos: List[Repo]) -> Set[str]: - """ - Find the list of tests that have changed. - - :param repos: List of repos to check. - :return: Set of all test files that have changed. - """ - tests_set = super().find_changed_tests(repos) - if BURN_IN_ENV_VAR in self.env_map: - # The burn in env var can be set to a list of tests the user has manually specified - # should be included. Add those to the already discovered tests. - tests_set.update(self.env_map[BURN_IN_ENV_VAR].split(",")) - return tests_set - - -def _tests_dict_to_generated_suites(task_info: TaskInfo, tests_runtimes: List[TestRuntime]): - """Convert diction of tests to `GenerateSuite` objects to conform to the *GenTaskService interface.""" - sub_suites = [] - for _, test in enumerate(task_info.tests): - sub_suites.append(SubSuite([test], 0, tests_runtimes)) - return GeneratedSuite(sub_suites=sub_suites, build_variant=task_info.build_variant, - task_name=task_info.display_task_name, suite_name=task_info.suite) - - -class GenerateBurnInExecutor(BurnInExecutor): - """A burn-in executor that generates tasks.""" - - # pylint: disable=too-many-arguments - def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig, - generate_tasks_file: Optional[str] = None) -> None: - """ - Create a new generate burn-in executor. - - :param generate_config: Configuration for how to generate tasks. - :param repeat_config: Configuration for how tests should be repeated. - :param generate_tasks_file: File to write generated task configuration to. - """ - self.generate_config = generate_config - self.repeat_config = repeat_config - self.generate_tasks_file = generate_tasks_file - - def get_task_runtime_history(self, task: str) -> List[TestRuntime]: - """ - Query the runtime history of the specified task. - - :param task: Task to query. - :return: List of runtime histories for all tests in specified task. - """ - project = self.generate_config.project - variant = self.generate_config.build_variant - test_stats = HistoricTaskData.from_s3(project, task, variant) - return test_stats.get_tests_runtimes() - - def _get_existing_tasks(self) -> Optional[Set[ExistingTask]]: - """Get any existing tasks that should be included in the generated display task.""" - if self.generate_config.include_gen_task: - return {ExistingTask(BURN_IN_TESTS_GEN_TASK)} - return None - - def generate_tasks_for_variant(self, tests_by_task: Dict[str, TaskInfo], variant: BuildVariant): - """Add tasks to the passed in variant.""" - tasks = set() - for task in sorted(tests_by_task): - task_info = tests_by_task[task] - task_history = self.get_task_runtime_history(task_info.display_task_name) - gen_suite = _tests_dict_to_generated_suites(task_info, task_history) - - task_generator = BurnInGenTaskService(self.generate_config, self.repeat_config, - task_history) - - params = BurnInGenTaskParams( - resmoke_args=task_info.resmoke_args, - require_multiversion_setup=task_info.require_multiversion_setup, - distro=task_info.distro, - ) - new_tasks = task_generator.generate_tasks(gen_suite, params) - tasks = tasks.union(new_tasks) - variant.display_task(BURN_IN_TESTS_TASK, tasks, - execution_existing_tasks=self._get_existing_tasks()) - - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: - """ - Execute the given tests in the given tasks. - - :param tests_by_task: Dictionary of tasks to run with tests to run in each. - """ - - build_variant = BuildVariant(self.generate_config.run_build_variant) - self.generate_tasks_for_variant(tests_by_task, build_variant) - - shrub_project = ShrubProject.empty() - shrub_project.add_build_variant(build_variant) - if not validate_task_generation_limit(shrub_project): - sys.exit(1) - - assert self.generate_tasks_file is not None - if self.generate_tasks_file: - write_file(self.generate_tasks_file, shrub_project.json()) - - -# pylint: disable=too-many-arguments -def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig, - repeat_config: RepeatConfig, evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, - repos: List[Repo], generate_tasks_file: str, install_dir: str) -> None: - """ - Run burn_in_tests. - - :param task_id: Id of task running. - :param build_variant: Build variant to run against. - :param generate_config: Configuration for how to generate tasks. - :param repeat_config: Configuration for how to repeat tests. - :param evg_api: Evergreen API client. - :param evg_conf: Evergreen project configuration. - :param repos: Git repos containing changes. - :param generate_tasks_file: File to write generate tasks configuration to. - :param install_dir: Path to bin directory of a testable installation - """ - change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ) - executor = GenerateBurnInExecutor(generate_config, repeat_config, generate_tasks_file) - - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) - burn_in_orchestrator.burn_in(repos, build_variant) - - -@click.command() -@click.option("--generate-tasks-file", "generate_tasks_file", default=None, metavar='FILE', - help="Run in 'generate.tasks' mode. Store task config to given file.") -@click.option("--build-variant", "build_variant", default=DEFAULT_VARIANT, metavar='BUILD_VARIANT', - help="Tasks to run will be selected from this build variant.") -@click.option("--run-build-variant", "run_build_variant", default=None, metavar='BUILD_VARIANT', - help="Burn in tasks will be generated on this build variant.") -@click.option("--distro", "distro", default=None, metavar='DISTRO', - help="The distro the tasks will execute on.") -@click.option("--project", "project", default=DEFAULT_PROJECT, metavar='PROJECT', - help="The evergreen project the tasks will execute on.") -@click.option("--repeat-tests", "repeat_tests_num", default=None, type=int, - help="Number of times to repeat tests.") -@click.option("--repeat-tests-min", "repeat_tests_min", default=None, type=int, - help="The minimum number of times to repeat tests if time option is specified.") -@click.option("--repeat-tests-max", "repeat_tests_max", default=None, type=int, - help="The maximum number of times to repeat tests if time option is specified.") -@click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS", - help="Repeat tests for the given time (in secs).") -@click.option("--evg-api-config", "evg_api_config", default=CONFIG_FILE, metavar="FILE", - help="Configuration file with connection info for Evergreen API.") -@click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") -@click.option("--task_id", "task_id", required=True, metavar='TASK_ID', - help="The evergreen task id.") -@click.option("--install-dir", "install_dir", required=True, - help="Path to bin directory of a testable installation.") -# pylint: disable=too-many-arguments,too-many-locals -def main(build_variant: str, run_build_variant: str, distro: str, project: str, - generate_tasks_file: str, repeat_tests_num: Optional[int], repeat_tests_min: Optional[int], - repeat_tests_max: Optional[int], repeat_tests_secs: Optional[int], evg_api_config: str, - verbose: bool, task_id: str, install_dir: str): - """ - Run new or changed tests in repeated mode to validate their stability. - - burn_in_tests detects jstests that are new or changed since the last git command and then - runs those tests in a loop to validate their reliability. - - The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can - either be repeated a specified number of times with the `--repeat-tests` option, or they can - be repeated for a certain time period with the `--repeat-tests-secs` option. - - Specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration - file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks - to do all the test executions. This is the mode used to run tests in patch builds. - - NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate - in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would - be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen - can handle. - \f - - :param build_variant: Build variant to query tasks from. - :param run_build_variant:Build variant to actually run against. - :param distro: Distro to run tests on. - :param project: Project to run tests on. - :param generate_tasks_file: Create a generate tasks configuration in this file. - :param repeat_tests_num: Repeat each test this number of times. - :param repeat_tests_min: Repeat each test at least this number of times. - :param repeat_tests_max: Once this number of repetitions has been reached, stop repeating. - :param repeat_tests_secs: Continue repeating tests for this number of seconds. - :param evg_api_config: Location of configuration file to connect to evergreen. - :param verbose: Log extra debug information. - :param task_id: Id of evergreen task being run in. - :param install_dir: path to bin directory of a testable installation - """ - _configure_logging(verbose) - - repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs, - repeat_tests_min=repeat_tests_min, - repeat_tests_max=repeat_tests_max, - repeat_tests_num=repeat_tests_num) # yapf: disable - - repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] - evg_conf = parse_evergreen_file(EVERGREEN_FILE) - evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) - - generate_config = GenerateConfig(build_variant=build_variant, - run_build_variant=run_build_variant, - distro=distro, - project=project, - task_id=task_id) # yapf: disable - generate_config.validate(evg_conf) - - burn_in(task_id, build_variant, generate_config, repeat_config, evg_api, evg_conf, repos, - generate_tasks_file, install_dir) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py deleted file mode 100644 index 86f8c046c5b..00000000000 --- a/buildscripts/evergreen_gen_build_variant.py +++ /dev/null @@ -1,455 +0,0 @@ -#!/usr/bin/env python3 -"""Generate configuration for a build variant.""" -import hashlib -import os -from concurrent.futures import ThreadPoolExecutor as Executor -from datetime import datetime, timedelta -from time import perf_counter -from typing import Any, Optional, Set, Tuple - -import click -import inject -import structlog -from pydantic import BaseModel -from evergreen import EvergreenApi, RetryingEvergreenApi -from evergreen import Task as EvgTask - -from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, Task, Variant, - parse_evergreen_file) -from buildscripts.task_generation.constants import (EXPANSION_RE, GEN_PARENT_TASK, - GENERATED_CONFIG_DIR, LOOKBACK_DURATION_DAYS, - MAX_TASK_PRIORITY, MAX_WORKERS) -from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder -from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.gen_task_validation import \ - GenTaskValidationService -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.task_generation.suite_split import (SuiteSplitConfig, SuiteSplitParameters) -from buildscripts.task_generation.suite_split_strategies import ( - FallbackStrategy, SplitStrategy, greedy_division, round_robin_fallback) -from buildscripts.task_generation.task_types.fuzzer_tasks import \ - FuzzerGenTaskParams -from buildscripts.task_generation.task_types.gentask_options import \ - GenTaskOptions -from buildscripts.task_generation.task_types.resmoke_tasks import \ - ResmokeGenTaskParams -from buildscripts.util.cmdutils import enable_logging -from buildscripts.util.fileops import read_yaml_file -from buildscripts.util.taskname import remove_gen_suffix - -LOGGER = structlog.get_logger(__name__) - - -class EvgExpansions(BaseModel): - """ - Evergreen expansions needed to generate tasks. - - build_id: Build ID being run under. - build_variant: Build variant being generated. - exec_timeout_secs: Seconds to wait before considering a task timed out. - gen_task_gran: Granularity of how tasks are being generated. - is_patch: Whether generation is part of a patch build. - project: Evergreen project being run under. - max_test_per_suite: Maximum amount of tests to include in a suite. - max_sub_suites: Maximum number of sub-suites to generate per task. - mainline_max_sub_suites: Max number of sub-suites to generate per task on mainline builds. - resmoke_repeat_suites: Number of times suites should be repeated. - revision: Git revision being run against. - task_name: Name of task running. - target_resmoke_time: Target time of generated sub-suites. - task_id: ID of task being run under. - timeout_secs: Seconds to wait with no output before considering a task timed out. - """ - - build_id: str - build_variant: str - exec_timeout_secs: Optional[int] = None - is_patch: Optional[bool] - project: str - max_tests_per_suite: Optional[int] = 100 - max_sub_suites: Optional[int] = 5 - mainline_max_sub_suites: Optional[int] = 1 - resmoke_repeat_suites: Optional[int] = None - revision: str - task_name: str - target_resmoke_time: Optional[int] = None - task_id: str - timeout_secs: Optional[int] = None - - @classmethod - def from_yaml_file(cls, path: str) -> "EvgExpansions": - """ - Read the evergreen expansions from the given YAML file. - - :param path: Path to expansions YAML file. - :return: Expansions read from file. - """ - return cls(**read_yaml_file(path)) - - def get_max_sub_suites(self) -> int: - """Get the max_sub_suites to use.""" - if not self.is_patch: - return self.mainline_max_sub_suites - return self.max_sub_suites - - def build_suite_split_config(self, start_date: datetime, - end_date: datetime) -> SuiteSplitConfig: - """ - Get the configuration for splitting suites based on Evergreen expansions. - - :param start_date: Start date for historic stats lookup. - :param end_date: End date for historic stats lookup. - :return: Configuration to use for splitting suites. - """ - return SuiteSplitConfig( - evg_project=self.project, - target_resmoke_time=self.target_resmoke_time if self.target_resmoke_time else 60, - max_sub_suites=self.get_max_sub_suites(), - max_tests_per_suite=self.max_tests_per_suite, - start_date=start_date, - end_date=end_date, - ) - - def build_evg_config_gen_options(self) -> GenTaskOptions: - """ - Get the configuration for generating tasks from Evergreen expansions. - - :return: Configuration to use for splitting suites. - """ - return GenTaskOptions( - create_misc_suite=True, - is_patch=self.is_patch, - generated_config_dir=GENERATED_CONFIG_DIR, - use_default_timeouts=False, - timeout_secs=self.timeout_secs, - exec_timeout_secs=self.exec_timeout_secs, - ) - - def config_location(self) -> str: - """Location where generated configuration is stored.""" - generated_task_name = remove_gen_suffix(self.task_name) - file = f"{self.build_variant}/{self.revision}/generate_tasks/{generated_task_name}_gen-{self.build_id}" - # hash 'file' to shorten the file path dramatically - sha1 = hashlib.sha1() - sha1.update(file.encode('utf-8')) - return f"gtcl/{sha1.hexdigest()}.tgz" - - -def translate_run_var(run_var: str, build_variant: Variant) -> Any: - """ - Translate the given "run_var" into an actual value. - - Run_vars can contain evergreen expansions, in which case, the expansion (and possible default - value) need to be translated into a value we can use. - - :param run_var: Run var to translate. - :param build_variant: Build variant configuration. - :return: Value of run_var. - """ - match = EXPANSION_RE.search(run_var) - if match: - value = build_variant.expansion(match.group("id")) - if value is None: - value = match.group("default") - return value - return run_var - - -class GenerateBuildVariantOrchestrator: - """Orchestrator for generating tasks in a build variant.""" - - # pylint: disable=too-many-arguments - @inject.autoparams() - def __init__( - self, - gen_task_validation: GenTaskValidationService, - gen_task_options: GenTaskOptions, - evg_project_config: EvergreenProjectConfig, - evg_expansions: EvgExpansions, - evg_api: EvergreenApi, - ) -> None: - """ - Initialize the orchestrator. - - :param gen_task_validation: Service to validate task generation. - :param gen_task_options: Options for how tasks should be generated. - :param evg_project_config: Configuration for Evergreen Project. - :param evg_expansions: Evergreen expansions for running task. - :param evg_api: Evergreen API client. - """ - self.gen_task_validation = gen_task_validation - self.gen_task_options = gen_task_options - self.evg_project_config = evg_project_config - self.evg_expansions = evg_expansions - self.evg_api = evg_api - - def get_build_variant_expansion(self, build_variant_name: str, expansion: str) -> Any: - """ - Get the value of the given expansion for the specified build variant. - - :param build_variant_name: Build Variant to query. - :param expansion: Expansion to query. - :return: Value of given expansion. - """ - build_variant = self.evg_project_config.get_variant(build_variant_name) - return build_variant.expansion(expansion) - - def task_def_to_split_params(self, task_def: Task, - build_variant_gen: str) -> SuiteSplitParameters: - """ - Build parameters for how a task should be split based on its task definition. - - :param task_def: Task definition in evergreen project config. - :param build_variant_gen: Name of Build Variant being generated. - :return: Parameters for how task should be split. - """ - build_variant = self.evg_project_config.get_variant(build_variant_gen) - task_name = remove_gen_suffix(task_def.name) - run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) - - suite_name = run_vars.get("suite", task_name) - return SuiteSplitParameters( - build_variant=build_variant_gen, - task_name=task_name, - suite_name=suite_name, - filename=suite_name, - is_asan=build_variant.is_asan_build(), - ) - - def task_def_to_gen_params(self, task_def: Task, build_variant: str) -> ResmokeGenTaskParams: - """ - Build parameters for how a task should be generated based on its task definition. - - :param task_def: Task definition in evergreen project config. - :param build_variant: Name of Build Variant being generated. - :return: Parameters for how task should be generated. - """ - run_func = task_def.generate_resmoke_tasks_command - run_vars = run_func.get("vars", {}) - - repeat_suites = 1 - if self.evg_expansions.resmoke_repeat_suites: - repeat_suites = self.evg_expansions.resmoke_repeat_suites - - return ResmokeGenTaskParams( - use_large_distro=run_vars.get("use_large_distro"), - require_multiversion_setup=task_def.require_multiversion_setup(), - require_multiversion_version_combo=task_def.require_multiversion_version_combo(), - repeat_suites=repeat_suites, - resmoke_args=run_vars.get("resmoke_args"), - resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), - large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), - config_location=self.evg_expansions.config_location(), - dependencies=self.determine_task_dependencies(task_def), - ) - - def determine_task_dependencies(self, task_def: Task) -> Set[str]: - """ - Determine the dependencies to use for tasks generated from the given task definition. - - This should include all tasks that the task definition depends on except for the currently - running task. - - :param task_def: Task definition to use. - :return: Set of dependencies to generate. - """ - dependency_set = { - task["name"] - for task in task_def.depends_on if task["name"] != self.evg_expansions.task_name - } - - return dependency_set - - def task_def_to_fuzzer_params(self, task_def: Task, build_variant: str) -> FuzzerGenTaskParams: - """ - Build parameters for how a fuzzer task should be generated based on its task definition. - - :param task_def: Task definition in evergreen project config. - :param build_variant: Name of Build Variant being generated. - :return: Parameters for how a fuzzer task should be generated. - """ - variant = self.evg_project_config.get_variant(build_variant) - run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) - run_vars = {k: translate_run_var(v, variant) for k, v in run_vars.items()} - - num_tasks = min(int(run_vars.get("num_tasks")), self.evg_expansions.get_max_sub_suites()) - - return FuzzerGenTaskParams( - task_name=remove_gen_suffix(task_def.name), - variant=build_variant, - suite=run_vars.get("suite"), - num_files=int(run_vars.get("num_files")), - num_tasks=num_tasks, - resmoke_args=run_vars.get("resmoke_args"), - npm_command=run_vars.get("npm_command", "jstestfuzz"), - jstestfuzz_vars=run_vars.get("jstestfuzz_vars", ""), - continue_on_failure=run_vars.get("continue_on_failure"), - resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), - should_shuffle=run_vars.get("should_shuffle"), - timeout_secs=run_vars.get("timeout_secs"), - require_multiversion_setup=task_def.require_multiversion_setup(), - use_large_distro=run_vars.get("use_large_distro", False), - large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), - config_location=self.evg_expansions.config_location(), - dependencies=self.determine_task_dependencies(task_def), - ) - - def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None: - """ - Write task configuration for a build variant to disk. - - :param task_id: ID of running task. - :param build_variant_name: Name of build variant to generate. - :param output_file: Filename to write generated configuration to. - """ - if not self.gen_task_validation.should_task_be_generated(task_id): - LOGGER.info("Not generating configuration due to previous successful generation.") - return - - builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter - builder = self.generate_build_variant(builder, build_variant_name) - - generated_config = builder.build(output_file) - generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir) - - with open('gtcl_update_expansions.yml', "w+") as fh: - fh.write(f"gtcl: {self.evg_expansions.config_location()}") - - # pylint: disable=too-many-locals - def generate_build_variant(self, builder: EvgConfigBuilder, - build_variant_name: str) -> EvgConfigBuilder: - """ - Generate task configuration for a build variant. - - :param builder: Evergreen configuration builder to use. - :param build_variant_name: Name of build variant to generate. - :return: Evergreen configuration builder with build variant configuration. - """ - LOGGER.info("Generating config", build_variant=build_variant_name) - start_time = perf_counter() - task_list = self.evg_project_config.get_variant(build_variant_name).task_names - tasks_to_hide = set() - with Executor(max_workers=MAX_WORKERS) as exe: - jobs = [] - for task_name in task_list: - task_def = self.evg_project_config.get_task(task_name) - if task_def.is_generate_resmoke_task: - tasks_to_hide.add(task_name) - - run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) - requires_npm = run_vars.get("is_jstestfuzz", False) - - if requires_npm: - fuzzer_params = self.task_def_to_fuzzer_params(task_def, build_variant_name) - jobs.append(exe.submit(builder.generate_fuzzer, fuzzer_params)) - else: - split_params = self.task_def_to_split_params(task_def, build_variant_name) - gen_params = self.task_def_to_gen_params(task_def, build_variant_name) - jobs.append(exe.submit(builder.generate_suite, split_params, gen_params)) - - [j.result() for j in jobs] # pylint: disable=expression-not-assigned - - end_time = perf_counter() - duration = end_time - start_time - - LOGGER.info("Finished BV", build_variant=build_variant_name, duration=duration, - task_count=len(tasks_to_hide)) - - builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide, build_variant_name) - self.adjust_gen_tasks_priority(tasks_to_hide) - return builder - - def adjust_task_priority(self, task: EvgTask) -> None: - """ - Increase the priority of the given task by 1. - - :param task: Task to increase priority of. - """ - priority = min(task.priority + 1, MAX_TASK_PRIORITY) - LOGGER.info("Configure task", task_id=task.task_id, priority=priority) - self.evg_api.configure_task(task.task_id, priority=priority) - - @classmethod - def _should_adjust_task_priority(cls, task, gen_tasks): - if task.display_name in gen_tasks: - return True - # Test out the effect of Evergreen capacity constraints. - if task.build_variant.endswith("-query-patch-only"): - return True - return False - - def adjust_gen_tasks_priority(self, gen_tasks: Set[str]) -> int: - """ - Increase the priority of any "_gen" tasks. - - We want to minimize the time it tasks for the "_gen" tasks to activate the generated - sub-tasks. We will do that by increase the priority of the "_gen" tasks. - - :param gen_tasks: Set of "_gen" tasks that were found. - """ - build = self.evg_api.build_by_id(self.evg_expansions.build_id) - task_list = build.get_tasks() - - with Executor(max_workers=MAX_WORKERS) as exe: - jobs = [ - exe.submit(self.adjust_task_priority, task) for task in task_list - if self._should_adjust_task_priority(task, gen_tasks) - ] - - results = [j.result() for j in jobs] - return len(results) - - -@click.command(context_settings=dict(ignore_unknown_options=True)) -@click.option("--expansion-file", type=str, required=True, - help="Location of expansions file generated by evergreen.") -@click.option("--evg-api-config", type=str, required=True, - help="Location of evergreen api configuration.") -@click.option("--evg-project-config", type=str, default="etc/evergreen.yml", - help="Location of Evergreen project configuration.") -@click.option("--output-file", type=str, help="Name of output file to write.") -@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") -@click.argument('resmoke_run_args', nargs=-1, type=click.UNPROCESSED) -def main(expansion_file: str, evg_api_config: str, evg_project_config: str, output_file: str, - verbose: bool, resmoke_run_args: Tuple[str]) -> None: - """ - Generate task configuration for a build variant. - \f - :param expansion_file: Location of evergreen expansions for task. - :param evg_api_config: Location of file containing evergreen API authentication information. - :param evg_project_config: Location of file containing evergreen project configuration. - :param output_file: Location to write generated configuration to. - :param verbose: Should verbose logging be used. - :param resmoke_run_args: Args to forward to `resmoke.py run`. - """ - enable_logging(verbose) - - end_date = datetime.utcnow().replace(microsecond=0) - start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS) - - evg_expansions = EvgExpansions.from_yaml_file(expansion_file) - - # pylint: disable=no-value-for-parameter - def dependencies(binder: inject.Binder) -> None: - binder.bind(EvgExpansions, evg_expansions) - binder.bind(SuiteSplitConfig, evg_expansions.build_suite_split_config(start_date, end_date)) - binder.bind(SplitStrategy, greedy_division) - binder.bind(FallbackStrategy, round_robin_fallback) - binder.bind(GenTaskOptions, evg_expansions.build_evg_config_gen_options()) - binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evg_api_config)) - binder.bind(EvergreenProjectConfig, parse_evergreen_file(evg_project_config)) - binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyService, ResmokeProxyService(" ".join(resmoke_run_args))) - - inject.configure(dependencies) - - orchestrator = GenerateBuildVariantOrchestrator() # pylint: disable=no-value-for-parameter - start_time = perf_counter() - orchestrator.generate(evg_expansions.task_id, evg_expansions.build_variant, output_file) - end_time = perf_counter() - - LOGGER.info("Total runtime", duration=end_time - start_time) - - -if __name__ == '__main__': - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_resmoke_job_count.py b/buildscripts/evergreen_resmoke_job_count.py index 438648dea89..4066c6f8f92 100644 --- a/buildscripts/evergreen_resmoke_job_count.py +++ b/buildscripts/evergreen_resmoke_job_count.py @@ -30,8 +30,8 @@ SYS_PLATFORM = sys.platform # Apply factor for a task based on the build variant it is running on. VARIANT_TASK_FACTOR_OVERRIDES = { - "enterprise-rhel-80-64-bit": [{"task": r"logical_session_cache_replication.*", "factor": 0.75}], - "enterprise-rhel-80-64-bit-inmem": [ + "enterprise-rhel-8-64-bit": [{"task": r"logical_session_cache_replication.*", "factor": 0.75}], + "enterprise-rhel-8-64-bit-inmem": [ {"task": "secondary_reads_passthrough", "factor": 0.3}, {"task": "multi_stmt_txn_jscore_passthrough_with_migration", "factor": 0.3}, ] diff --git a/buildscripts/idl/idl/ast.py b/buildscripts/idl/idl/ast.py index d86ccbbb257..c81a8d26c0a 100644 --- a/buildscripts/idl/idl/ast.py +++ b/buildscripts/idl/idl/ast.py @@ -34,6 +34,7 @@ This is a lossy translation from the IDL Syntax tree as the IDL AST only contain the enums and structs that need code generated for them, and just enough information to do that. """ from abc import ABCMeta, abstractmethod +import enum from typing import Any, Dict, List, Optional from . import common, errors @@ -113,6 +114,9 @@ class Type(common.SourceLocation): # A variant can have at most one alternative type which is a struct. Otherwise, if we saw # a sub-object while parsing BSON, we wouldn't know which struct to interpret it as. self.variant_struct_type = None # type: Type + # Marks whether this type is a query shape component. + # Can only be true if is_struct is true. + self.is_query_shape_component = False # type: bool super(Type, self).__init__(file_name, line, column) @@ -144,6 +148,9 @@ class Struct(common.SourceLocation): self.allow_global_collection_name = False # type: bool self.non_const_getter = False # type: bool self.cpp_validator_func = None # type: str + + # Determines whether or not this IDL struct can be a component of a query shape. See WRITING-13831. + self.query_shape_component = False # type: bool super(Struct, self).__init__(file_name, line, column) @@ -184,6 +191,34 @@ class Validator(common.SourceLocation): super(Validator, self).__init__(file_name, line, column) +@enum.unique +class QueryShapeFieldType(enum.Enum): + """Enum describing how to treat a field in the context of query shape computation.""" + + # Abstract literal from shape. + LITERAL = enum.auto() + # Leave value as-is in shape. + PARAMETER = enum.auto() + # Anonymize string value. + ANONYMIZE = enum.auto() + # IDL type uses custom serializer -- defer to that serializer. + CUSTOM = enum.auto() + + @classmethod + def bind(cls, string_value): + # type: (Optional[str]) -> Optional[QueryShapeFieldType] + """Parses the string to the enum type.""" + if string_value is None: + return None + bindings = { + "literal": cls.LITERAL, + "parameter": cls.PARAMETER, + "anonymize": cls.ANONYMIZE, + "custom": cls.CUSTOM, + } + return bindings.get(string_value, None) + + class Field(common.SourceLocation): """ An instance of a field in a struct. @@ -223,8 +258,26 @@ class Field(common.SourceLocation): # Validation rules. self.validator = None # type: Optional[Validator] + # Determines whether or not this field represents a literal value that should be abstracted when serializing a query shape. + # See WRITING-13831 for details on query shape. + self.query_shape = None # type: Optional[QueryShapeFieldType] + super(Field, self).__init__(file_name, line, column) + @property + def should_serialize_with_options(self): + # type: () -> bool + """Returns true if the IDL compiler should add a call to serialization options for this field.""" + return self.query_shape is not None and self.query_shape in [ + QueryShapeFieldType.LITERAL, QueryShapeFieldType.ANONYMIZE + ] + + @property + def should_shapify(self): + # type: () -> bool + """Returns true if the IDL compiler should treat this field as a query literal.""" + return self.query_shape is not None and self.query_shape != QueryShapeFieldType.PARAMETER + class Privilege(common.SourceLocation): """IDL privilege information.""" diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py index 1511b6c1b07..853088cede6 100644 --- a/buildscripts/idl/idl/binder.py +++ b/buildscripts/idl/idl/binder.py @@ -271,6 +271,7 @@ def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): ast_struct.qualified_cpp_name = _get_struct_qualified_cpp_name(struct) ast_struct.allow_global_collection_name = struct.allow_global_collection_name ast_struct.non_const_getter = struct.non_const_getter + ast_struct.query_shape_component = struct.query_shape_component # Validate naming restrictions if ast_struct.name.startswith("array<"): @@ -307,6 +308,20 @@ def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): if not _is_duplicate_field(ctxt, ast_struct.name, ast_struct.fields, ast_field): ast_struct.fields.append(ast_field) + # Verify that each field on the struct defines a query shape type on the field if and only if + # query_shape_component is defined on the struct. + if not field.hidden and struct.query_shape_component and ast_field.query_shape is None: + ctxt.add_must_declare_shape_type(ast_field, ast_struct.name, ast_field.name) + + if not struct.query_shape_component and ast_field.query_shape is not None: + ctxt.add_must_be_query_shape_component(ast_field, ast_struct.name, ast_field.name) + + if ast_field.query_shape == ast.QueryShapeFieldType.ANONYMIZE and not ( + ast_field.type.cpp_type in ["std::string", "std::vector<std::string>"] + or 'string' in ast_field.type.bson_serialization_type): + ctxt.add_query_shape_anonymize_must_be_string(ast_field, ast_field.name, + ast_field.type.cpp_type) + # Fill out the field comparison_order property as needed if ast_struct.generate_comparison_operators and ast_struct.fields: # If the user did not specify an ordering of fields, then number all fields in @@ -420,6 +435,7 @@ def _bind_struct_type(struct): ast_type.name = struct.name ast_type.cpp_type = _get_struct_qualified_cpp_name(struct) ast_type.bson_serialization_type = ["object"] + ast_type.is_query_shape_component = struct.query_shape_component return ast_type @@ -969,6 +985,7 @@ def _bind_type(idltype): ast_type.bindata_subtype = idltype.bindata_subtype ast_type.serializer = _normalize_method_name(idltype.cpp_type, idltype.serializer) ast_type.deserializer = _normalize_method_name(idltype.cpp_type, idltype.deserializer) + ast_type.is_query_shape_component = True return ast_type @@ -993,6 +1010,11 @@ def _bind_field(ctxt, parsed_spec, field): ast_field.unstable = field.unstable ast_field.always_serialize = field.always_serialize + if field.query_shape is not None: + ast_field.query_shape = ast.QueryShapeFieldType.bind(field.query_shape) + if ast_field.query_shape is None: + ctxt.add_invalid_query_shape_value(ast_field, field.query_shape) + ast_field.cpp_name = field.name if field.cpp_name: ast_field.cpp_name = field.cpp_name @@ -1073,6 +1095,8 @@ def _bind_field(ctxt, parsed_spec, field): if ast_field.validator is None: return None + if ast_field.should_shapify and not ast_field.type.is_query_shape_component: + ctxt.add_must_be_query_shape_component(ast_field, ast_field.type.name, ast_field.name) return ast_field diff --git a/buildscripts/idl/idl/bson.py b/buildscripts/idl/idl/bson.py index 8216b5d743d..c7ead4cd66e 100644 --- a/buildscripts/idl/idl/bson.py +++ b/buildscripts/idl/idl/bson.py @@ -73,6 +73,7 @@ _BINDATA_SUBTYPE = { "uuid": {'scalar': True, 'bindata_enum': 'newUUID'}, "md5": {'scalar': True, 'bindata_enum': 'MD5Type'}, "encrypt": {'scalar': True, 'bindata_enum': 'Encrypt'}, + "sensitive": {'scalar': True, 'bindata_enum': 'Sensitive'}, } diff --git a/buildscripts/idl/idl/cpp_types.py b/buildscripts/idl/idl/cpp_types.py index d8e37dfcc56..ba3a0b6bce6 100644 --- a/buildscripts/idl/idl/cpp_types.py +++ b/buildscripts/idl/idl/cpp_types.py @@ -577,14 +577,14 @@ class BsonCppTypeBase(object, metaclass=ABCMeta): pass @abstractmethod - def gen_serializer_expression(self, indented_writer, expression): - # type: (writer.IndentedTextWriter, str) -> str + def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): + # type: (writer.IndentedTextWriter, str, bool) -> str """Generate code with the text writer and return an expression to serialize the type.""" pass -def _call_method_or_global_function(expression, method_name): - # type: (str, str) -> str +def _call_method_or_global_function(expression, method_name, should_shapify=False): + # type: (str, str, bool) -> str """ Given a fully-qualified method name, call it correctly. @@ -592,13 +592,19 @@ def _call_method_or_global_function(expression, method_name): not treated as a global C++ function though. This notion of functions is designed to support enum deserializers/serializers which are not methods. """ + shape_options = '' + if should_shapify: + shape_options = 'options' + short_method_name = writer.get_method_name(method_name) if writer.is_function(method_name): - return common.template_args('${method_name}(${expression})', expression=expression, - method_name=method_name) + return common.template_args('${method_name}(${expression}${shape_options})', + expression=expression, method_name=method_name, + shape_options=shape_options) - return common.template_args('${expression}.${method_name}()', expression=expression, - method_name=short_method_name) + return common.template_args('${expression}.${method_name}(${shape_options})', + expression=expression, method_name=short_method_name, + shape_options=shape_options) class _CommonBsonCppTypeBase(BsonCppTypeBase): @@ -619,9 +625,10 @@ class _CommonBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression): - # type: (writer.IndentedTextWriter, str) -> str - return _call_method_or_global_function(expression, self._ast_type.serializer) + def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): + # type: (writer.IndentedTextWriter, str, bool) -> str + return _call_method_or_global_function(expression, self._ast_type.serializer, + should_shapify) class _ObjectBsonCppTypeBase(BsonCppTypeBase): @@ -643,12 +650,19 @@ class _ObjectBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression): - # type: (writer.IndentedTextWriter, str) -> str + def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): + # type: (writer.IndentedTextWriter, str, bool) -> str method_name = writer.get_method_name(self._ast_type.serializer) + function_arguments = [] + # Provide options if custom shapification required. + if should_shapify: + function_arguments.append('options') + indented_writer.write_line( - common.template_args('const BSONObj localObject = ${expression}.${method_name}();', - expression=expression, method_name=method_name)) + common.template_args( + 'const BSONObj localObject = ${expression}.${method_name}(${function_arguments});', + expression=expression, method_name=method_name, + function_arguments=', '.join(function_arguments))) return "localObject" @@ -671,8 +685,8 @@ class _ArrayBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression): - # type: (writer.IndentedTextWriter, str) -> str + def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): + # type: (writer.IndentedTextWriter, str, bool) -> str method_name = writer.get_method_name(self._ast_type.serializer) indented_writer.write_line( common.template_args('BSONArray localArray(${expression}.${method_name}());', @@ -695,8 +709,8 @@ class _BinDataBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return True - def gen_serializer_expression(self, indented_writer, expression): - # type: (writer.IndentedTextWriter, str) -> str + def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): + # type: (writer.IndentedTextWriter, str, bool) -> str if self._ast_type.serializer: method_name = writer.get_method_name(self._ast_type.serializer) indented_writer.write_line( diff --git a/buildscripts/idl/idl/errors.py b/buildscripts/idl/idl/errors.py index 47cd8ecf3f4..0f030cf6573 100644 --- a/buildscripts/idl/idl/errors.py +++ b/buildscripts/idl/idl/errors.py @@ -128,6 +128,12 @@ ERROR_ID_DUPLICATE_ACCESS_CHECK = "ID0087" ERROR_ID_DUPLICATE_PRIVILEGE = "ID0088" ERROR_ID_EMPTY_ACCESS_CHECK = "ID0089" ERROR_ID_MISSING_ACCESS_CHECK = "ID0090" +ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL = "ID0094" +ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL = "ID0095" +ERROR_ID_INVALID_TYPE_FOR_SHAPIFY = "ID0096" +ERROR_ID_QUERY_SHAPE_PROPERTIES_MUTUALLY_EXCLUSIVE = "ID0097" +ERROR_ID_QUERY_SHAPE_PROPERTY_CANNOT_BE_FALSE = "ID0098" +ERROR_ID_QUERY_SHAPE_INVALID_VALUE = "ID0102" class IDLError(Exception): @@ -369,6 +375,15 @@ class ParserContext(object): return True return False + def get_required_bool(self, node): + # type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> bool + """Get a YAML bool and enforce it is specified.""" + boolean_value = yaml.safe_load(node.value) + if not isinstance(boolean_value, bool): + self._add_node_error(node, ERROR_ID_IS_NODE_VALID_BOOL, + "Illegal bool value, expected either 'true' or 'false'.") + return boolean_value + def get_list(self, node): # type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> List[str] """Get a YAML scalar or sequence node as a list of strings.""" @@ -967,6 +982,34 @@ class ParserContext(object): self._add_error(location, ERROR_ID_MISSING_ACCESS_CHECK, 'Command "%s" has api_version != "" but is missing access_check.' % (name)) + def add_must_declare_shape_type(self, location, struct_name, field_name): + # type: (common.SourceLocation, str, str) -> None + """Add an error about a field not specifying either query_shape_literal or query_shape_anonymize if the struct is query_shape_component.""" + self._add_error( + location, ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL, + f"Field '{field_name}' must specify either 'query_shape_literal' or 'query_shape_anonymize' since struct '{struct_name}' is a query shape component." + ) + + def add_must_be_query_shape_component(self, location, struct_name, field_name): + # type: (common.SourceLocation, str, str) -> None + """Add an error about specifying 'query_shape_literal' without being a shape component.""" + self._add_error( + location, ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL, + f"Field '{field_name}' cannot specify 'query_shape_literal' property since struct '{struct_name}' is not a query shape component." + ) + + def add_query_shape_anonymize_must_be_string(self, location, field_name, field_type): + """Add an error about field paths needing to be strings.""" + self._add_error( + location, ERROR_ID_INVALID_TYPE_FOR_SHAPIFY, + f"In order for {field_name} to be marked as a query shape fieldpath, it must have a string type, not {field_type}." + ) + + def add_invalid_query_shape_value(self, location, query_shape_value): + """Assert that the query shape value is one of the accepted values.""" + self._add_error(location, ERROR_ID_QUERY_SHAPE_INVALID_VALUE, + f"'{query_shape_value}' is not a valid value for 'query_shape'.") + def _assert_unique_error_messages(): # type: () -> None diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py index d8b38d51a0c..d3c62204623 100644 --- a/buildscripts/idl/idl/generator.py +++ b/buildscripts/idl/idl/generator.py @@ -1032,6 +1032,10 @@ class _CppHeaderFileWriter(_CppFileWriterBase): if any(command.api_version for command in spec.commands): header_list.append('mongo/db/commands.h') + # Include serialization options only if there is a struct which is part of a query shape. + if any(struct.query_shape_component for struct in spec.structs): + header_list.append('mongo/db/query/query_shape/serialization_options.h') + header_list.sort() for include in header_list: @@ -1875,6 +1879,23 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._gen_command_deserializer(struct, "request.body") + def _gen_single_field_serialize_expression(self, template_params, field, bson_cpp_type): + """Helper to append the code to serialize a single field, as part of a custom type.""" + expression = bson_cpp_type.gen_serializer_expression( + self._writer, _access_member(field), + field.query_shape == ast.QueryShapeFieldType.CUSTOM) + template_params['expression'] = expression + if not field.should_serialize_with_options: + self._writer.write_template('builder->append(${field_name}, ${expression});') + elif field.query_shape == ast.QueryShapeFieldType.LITERAL: + self._writer.write_template( + 'options.serializeLiteral(${expression}).serializeForIDL(${field_name}, builder);') + else: + assert field.query_shape == ast.QueryShapeFieldType.ANONYMIZE + self._writer.write_template( + 'builder->append(${field_name}, options.serializeFieldPathFromString(${expression}));' + ) + def _gen_serializer_method_custom(self, field): # type: (ast.Field) -> None """Generate the serialize method definition for a custom type.""" @@ -1895,14 +1916,14 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_template( 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') with self._block('for (const auto& item : ${access_member}) {', '}'): - expression = bson_cpp_type.gen_serializer_expression(self._writer, 'item') + expression = bson_cpp_type.gen_serializer_expression( + self._writer, 'item', + field.query_shape == ast.QueryShapeFieldType.CUSTOM) template_params['expression'] = expression self._writer.write_template('arrayBuilder.append(${expression});') else: - expression = bson_cpp_type.gen_serializer_expression( - self._writer, _access_member(field)) - template_params['expression'] = expression - self._writer.write_template('builder->append(${field_name}, ${expression});') + self._gen_single_field_serialize_expression(template_params, field, + bson_cpp_type) elif field.type.bson_serialization_type[0] == 'any': # Any types are special @@ -1918,14 +1939,17 @@ class _CppSourceFileWriter(_CppFileWriterBase): # Call a method like class::method(BSONArrayBuilder*) self._writer.write_template('item.${method_name}(&arrayBuilder);') else: + template_params[ + 'query_shape_options'] = ', options' if field.query_shape == ast.QueryShapeFieldType.CUSTOM else '' if writer.is_function(field.type.serializer): - # Call a method like method(value, StringData, BSONObjBuilder*) self._writer.write_template( - '${method_name}(${access_member}, ${field_name}, builder);') + '${method_name}(${access_member}, ${field_name}, builder${query_shape_options});' + ) else: - # Call a method like class::method(StringData, BSONObjBuilder*) + # Call a method like class::method(StringData, BSONObjBuilder*, SerializationOptions) self._writer.write_template( - '${access_member}.${method_name}(${field_name}, builder);') + '${access_member}.${method_name}(${field_name}, builder${query_shape_options});' + ) else: method_name = writer.get_method_name(field.type.serializer) @@ -1960,18 +1984,29 @@ class _CppSourceFileWriter(_CppFileWriterBase): if field.chained: # Just directly call the serializer for chained structs without opening up a nested # document. - self._writer.write_template('${access_member}.serialize(builder);') + if not field.should_serialize_with_options: + self._writer.write_template('${access_member}.serialize(builder);') + else: + self._writer.write_template('${access_member}.serialize(builder, options);') + elif field.type.is_array: self._writer.write_template( 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') with self._block('for (const auto& item : ${access_member}) {', '}'): self._writer.write_line( 'BSONObjBuilder subObjBuilder(arrayBuilder.subobjStart());') - self._writer.write_line('item.serialize(&subObjBuilder);') + if not field.should_serialize_with_options: + self._writer.write_line('item.serialize(&subObjBuilder);') + else: + self._writer.write_line('item.serialize(&subObjBuilder, options);') else: self._writer.write_template( 'BSONObjBuilder subObjBuilder(builder->subobjStart(${field_name}));') - self._writer.write_template('${access_member}.serialize(&subObjBuilder);') + if not field.should_serialize_with_options: + self._writer.write_template('${access_member}.serialize(&subObjBuilder);') + else: + self._writer.write_template( + '${access_member}.serialize(&subObjBuilder, options);') def _gen_serializer_method_variant(self, field): # type: (ast.Field) -> None @@ -1990,18 +2025,44 @@ class _CppSourceFileWriter(_CppFileWriterBase): template_params[ 'cpp_type'] = 'std::vector<' + variant_type.cpp_type + '>' if variant_type.is_array else variant_type.cpp_type - with self._block('[builder](const ${cpp_type}& value) {', '},'): + template_params['param_opt'] = "builder" + if field.should_serialize_with_options: + template_params['param_opt'] += ', options' + with self._block('[${param_opt}](const ${cpp_type}& value) {', '},'): bson_cpp_type = cpp_types.get_bson_cpp_type(variant_type) if bson_cpp_type and bson_cpp_type.has_serializer(): assert not field.type.is_array expression = bson_cpp_type.gen_serializer_expression( - self._writer, 'value') + self._writer, 'value', + field.query_shape == ast.QueryShapeFieldType.CUSTOM) template_params['expression'] = expression - self._writer.write_template( - 'builder->append(${field_name}, ${expression});') + if not field.should_serialize_with_options: + self._writer.write_template( + 'builder->append(${field_name}, ${expression});') + elif field.query_shape == ast.QueryShapeFieldType.LITERAL: + self._writer.write_template( + 'options.serializeLiteral(${expression}).serializeForIDL(${field_name}, builder);' + ) + elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: + self._writer.write_template( + 'builder->append(${field_name}, options.serializeFieldPathFromString(${expression}));' + ) + else: + assert False else: - self._writer.write_template( - 'idl::idlSerialize(builder, ${field_name}, value);') + if not field.should_serialize_with_options: + self._writer.write_template( + 'idl::idlSerialize(builder, ${field_name}, value);') + elif field.query_shape == ast.QueryShapeFieldType.LITERAL: + self._writer.write_template( + 'options.serializeLiteral(value).serializeForIDL(${field_name}, builder);' + ) + elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: + self._writer.write_template( + 'idl::idlSerialize(builder, ${field_name}, options.serializeFieldPathFromString(value));' + ) + else: + assert False def _gen_serializer_method_common(self, field): # type: (ast.Field) -> None @@ -2030,11 +2091,27 @@ class _CppSourceFileWriter(_CppFileWriterBase): elif field.type.is_variant: self._gen_serializer_method_variant(field) else: - # Generate default serialization using BSONObjBuilder::append - # Note: BSONObjBuilder::append has overrides for std::vector also - self._writer.write_line( - 'builder->append(%s, %s);' % (_get_field_constant_name(field), - _access_member(field))) + # Generate default serialization + # Note: BSONObjBuilder::append, which all three branches use, has overrides for std::vector also + if not field.should_serialize_with_options: + self._writer.write_line( + 'builder->append(%s, %s);' % (_get_field_constant_name(field), + _access_member(field))) + elif field.query_shape == ast.QueryShapeFieldType.LITERAL: + # serializeLiteral expects an ImplicitValue, which can't be constructed with an int64_t + expression_cast = "" + if field.type.cpp_type == "std::int64_t": + expression_cast = "(long long)" + self._writer.write_line( + 'options.serializeLiteral(%s%s).serializeForIDL(%s, builder);' + % (expression_cast, _access_member(field), + _get_field_constant_name(field))) + elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: + self._writer.write_line( + 'builder->append(%s, options.serializeFieldPathFromString(%s));' % + (_get_field_constant_name(field), _access_member(field))) + else: + assert False else: self._gen_serializer_method_struct(field) diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index 356edf0e265..2582a6626b9 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -134,6 +134,8 @@ def _generic_parser( if ctxt.is_mapping_node(second_node, first_name): syntax_node.__dict__[first_name] = rule_desc.mapping_parser_func( ctxt, second_node) + elif rule_desc.node_type == "required_bool_scalar": + syntax_node.__dict__[first_name] = ctxt.get_required_bool(second_node) else: raise errors.IDLError( "Unknown node_type '%s' for parser rule" % (rule_desc.node_type)) @@ -149,7 +151,7 @@ def _generic_parser( # A bool is never "None" like other types, it simply defaults to "false". # It means "if bool is None" will always return false and there is no support for required - # 'bool' at this time. + # 'bool' at this time. Use the node type 'required_bool_scalar' if this behavior is not desired. if not rule_desc.node_type == 'bool_scalar': if syntax_node.__dict__[name] is None: ctxt.add_missing_required_field_error(node, syntax_node_name, name) @@ -374,6 +376,8 @@ def _parse_field(ctxt, name, node): _RuleDesc("bool_scalar"), "always_serialize": _RuleDesc("bool_scalar"), + "query_shape": + _RuleDesc('scalar'), }) return field @@ -527,6 +531,7 @@ def _parse_struct(ctxt, spec, name, node): "generate_comparison_operators": _RuleDesc("bool_scalar"), "non_const_getter": _RuleDesc('bool_scalar'), "cpp_validator_func": _RuleDesc('scalar'), + "query_shape_component": _RuleDesc('bool_scalar'), }) # PyLint has difficulty with some iterables: https://github.com/PyCQA/pylint/issues/3105 diff --git a/buildscripts/idl/idl/struct_types.py b/buildscripts/idl/idl/struct_types.py index b3b0df60532..7a8d0867a6d 100644 --- a/buildscripts/idl/idl/struct_types.py +++ b/buildscripts/idl/idl/struct_types.py @@ -67,15 +67,27 @@ class ArgumentInfo(object): def __init__(self, arg): # type: (str) -> None """Create a instance of the ArgumentInfo class by parsing the argument string.""" - parts = arg.split(' ') - self.type = ' '.join(parts[0:-1]) - self.name = parts[-1] + self.defaults = None + equal_tokens = arg.split('=') + if len(equal_tokens) > 1: + self.defaults = equal_tokens[-1].strip() + + space_tokens = equal_tokens[0].strip().split(' ') + self.type = ' '.join(space_tokens[0:-1]) + self.name = space_tokens[-1] def __str__(self): # type: () -> str """Return a formatted argument string.""" return "%s %s" % (self.type, self.name) # type: ignore + def get_string(self, get_defaults): + # type: (bool) -> str + """Return a formatted argument string.""" + if self.defaults and get_defaults: + return "%s %s = %s" % (self.type, self.name, self.defaults) # type: ignore + return "%s %s" % (self.type, self.name) # type: ignore + class MethodInfo(object): """Class that encapslates information about a method and how to declare, define, and call it.""" @@ -115,7 +127,8 @@ class MethodInfo(object): return common.template_args( "${pre_modifiers}${return_type}${method_name}(${args})${post_modifiers};", pre_modifiers=pre_modifiers, return_type=return_type_str, method_name=self.method_name, - args=', '.join([str(arg) for arg in self.args]), post_modifiers=post_modifiers) + args=', '.join( + [arg.get_string(True) for arg in self.args]), post_modifiers=post_modifiers) def get_definition(self): # type: () -> str @@ -134,7 +147,7 @@ class MethodInfo(object): "${pre_modifiers}${return_type}${class_name}::${method_name}(${args})${post_modifiers}", pre_modifiers=pre_modifiers, return_type=return_type_str, class_name=self.class_name, method_name=self.method_name, args=', '.join( - [str(arg) for arg in self.args]), post_modifiers=post_modifiers) + [arg.get_string(False) for arg in self.args]), post_modifiers=post_modifiers) def get_call(self, obj): # type: (Optional[str]) -> str @@ -268,14 +281,19 @@ class _StructTypeInfo(StructTypeInfoBase): def get_serializer_method(self): # type: () -> MethodInfo + args = ['BSONObjBuilder* builder'] + if self._struct.query_shape_component: + args.append("const SerializationOptions& options = {}") return MethodInfo( - common.title_case(self._struct.cpp_name), 'serialize', ['BSONObjBuilder* builder'], - 'void', const=True) + common.title_case(self._struct.cpp_name), 'serialize', args, 'void', const=True) def get_to_bson_method(self): # type: () -> MethodInfo + args = [] + if self._struct.query_shape_component: + args.append("const SerializationOptions& options = {}") return MethodInfo( - common.title_case(self._struct.cpp_name), 'toBSON', [], 'BSONObj', const=True) + common.title_case(self._struct.cpp_name), 'toBSON', args, 'BSONObj', const=True) def get_op_msg_request_serializer_method(self): # type: () -> Optional[MethodInfo] diff --git a/buildscripts/idl/idl/syntax.py b/buildscripts/idl/idl/syntax.py index 0cfcf88e73d..27edf420c43 100644 --- a/buildscripts/idl/idl/syntax.py +++ b/buildscripts/idl/idl/syntax.py @@ -471,6 +471,10 @@ class Field(common.SourceLocation): self.serialize_op_msg_request_only = False # type: bool self.constructed = False # type: bool + self.query_shape = None # type: Optional[str] + + self.hidden = False # type: bool + super(Field, self).__init__(file_name, line, column) @@ -541,6 +545,8 @@ class Struct(common.SourceLocation): # Internal property: cpp_namespace from globals section self.cpp_namespace = None # type: str + self.query_shape_component = False # type: bool + super(Struct, self).__init__(file_name, line, column) diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py index 64fd3541bc7..7b3ad4bee94 100644 --- a/buildscripts/idl/idl_check_compatibility.py +++ b/buildscripts/idl/idl_check_compatibility.py @@ -184,6 +184,12 @@ IGNORE_UNSTABLE_LIST: List[str] = [ # The 'runtimeConstants' field is a legacy field for internal use only and is not documented to # users. 'delete-param-runtimeConstants', + # The 'bypassEmptyTsReplacement' field is used by mongorestore and mongosync and is not + # documented to users. + 'insert-param-bypassEmptyTsReplacement', + 'update-param-bypassEmptyTsReplacement', + 'delete-param-bypassEmptyTsReplacement', + 'findAndModify-param-bypassEmptyTsReplacement', ] SKIPPED_FILES = [ diff --git a/buildscripts/idl/tests/test_binder.py b/buildscripts/idl/tests/test_binder.py index b52c755e34b..32f42772e8d 100644 --- a/buildscripts/idl/tests/test_binder.py +++ b/buildscripts/idl/tests/test_binder.py @@ -71,6 +71,39 @@ class TestBinder(testcase.IDLTestcase): # pylint: disable=too-many-public-methods + # Create a text wrap for common types. + common_types = textwrap.dedent(""" + types: + object: + description: foo + cpp_type: foo + bson_serialization_type: object + serializer: foo + deserializer: foo + + bool: + description: foo + cpp_type: foo + bson_serialization_type: any + serializer: foo + deserializer: foo + + string: + description: foo + cpp_type: foo + bson_serialization_type: string + serializer: foo + deserializer: foo + + any_type: + description: foo + cpp_type: foo + bson_serialization_type: any + serializer: foo + deserializer: foo + + """) + def test_empty(self): # type: () -> None """Test an empty document works.""" @@ -2742,6 +2775,172 @@ class TestBinder(testcase.IDLTestcase): reply_type: reply """), idl.errors.ERROR_ID_MISSING_ACCESS_CHECK) + def test_query_shape_component_validation(self): + """Tests for the query shape component fields.""" + self.assert_bind(self.common_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + query_shape: literal + type: string + field2: + type: bool + query_shape: parameter + """)) + + self.assert_bind_fail( + self.common_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + type: string + field2: + type: bool + query_shape: parameter + """), idl.errors.ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL) + + self.assert_bind_fail( + self.common_types + textwrap.dedent(""" + structs: + struct1: + strict: true + description: "" + fields: + field1: + type: string + field2: + type: bool + query_shape: parameter + """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) + + # Validating query_shape_anonymize relies on std::string + basic_types = textwrap.dedent(""" + types: + string: + bson_serialization_type: string + description: "A BSON UTF-8 string" + cpp_type: "std::string" + deserializer: "mongo::BSONElement::str" + bool: + bson_serialization_type: bool + description: "A BSON bool" + cpp_type: "bool" + deserializer: "mongo::BSONElement::boolean" + """) + self.assert_bind(basic_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + query_shape: anonymize + type: string + field2: + query_shape: parameter + type: bool + """)) + + self.assert_bind(basic_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + query_shape: anonymize + type: array<string> + field2: + query_shape: parameter + type: bool + """)) + + self.assert_bind_fail( + basic_types + textwrap.dedent(""" + structs: + struct1: + strict: true + description: "" + fields: + field1: + query_shape: blah + type: string + """), idl.errors.ERROR_ID_QUERY_SHAPE_INVALID_VALUE) + + self.assert_bind_fail( + basic_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + query_shape: anonymize + type: bool + field2: + query_shape: parameter + type: bool + """), idl.errors.ERROR_ID_INVALID_TYPE_FOR_SHAPIFY) + + self.assert_bind_fail( + basic_types + textwrap.dedent(""" + structs: + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field1: + query_shape: anonymize + type: array<bool> + field2: + query_shape: parameter + type: bool + """), idl.errors.ERROR_ID_INVALID_TYPE_FOR_SHAPIFY) + + self.assert_bind_fail( + basic_types + textwrap.dedent(""" + structs: + StructZero: + strict: true + description: "" + fields: + field1: + query_shape: literal + type: string + """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) + + self.assert_bind_fail( + basic_types + textwrap.dedent(""" + structs: + StructZero: + strict: true + description: "" + fields: + field1: + type: string + struct1: + query_shape_component: true + strict: true + description: "" + fields: + field2: + type: StructZero + description: "" + query_shape: literal + """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) + if __name__ == '__main__': diff --git a/buildscripts/idl/tests/test_generator.py b/buildscripts/idl/tests/test_generator.py index ee8a251aa53..97020f1eb8e 100644 --- a/buildscripts/idl/tests/test_generator.py +++ b/buildscripts/idl/tests/test_generator.py @@ -38,6 +38,7 @@ $ coverage run run_tests.py && coverage html import os import unittest +from textwrap import dedent # import package so that it works regardless of whether we run as a module or file if __package__ is None: @@ -132,6 +133,79 @@ class TestGenerator(testcase.IDLTestcase): self.assertTrue(found, "Bad Header: " + header) + def test_object_with_custom_serializer_and_query_shape(self) -> None: + """Serialization with custom query_shape.""" + _, source = self.assert_generate(""" + types: + object_type_with_custom_serializer: + bson_serialization_type: object + description: ObjWithCustomSerializer + cpp_type: ObjWithCustomSerializer + serializer: ObjWithCustomSerializer::toBSON + deserializer: ObjWithCustomSerializer::parse + + structs: + QueryShapeSpec: + description: QueryShape + query_shape_component: true + fields: + internalObject: + type: object_type_with_custom_serializer + optional: false + description: internalObject + query_shape: custom + """) + + expected = dedent(""" + void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const { + invariant(_hasInternalObject); + + { + const BSONObj localObject = _internalObject.toBSON(options); + builder->append(kInternalObjectFieldName, localObject); + } + + }""") + self.assertIn(expected, source) + + def test_array_with_custom_serializer_and_query_shape(self) -> None: + """Serialization with custom query_shape used, array use case.""" + _, source = self.assert_generate(""" + types: + object_type_with_custom_serializer: + bson_serialization_type: object + description: ObjWithCustomSerializer + cpp_type: ObjWithCustomSerializer + serializer: ObjWithCustomSerializer::toBSON + deserializer: ObjWithCustomSerializer::parse + + structs: + QueryShapeSpec: + description: QueryShape + query_shape_component: true + fields: + internalObjectArray: + type: array<object_type_with_custom_serializer> + optional: false + description: internalObjectArray + query_shape: custom + """) + + expected = dedent(""" + void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const { + invariant(_hasInternalObjectArray); + + { + BSONArrayBuilder arrayBuilder(builder->subarrayStart(kInternalObjectArrayFieldName)); + for (const auto& item : _internalObjectArray) { + const BSONObj localObject = item.toBSON(options); + arrayBuilder.append(localObject); + } + } + + }""") + self.assertIn(expected, source) + if __name__ == '__main__': diff --git a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml index 36aeaad2531..f87c2c710cf 100644 --- a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml +++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml @@ -110,24 +110,24 @@ evergreen_buildvariants: platform: rhel70 architecture: x86_64 - - name: rhel80 + - name: rhel8 edition: targeted - platform: rhel80 + platform: rhel8 architecture: x86_64 - - name: enterprise-rhel-80-64-bit + - name: enterprise-rhel-8-64-bit edition: enterprise - platform: rhel80 + platform: rhel8 architecture: x86_64 - - name: rhel-82-arm64 + - name: rhel-8-arm64 edition: targeted - platform: rhel82 + platform: rhel8 architecture: arm64 - - name: enterprise-rhel-82-arm64 + - name: enterprise-rhel-8-arm64 edition: enterprise - platform: rhel82 + platform: rhel8 architecture: arm64 - name: enterprise-rhel-71-ppc64le diff --git a/buildscripts/resmokeconfig/suites/benchmarks.yml b/buildscripts/resmokeconfig/suites/benchmarks.yml index 9fe4e74f0ff..011b16bd01a 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks.yml @@ -18,6 +18,7 @@ selector: - build/install/bin/simple8b_bm* # Hash table benchmark is really slow, don't run on evergreen - build/install/bin/hash_table_bm* + - build/install/bin/rate_limiting_bm* # These benchmarks are only run when modifying or upgrading the immutable library. - build/install/bin/immutable_absl_comparison_bm* - build/install/bin/immutable_std_comparison_bm* diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml index 871c7f72c30..115249a7966 100644 --- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml +++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml @@ -10,6 +10,7 @@ selector: - buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3. - buildscripts/tests/resmokelib/powercycle/test_remote_operations.py # Requires ssh to be enabled locally. - buildscripts/tests/resmoke_end2end/**/test_*.py # Requires compile task. Test run in resmoke_end2end_tests.yml instead. + - buildscripts/tests/resmoke_validation/**/test_*.py # Ran in commit queue in resmoke_validation_tests executor: {} diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml index 42bc4a7be4d..65cb93e7f5d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml @@ -54,6 +54,7 @@ selector: - catches_command_failures # time-series collections do not support write transactions - requires_timeseries + - does_not_support_transactions executor: archive: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index d156fb2da8b..98ee4120781 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -110,6 +110,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/remove_multiple_documents.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml index aa1599a3d1e..dfcb47c9199 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml @@ -178,6 +178,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_ordered_bulk.js - jstests/concurrency/fsm_workloads/indexed_insert_unordered_bulk.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js # Uses non-retryable commands in the same state function as a command not supported in a # transaction. diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml index cc53c9d8b91..ca02cc7d9e8 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml @@ -178,6 +178,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_ordered_bulk.js - jstests/concurrency/fsm_workloads/indexed_insert_unordered_bulk.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js # Uses non-retryable commands in the same state function as a command not supported in a # transaction. diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index d4e5410f904..caeb219f6f1 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -110,6 +110,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/remove_multiple_documents.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index f075f53951b..420b427f42a 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -96,6 +96,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index 14b88e5170e..9c3635e7b78 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -102,6 +102,7 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js + - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js diff --git a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml index b40e2783d5c..fca4ed4e214 100755 --- a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml @@ -307,7 +307,6 @@ selector: - jstests/core/not1.js - jstests/core/not2.js - jstests/core/not3.js - - jstests/core/notablescan.js - jstests/core/null_query_semantics.js - jstests/core/null_query_semantics.js - jstests/core/objectfind.js diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz.yml b/buildscripts/resmokeconfig/suites/jstestfuzz.yml index f70bfae03e6..45789c08ab8 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz.yml @@ -13,6 +13,8 @@ executor: crashOnInvalidBSONError: "" objcheck: "" hooks: + - class: RunQueryStats + allow_feature_not_supported: true - class: FuzzerRestoreSettings - class: ValidateCollections shell_options: @@ -26,4 +28,6 @@ executor: set_parameters: disableLogicalSessionCacheRefresh: false enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true verbose: '' diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml index f1ffdb2f014..4a872221272 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml @@ -19,6 +19,8 @@ executor: # Other fuzzers test commands against replica sets with logical session ids. disableImplicitSessions: true hooks: + - class: RunQueryStats + allow_feature_not_supported: true - class: FuzzerRestoreSettings # The CheckReplDBHash hook waits until all operations have replicated to and have been applied # on the secondaries, so we run the ValidateCollections hook after it to ensure we're @@ -42,5 +44,7 @@ executor: enableTestCommands: 1 transactionLifetimeLimitSeconds: 1 writePeriodicNoops: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true verbose: '' num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index 7ddb9581092..de2863d100a 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -18,6 +18,8 @@ executor: crashOnInvalidBSONError: "" objcheck: "" hooks: + - class: RunQueryStats + allow_feature_not_supported: true - class: FuzzerRestoreSettings - class: CheckReplDBHash shell_options: @@ -34,6 +36,8 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true verbose: '' mongod_options: set_parameters: @@ -41,6 +45,8 @@ executor: enableTestCommands: 1 transactionLifetimeLimitSeconds: 1 writePeriodicNoops: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true verbose: '' num_rs_nodes_per_shard: 1 num_shards: 2 diff --git a/buildscripts/resmokeconfig/suites/mqlrun.yml b/buildscripts/resmokeconfig/suites/mqlrun.yml deleted file mode 100644 index d9f75ff7697..00000000000 --- a/buildscripts/resmokeconfig/suites/mqlrun.yml +++ /dev/null @@ -1,12 +0,0 @@ -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/mqlrun/*.js - -# mqlrun tests don't have a test fixture. Instead, they use the shell to spawn an mqlrun subprocess. -executor: - archive: - config: - shell_options: - nodb: '' diff --git a/buildscripts/resmokeconfig/suites/multiversion.yml b/buildscripts/resmokeconfig/suites/multiversion.yml index 674aabc6a65..e865729aac1 100644 --- a/buildscripts/resmokeconfig/suites/multiversion.yml +++ b/buildscripts/resmokeconfig/suites/multiversion.yml @@ -31,3 +31,7 @@ executor: config: shell_options: nodb: '' + global_vars: + TestData: + setParameters: + bsonTestValidationVersion: 1 diff --git a/buildscripts/resmokeconfig/suites/multiversion_auth.yml b/buildscripts/resmokeconfig/suites/multiversion_auth.yml index d76515abf50..aa77936c5a2 100644 --- a/buildscripts/resmokeconfig/suites/multiversion_auth.yml +++ b/buildscripts/resmokeconfig/suites/multiversion_auth.yml @@ -37,6 +37,8 @@ executor: shell_options: global_vars: TestData: + setParameters: + bsonTestValidationVersion: 1 auth: true # authMechanism: SCRAM-SHA-256 keyFile: *keyFile diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 7ab97e1f2ff..1fe41d46475 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -1,8 +1,14 @@ test_kind: js_test +description: | + "Passthrough" means running a test against different runtime Cluster + configurations, including topology, runtime flags, fault injections, and other + parameters. Most tests by default are able to run in "passthrough" suites. + NoPassthrough is an exception, where tests here only run in the exact + configuration predefined in the tests themselves. selector: roots: - - jstests/noPassthrough/*.js + - jstests/noPassthrough/**/*.js - src/mongo/db/modules/*/jstests/hot_backups/*.js - src/mongo/db/modules/*/jstests/live_import/*.js - src/mongo/db/modules/*/jstests/no_passthrough/*.js @@ -10,8 +16,10 @@ selector: # Self-tests for the Concurrency testing framework are run as part of this test suite. - jstests/concurrency/*.js exclude_files: + - jstests/noPassthrough/libs/*.js # Disable inmem_full as per SERVER-27014 - jstests/noPassthrough/inmem_full.js + exclude_with_any_tags: # noPassthrough tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/parallel.yml b/buildscripts/resmokeconfig/suites/parallel.yml index 28d13aa2bbc..c4efc723d56 100644 --- a/buildscripts/resmokeconfig/suites/parallel.yml +++ b/buildscripts/resmokeconfig/suites/parallel.yml @@ -1,10 +1,3 @@ -# If the failure here is due to a test unexpected being run, -# it may be due to the parallel suite not honoring feature flag tags. -# If you want to skip such tests in parallel suite, -# please add them to the exclusion list at -# https://github.com/mongodb/mongo/blob/eb75b6ccc62f7c8ea26a57c1b5eb96a41809396a/jstests/libs/parallelTester.js#L149. - - test_kind: js_test selector: diff --git a/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml new file mode 100644 index 00000000000..0a6999fc887 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml @@ -0,0 +1,38 @@ +test_kind: js_test +description: | + This suite enables the collection of query stats metrics on a mongod server, then runs the tests in + aggregation as normal. This should cause each aggregation to compute a query + shape and query stats key, and record in-memory some metrics like execution time and number of + scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of + each test, once with HMAC application enabled and once without. It doesn't assert anything about + the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. + crashing). + +selector: + roots: + - jstests/aggregation/**/*.js + exclude_files: + # TODO SERVER-92403 Re-enable this test. + - jstests/aggregation/sources/setWindowFields/n_accumulators.js + exclude_with_any_tags: + # Running $queryStats will increment these counters which can screw up some test assertions. + - inspects_command_opcounters + +executor: + archive: + hooks: + - ValidateCollections + hooks: + # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. + # That way the fixture restart can't cause any trouble for the other hooks. + - class: RunQueryStats + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: MongoDFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true diff --git a/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml new file mode 100644 index 00000000000..a6c9843879c --- /dev/null +++ b/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml @@ -0,0 +1,138 @@ +test_kind: js_test +description: | + This suite enables the collection of query stats metrics on a mongos server, then runs the tests in + core and aggregation as normal. This should cause each aggregation to compute a query + shape and query stats key, and record in-memory some metrics like execution time and number of + scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of + each test, once with HMAC application enabled and once without. It doesn't assert anything about + the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. + crashing). + +selector: + roots: + - jstests/aggregation/**/*.js + exclude_files: + - jstests/aggregation/extras/*.js + - jstests/aggregation/data/*.js + # TODO: Remove when SERVER-23229 is fixed. + - jstests/aggregation/bugs/groupMissing.js + # Mongos does not support runtimeConstants. + - jstests/aggregation/accumulators/internal_js_reduce_with_scope.js + - jstests/aggregation/expressions/internal_js_emit_with_scope.js + # $unionWith explain output does not check whether the collection is sharded in a sharded + # cluster. + - jstests/aggregation/sources/unionWith/unionWith_explain.js + # TODO SERVER-92403 Re-enable this test. + - jstests/aggregation/sources/setWindowFields/n_accumulators.js + # SERVER-94231 These tests are excluded because query stats changes the error code that + # certain queries return. Specifically, queries that a) end in error and b) pass through + # mongos end up erroring in the query stats makeKey() callback. Query stats propagates the + # error with a different code, which makes the test fail. This behavior only exists pre-8.0, + # so we don't care to address this more robustly. + - jstests/aggregation/bugs/server6290.js + - jstests/aggregation/sources/graphLookup/error.js + - jstests/aggregation/sources/graphLookup/filter.js + - jstests/aggregation/expressions/date_from_parts.js + - jstests/aggregation/expressions/trim.js + - jstests/aggregation/variables/remove_system_variable.js + - jstests/aggregation/accumulators/first_n_last_n.js + - jstests/aggregation/expressions/expression_function.js + - jstests/aggregation/sources/setWindowFields/parse.js + - jstests/aggregation/expressions/day_of_expressions.js + - jstests/aggregation/sources/multiple_unpack_bucket_error.js + - jstests/aggregation/expressions/let.js + - jstests/aggregation/sources/setWindowFields/first.js + - jstests/aggregation/expressions/reduce.js + - jstests/aggregation/bugs/server533.js + - jstests/aggregation/sources/setWindowFields/linear_fill.js + - jstests/aggregation/bugs/server6177.js + - jstests/aggregation/expressions/date_to_string.js + - jstests/aggregation/dbref.js + - jstests/aggregation/sources/setWindowFields/rank.js + - jstests/aggregation/expressions/date_to_parts.js + - jstests/aggregation/expressions/round_trunc.js + - jstests/aggregation/bugs/server6335.js + - jstests/aggregation/sources/match/skip_with_limit.js + - jstests/aggregation/bugs/server6074.js + - jstests/aggregation/bugs/server6190.js + - jstests/aggregation/bugs/server6198.js + - jstests/aggregation/bugs/server11118.js + - jstests/aggregation/expressions/filter.js + - jstests/aggregation/sources/setWindowFields/exp_moving_avg.js + - jstests/aggregation/bugs/upperlower.js + - jstests/aggregation/sources/setWindowFields/last.js + - jstests/aggregation/sources/densify/parse.js + - jstests/aggregation/variables/search_meta.js + - jstests/aggregation/sources/densify/decimal.js + - jstests/aggregation/ifnull.js + - jstests/aggregation/expressions/regex.js + - jstests/aggregation/sources/shred_documents.js + - jstests/aggregation/expressions/date_add_subtract.js + - jstests/aggregation/expressions/in.js + - jstests/aggregation/bugs/match.js + - jstests/aggregation/expressions/split.js + - jstests/aggregation/accumulators/top_bottom_top_n_bottom_n.js + - jstests/aggregation/bugs/cond.js + - jstests/aggregation/bugs/server4589.js + - jstests/aggregation/sources/setWindowFields/time.js + - jstests/aggregation/sources/fill/fill_parse.js + - jstests/aggregation/expressions/sortArray.js + - jstests/aggregation/bugs/server20169.js + - jstests/aggregation/max_subpipeline_depth.js + - jstests/aggregation/expressions/date_trunc.js + - jstests/aggregation/bugs/server20163.js + - jstests/aggregation/sources/setWindowFields/locf.js + - jstests/aggregation/accumulators/min_n_max_n.js + - jstests/aggregation/sources/setWindowFields/comprehensive_parse.js + - jstests/aggregation/sources/unionWith/unionWith_allows_stages.js + - jstests/aggregation/expressions/expression_get_field.js + - jstests/aggregation/expressions/expression_set_field.js + - jstests/aggregation/sources/lookup/lookup_subpipeline.js + - jstests/aggregation/bugs/server11675.js + - jstests/aggregation/accumulators/internal_js_reduce.js + - jstests/aggregation/sources/collStats/count.js + - jstests/aggregation/sources/setWindowFields/shift.js + - jstests/aggregation/expressions/map.js + - jstests/aggregation/sources/setWindowFields/derivative.js + - jstests/aggregation/bugs/server6530.js + - jstests/aggregation/expressions/switch_errors.js + - jstests/aggregation/expressions/date_from_string.js + - jstests/aggregation/sources/setWindowFields/integral.js + - jstests/aggregation/bugs/server6238.js + - jstests/aggregation/expressions/expression_cond.js + - jstests/aggregation/unwind.js + exclude_with_any_tags: + # The following tests start their own ShardingTest or ReplSetTest, respectively. + - requires_sharding + - requires_replication + - assumes_standalone_mongod + - assumes_against_mongod_not_mongos + # system.profile collection doesn't exist on mongos. + - requires_profiling + # Running $queryStats will increment these counters which can screw up some test assertions. + - inspects_command_opcounters + +executor: + archive: + hooks: + - ValidateCollections + hooks: + # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. + # That way the fixture restart can't cause any trouble for the other hooks. + - class: RunQueryStats + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: ShardedClusterFixture + mongos_options: + set_parameters: + enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true + mongod_options: + set_parameters: + enableTestCommands: 1 + num_rs_nodes_per_shard: 1 + enable_sharding: + - test diff --git a/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml new file mode 100644 index 00000000000..77b6e83f02a --- /dev/null +++ b/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml @@ -0,0 +1,94 @@ +test_kind: js_test +description: | + This suite enables the collection of query stats metrics on a mongos server, then runs the tests in + core and aggregation as normal. This should cause each query to compute a query + shape and query stats key, and record in-memory some metrics like execution time and number of + scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of + each test, once with HMAC application enabled and once without. It doesn't assert anything about + the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. + crashing). + +selector: + roots: + - jstests/core/**/*.js + exclude_files: + - jstests/core/txns/**/*.js + # The following tests fail because a certain command or functionality is not supported on + # mongos. This command or functionality is placed in a comment next to the failing test. + - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. + - jstests/core/**/check_shard_index.js # checkShardingIndex. + - jstests/core/**/collection_truncate.js # emptycapped. + - jstests/core/**/compact_keeps_indexes.js # compact. + - jstests/core/**/currentop.js # uses fsync. + - jstests/core/**/dbhash.js # dbhash. + - jstests/core/**/dbhash2.js # dbhash. + - jstests/core/**/fsync.js # uses fsync. + - jstests/core/**/geo_s2cursorlimitskip.js # profiling. + - jstests/core/**/geo_update_btree2.js # notablescan. + - jstests/core/**/index9.js # "local" database. + - jstests/core/**/queryoptimizera.js # "local" database. + - jstests/core/**/stages*.js # stageDebug. + - jstests/core/**/startup_log.js # "local" database. + - jstests/core/**/top.js # top. + # The following tests fail because mongos behaves differently from mongod when testing certain + # functionality. The differences are in a comment next to the failing test. + - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos. + - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). + - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). + - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). + - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. + - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. + - jstests/core/**/or_to_in.js # queryPlanner in different spot in explain() + # The following tests fail because of divergent dropCollection behavior between standalones and + # sharded clusters. These tests expect a second drop command to error, whereas in sharded clusters + # we expect a second drop to return status OK. + - jstests/core/**/explain_upsert.js + # TODO SERVER-94818 Re-enable this test in query stats passthroughs. + # Query stats changes the error code expected by this test due to how it propagates parsing + # errors in test environments. + - jstests/core/json_schema/misc_validation.js + # SERVER-94231 These tests are excluded because query stats changes the error code that + # certain queries return. Specifically, queries that a) end in error and b) pass through + # mongos end up erroring in the query stats makeKey() callback. Query stats propagates the + # error with a different code, which makes the test fail. This behavior only exists pre-8.0, + # so we don't care to address this more robustly. + - jstests/core/list_all_sessions.js + - jstests/core/collation.js + - jstests/core/timeseries/timeseries_explicit_unpack_bucket.js + - jstests/core/sample_rate.js + - jstests/core/sort_with_meta_operator.js + - jstests/core/geonear_key.js + - jstests/core/timeseries/timeseries_streaming_group.js + - jstests/core/timeseries/timeseries_internal_bucket_geo_within.js + exclude_with_any_tags: + - assumes_standalone_mongod + - assumes_against_mongod_not_mongos + # system.profile collection doesn't exist on mongos. + - requires_profiling + # Running $queryStats will increment these counters which can screw up some test assertions. + - inspects_command_opcounters + +executor: + archive: + hooks: + - ValidateCollections + hooks: + # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. + # That way the fixture restart can't cause any trouble for the other hooks. + - class: RunQueryStats + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: ShardedClusterFixture + mongos_options: + set_parameters: + enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true + mongod_options: + set_parameters: + enableTestCommands: 1 + num_rs_nodes_per_shard: 1 + enable_sharding: + - test diff --git a/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml new file mode 100644 index 00000000000..7fa191ce16c --- /dev/null +++ b/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml @@ -0,0 +1,43 @@ +test_kind: js_test +description: | + This suite enables the collection of query stats metrics on a mongod server, then runs the tests in + core and aggregation as normal. This should cause each query to compute a query + shape and query stats key, and record in-memory some metrics like execution time and number of + scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of + each test, once with HMAC application enabled and once without. It doesn't assert anything about + the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. + crashing). + +selector: + roots: + - jstests/core/**/*.js + exclude_files: + # Transactions are not supported on MongoDB standalone nodes, so we do not run these tests. + - jstests/core/txns/**/*.js + - jstests/core/timeseries/timeseries_union_with.js + # TODO SERVER-94818 Re-enable this test in query stats passthroughs. + # Query stats changes the error code expected by this test due to how it propagates parsing + # errors in test environments. + - jstests/core/json_schema/misc_validation.js + exclude_with_any_tags: + # Running $queryStats will increment these counters which can screw up some test assertions. + - inspects_command_opcounters + +executor: + archive: + hooks: + - ValidateCollections + hooks: + # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. + # That way the fixture restart can't cause any trouble for the other hooks. + - class: RunQueryStats + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: MongoDFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + internalQueryStatsRateLimit: -1 + internalQueryStatsErrorsAreCommandFatal: true diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml index b6316ac186a..8408255eea1 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -41,7 +41,6 @@ selector: - jstests/core/json_schema/json_schema.js - jstests/core/mr_bigobject.js - jstests/core/not2.js - - jstests/core/notablescan.js - jstests/core/null_query_semantics.js - jstests/core/or1.js - jstests/core/or2.js diff --git a/buildscripts/resmokeconfig/suites/resmoke_validation_tests.yml b/buildscripts/resmokeconfig/suites/resmoke_validation_tests.yml new file mode 100644 index 00000000000..24b00c874b8 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/resmoke_validation_tests.yml @@ -0,0 +1,8 @@ +test_kind: py_test + +selector: + roots: + - buildscripts/tests/resmoke_validation/**/test_*.py + + +executor: {} diff --git a/buildscripts/resmokeconfig/suites/sasl.yml b/buildscripts/resmokeconfig/suites/sasl.yml index 4d1a49cc95d..d7888475e08 100644 --- a/buildscripts/resmokeconfig/suites/sasl.yml +++ b/buildscripts/resmokeconfig/suites/sasl.yml @@ -3,6 +3,8 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/sasl/*.js + exclude_files: + - src/mongo/db/modules/enterprise/jstests/sasl/sasl_plugins.js # sasl tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml b/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml new file mode 100644 index 00000000000..0c1cf6fc6ce --- /dev/null +++ b/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml @@ -0,0 +1,11 @@ +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/sasl/sasl_plugins.js + +# sasl tests start their own mongod's. +executor: + config: + shell_options: + nodb: "" diff --git a/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml b/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml index 6b6b0e07bdc..fc3cecf4018 100644 --- a/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml +++ b/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml @@ -11,6 +11,11 @@ selector: exclude_files: # Skip any tests that run with auth explicitly. - src/mongo/db/modules/*/jstests/search/auth_list_search_indexes_agg.js + # This test creates a race condition with the network in pinned connections mode: if mongod + # is still waiting on a response from mongot following the getMore, mongod must close the + # connection because it cannot send the killCursor command to mongot while the getMore + # command is on-going. + - src/mongo/db/modules/enterprise/jstests/mongot/mongot_kill_cursors.js executor: config: diff --git a/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml deleted file mode 100644 index 0d7391ad637..00000000000 --- a/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This passthrough runs all sharding JS tests and automatically batches multi-deletes. -test_kind: js_test - -selector: - roots: - - jstests/sharding/**/*.js - exclude_files: - - jstests/sharding/libs/*.js - # TODO (SERVER-64506): these tests use transactions under the hood to modify user roles. - - jstests/sharding/api_params_nontransaction_sharded.js - - jstests/sharding/api_params_nontransaction_unsharded.js - # Expects DELETE stage - - jstests/sharding/query/explain_cmd.js - # TODO: (SERVER-64972): add change stream support for batched deletes. - - jstests/sharding/change_stream_no_orphans.js - - exclude_with_any_tags: - - assumes_standalone_mongod - # TODO (SERVER-64506): make groupOplogEntries WUOW's nestable (e.g. inside multi-doc txns). - - uses_multi_shard_transaction - - uses_prepare_transaction - - uses_transactions - -executor: - archive: - hooks: - - CheckReplDBHashInBackground - - CheckReplDBHash - - ValidateCollections - config: - shell_options: - nodb: '' - global_vars: - TestData: - setParameters: - enableTestCommands: 1 - failpoint.batchDeletesByDefault: "{mode: 'alwaysOn'}" diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 74364173486..b48d73bf3f2 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -235,8 +235,13 @@ selector: # SERVER-51805 splitChunk op is not idempotent - jstests/sharding/mongos_get_shard_version.js + # Expects reshardCollection executes without config server stepdown + - jstests/sharding/shard_encrypted_collection.js + # Test will fail if it is unable to lock the config server primary successfully. - jstests/sharding/fsync_lock_unlock.js + - jstests/sharding/fsync_deadlock.js + - jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js exclude_with_any_tags: - does_not_support_stepdowns diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 039ff78dbc5..f3ab984756c 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -101,7 +101,7 @@ DEFAULTS = { "stagger_jobs": None, "majority_read_concern": "on", "storage_engine": "wiredTiger", - "enable_enterprise_tests": None, + "enable_enterprise_tests": "on", "storage_engine_cache_size_gb": None, "suite_files": "with_server", "tag_files": [], @@ -295,6 +295,9 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None +# if set, enables enterprise jstest to automatically be included +ENABLE_ENTERPRISE_TESTS = None + # URL to connect to the Evergreen service. EVERGREEN_URL = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 35e79bbd72e..ed659d2f79b 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -1,5 +1,6 @@ """Configure the command line input for the resmoke 'run' subcommand.""" +import argparse import collections import configparser import datetime @@ -22,6 +23,7 @@ from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils from buildscripts.resmokelib import mongod_fuzzer_configs from buildscripts.resmokelib.suitesconfig import SuiteFinder +from buildscripts.util.read_config import read_config_file def validate_and_update_config(parser, args): @@ -520,3 +522,26 @@ def _update_symbolizer_secrets(): yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") + + +def detect_evergreen_config(parsed_args: argparse.Namespace, + expansions_file: str = "../expansions.yml"): + """Detect evergreen expansions.""" + if not os.path.exists(expansions_file): + return + + expansions = read_config_file(expansions_file) + + parsed_args.build_id = expansions.get("build_id", None) + parsed_args.distro_id = expansions.get("distro_id", None) + parsed_args.execution_number = expansions.get("execution", None) + parsed_args.project_name = expansions.get("project", None) + parsed_args.git_revision = expansions.get("revision", None) + parsed_args.revision_order_id = expansions.get("revision_order_id", None) + parsed_args.task_id = expansions.get("task_id", None) + parsed_args.task_name = expansions.get("task_name", None) + parsed_args.variant_name = expansions.get("build_variant", None) + parsed_args.version_id = expansions.get("version_id", None) + parsed_args.work_dir = expansions.get("workdir", None) + parsed_args.evg_project_config_path = expansions.get("evergreen_config_file_path", None) + parsed_args.requester = expansions.get("requester", None) diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 2a22d177a29..fa9a2e98ebd 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -6,7 +6,9 @@ Handles all the nitty-gritty parameter conversion. import json import os import os.path +import re import stat +from packaging import version from buildscripts.resmokelib import config from buildscripts.resmokelib import utils @@ -43,6 +45,31 @@ def get_path_env_var(env_vars): return path +def get_binary_version(executable): + """Return the string for the binary version of the given executable.""" + + # pylint: disable=wrong-import-position + from buildscripts.resmokelib.multiversionconstants import LATEST_FCV + + split_executable = executable.split("-") + version_regex = re.compile(version.VERSION_PATTERN, re.VERBOSE | re.IGNORECASE) + if len(split_executable) > 1 and version_regex.match(split_executable[-1]): + return split_executable[-1] + return LATEST_FCV + + +def remove_set_parameter_if_before_version(set_parameters, parameter_name, bin_version, + required_bin_version): + """ + Used for removing a server parameter that does not exist prior to a specified version. + + Remove 'parameter_name' from the 'set_parameters' dictionary if 'bin_version' is older than + 'required_bin_version'. + """ + if version.parse(bin_version) < version.parse(required_bin_version): + set_parameters.pop(parameter_name, None) + + def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): """ Return a Process instance that starts mongod arguments constructed from 'mongod_options'. @@ -54,12 +81,17 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): @param mongod_options - A HistoryDict describing the various options to pass to the mongod. """ + bin_version = get_binary_version(executable) args = [executable] mongod_options = mongod_options.copy() if "port" not in mongod_options: mongod_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongod_options.get("set_parameters", {}) + remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", + bin_version, "6.0") + remove_set_parameter_if_before_version( + suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongod_options.pop("set_parameters") @@ -78,6 +110,7 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos_options=None): # pylint: disable=too-many-arguments """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" + bin_version = get_binary_version(executable) args = [executable] mongos_options = mongos_options.copy() @@ -85,6 +118,10 @@ def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos if "port" not in mongos_options: mongos_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongos_options.get("set_parameters", {}) + remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", + bin_version, "6.0") + remove_set_parameter_if_before_version( + suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongos_options.pop("set_parameters") diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 2c4d4b761d7..97788b0469b 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -5,12 +5,14 @@ import logging import os import sys import tempfile +from datetime import datetime from abc import ABCMeta, abstractmethod from collections import namedtuple from distutils import spawn # pylint: disable=no-name-in-module from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program from buildscripts.resmokelib.hang_analyzer.process_list import Pinfo +from buildscripts.resmokelib import config as resmoke_config Dumpers = namedtuple('Dumpers', ['dbg', 'jstack']) @@ -330,6 +332,20 @@ class LLDBDumper(Dumper): class GDBDumper(Dumper): """GDBDumper class.""" + def __init__(self, root_logger: logging.Logger, dbg_output: str, + timeout_seconds_for_gdb_process=720): + """Initialize GDBDumper.""" + if resmoke_config.EVERGREEN_TASK_ID is None: + # Set 24 hours time out for hang analyzer being run in locally + timeout_seconds_for_gdb_process = 86400 + #Timeout for hang analyzer, default timeout is 12mins(out of total 15mins) in Evergreen + self._timeout_seconds_for_gdb_process = timeout_seconds_for_gdb_process + super().__init__(root_logger, dbg_output) + + def _reduce_timeout_for_gdb_process(self, timeout_period: int): + """Reduce timeout for remaining gdb processes.""" + self._timeout_seconds_for_gdb_process -= timeout_period + def _find_debugger(self, debugger): """Find the installed debugger.""" return find_program(debugger, ['/opt/mongodbtoolchain/v3/bin', '/usr/bin']) @@ -445,12 +461,19 @@ class GDBDumper(Dumper): debugger = "gdb" dbg = self._find_debugger(debugger) logger = _get_process_logger(self._dbg_output, pinfo.name) + _start_time = datetime.now() if dbg is None: self._root_logger.warning("Debugger %s not found, skipping dumping of %s", debugger, str(pinfo.pidv)) return + if self._timeout_seconds_for_gdb_process <= 0: + self._root_logger.warning( + "Skipping dumping of %s processes with PIDs %s because the time limit expired", + pinfo.name, str(pinfo.pidv)) + return + self._root_logger.info("Debugger %s, analyzing %s processes with PIDs %s", dbg, pinfo.name, str(pinfo.pidv)) @@ -467,8 +490,11 @@ class GDBDumper(Dumper): skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( - itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) + itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger, + self._timeout_seconds_for_gdb_process, pinfo) + time_period = (datetime.now() - _start_time).total_seconds() + self._reduce_timeout_for_gdb_process(time_period) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, str(pinfo.pidv)) diff --git a/buildscripts/resmokelib/hang_analyzer/process.py b/buildscripts/resmokelib/hang_analyzer/process.py index 629f78a2594..ee528aa6de6 100644 --- a/buildscripts/resmokelib/hang_analyzer/process.py +++ b/buildscripts/resmokelib/hang_analyzer/process.py @@ -22,7 +22,7 @@ if _IS_WINDOWS: PROCS_TIMEOUT_SECS = 60 -def call(args, logger): +def call(args, logger, timeout_seconds=None, pinfo=None): """Call subprocess on args list.""" logger.info(str(args)) @@ -31,7 +31,16 @@ def call(args, logger): logger_pipe = core.pipe.LoggerPipe(logger, logging.INFO, process.stdout) logger_pipe.wait_until_started() - ret = process.wait() + try: + ret = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + logger.error("Killing %s processes with PIDs %s because time limit expired", pinfo.name, + str(pinfo.pidv)) + process.kill() + process.wait() + logger_pipe.wait_until_finished() + return + logger_pipe.wait_until_finished() if ret != 0: diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 692b4532c47..07bc214517a 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -161,6 +161,29 @@ class BufferedHandler(logging.Handler): logging.Handler.close(self) +class BufferedFileHandler(BufferedHandler): + """File handler with in-memory buffering.""" + + def __init__(self, filename, capacity=2000, interval_secs=600): + """Initialize the handler with the filename and buffer capacity and flush interval.""" + super().__init__(capacity, interval_secs) + self.file = open(filename, "a", encoding="utf-8") + + def process_record(self, record): + """Return the formatted record message appended with a newline.""" + return self.format(record) + "\n" + + def _flush_buffer_with_lock(self, buf, close_called): + """Write the buffered log lines to the destination file.""" + self.file.writelines(buf) + + def close(self): + """Close the handler and the file descriptor.""" + super().close() + + self.file.close() + + class HTTPHandler(object): """A class which sends data to a web server using POST requests.""" diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index f93cfbd4f4a..2aade997a42 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -13,6 +13,7 @@ from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import redirect as redirect_lib from buildscripts.resmokelib.logging import buildlogger from buildscripts.resmokelib.logging import formatters +from buildscripts.resmokelib.logging.handlers import BufferedFileHandler _DEFAULT_FORMAT = "[%(name)s] %(message)s" @@ -37,6 +38,11 @@ _BUILD_ID_REGISTRY: dict = {} # Maps job nums to fixture loggers. _FIXTURE_LOGGER_REGISTRY: dict = {} +# URL of parsley logs. +RAW_TEST_LOGS_URL = "https://evergreen.mongodb.com/rest/v2/tasks/{task_id}/build/TestLogs/job{job_num}%2F{test_id}.log?execution={execution}&print_time=true" +RAW_JOBS_LOGS_URL = "https://evergreen.mongodb.com/rest/v2/tasks/{task_id}/build/TestLogs/job{job_num}?execution={execution}&print_time=true" +PARSLEY_JOBS_LOGS_URL = "https://parsley.mongodb.com/test/{task_id}/{execution}/job{job_num}/all" + def _build_logger_server(): """Create and return a new BuildloggerServer. @@ -209,7 +215,7 @@ def new_test_logger(test_shortname, test_basename, command, parent, job_num, tes name = "%s:%s" % (parent.name, test_shortname) logger = logging.Logger(name) logger.parent = parent - _add_evergreen_handler(logger, job_num, test_id) + _add_evergreen_handler(logger, job_num, test_id, test_basename) def _get_test_endpoint(job_num, test_basename, command, meta_logger): """Get a new test endpoint for the buildlogger server.""" @@ -363,7 +369,7 @@ def _write_evergreen_log_spec(): yaml.dump(log_spec, fd) -def _add_evergreen_handler(logger, job_num, test_id=None): +def _add_evergreen_handler(logger, job_num, test_id=None, test_name=None): """Add a new evergreen handler to a logger.""" logger_info = config.LOGGING_CONFIG[TESTS_LOGGER_NAME] evergreen_handler_info = None @@ -376,15 +382,35 @@ def _add_evergreen_handler(logger, job_num, test_id=None): fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}" os.makedirs(os.path.dirname(fp), exist_ok=True) - handler = logging.FileHandler(filename=fp, mode="a") + handler = BufferedFileHandler(fp) handler.setFormatter( formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT))) logger.addHandler(handler) if test_id: + raw_url = RAW_TEST_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + test_id=test_id, + execution=config.EVERGREEN_EXECUTION, + ) ROOT_EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp) + ROOT_EXECUTOR_LOGGER.info("Raw logs for %s can be viewed at %s", test_name, raw_url) else: + parsley_url = PARSLEY_JOBS_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + execution=config.EVERGREEN_EXECUTION, + ) + raw_url = RAW_JOBS_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + execution=config.EVERGREEN_EXECUTION, + ) ROOT_EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp) + ROOT_EXECUTOR_LOGGER.info("Parsley logs for job #%s can be viewed at %s", job_num, + parsley_url) + ROOT_EXECUTOR_LOGGER.info("Raw logs for job #%s can be viewed at %s", job_num, raw_url) def _get_evergreen_log_dirname(): diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 71579bd0fea..7160d1ff026 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -96,7 +96,8 @@ class MultiversionPlugin(PluginInterface): :param kwargs: additional args. :return: None or a Subcommand. """ - configure_resmoke.validate_and_update_config(parser, parsed_args) if subcommand == MULTIVERSION_SUBCOMMAND: + configure_resmoke.detect_evergreen_config(parsed_args) + configure_resmoke.validate_and_update_config(parser, parsed_args) return MultiversionConfigSubcommand(parsed_args) return None diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 621d5ade9ae..f009e44fe45 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -78,6 +78,10 @@ class VersionConstantValues(NamedTuple): """Get a string version of the latest FCV.""" return version_str(self.latest) + def get_fcv_tags_less_than_latest(self) -> List[str]: + """Get the list of all fcv tags less than the latest.""" + return [tag_str(fcv) for fcv in self.fcvs_less_than_latest] + def build_last_lts_binary(self, base_name: str) -> str: """ Build the name of the binary that the LTS version of the given tool will have. diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 3646d139b1f..a8aacb3f60f 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,7 +2,9 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output +import http import requests +from retry import retry import structlog @@ -17,7 +19,7 @@ LAST_CONTINUOUS = "last_continuous" # We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date # with the latest EOL versions. If a "last-continuous" version is EOL, we don't include # it in the multiversion config and therefore don't test against it. -MASTER_RELEASES_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" +MASTER_RELEASES_REMOTE_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" LOGGER = structlog.getLogger(__name__) @@ -36,11 +38,17 @@ def generate_mongo_version_file(): mongo_version_fh.write("mongo_version: " + res) +@retry(tries=5, delay=3) def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. with open(RELEASES_YAML, "wb") as file: - file.write(requests.get(MASTER_RELEASES_FILE).content) + response = requests.get(MASTER_RELEASES_REMOTE_FILE) + if response.status_code != http.HTTPStatus.OK: + raise RuntimeError( + f"Fetching releases.yml file returned unsuccessful status: {response.status_code}, " + f"response body: {response.text}\n") + file.write(response.content) def in_git_root_dir(): @@ -102,6 +110,8 @@ REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() # All multiversion tests should be run with these tags excluded. REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() +REQUIRES_FCV_TAGS_LESS_THAN_LATEST = version_constants.get_fcv_tags_less_than_latest() + # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) diff --git a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py index d392c6ae0b8..c45f93192eb 100644 --- a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py +++ b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py @@ -4,70 +4,70 @@ This hook runs continuously, but the run_aggregate_metrics_background.js file it internally sleep for 1 second between runs. """ -import os.path +import random +import pymongo -from buildscripts.resmokelib import errors -from buildscripts.resmokelib.testing.hooks import jsfile -from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase +from buildscripts.resmokelib.testing.hooks.bghook import BGHook -class AggregateResourceConsumptionMetricsInBackground(jsfile.JSHook): - """A hook to run $operationMetrics stage in the background.""" - - IS_BACKGROUND = True - - def __init__(self, hook_logger, fixture, shell_options=None): - """Initialize AggregateResourceConsumptionMetricsInBackground.""" - description = "Run background $operationMetrics on all mongods while a test is running" - js_filename = os.path.join("jstests", "hooks", "run_aggregate_metrics_background.js") - jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description, - shell_options=shell_options) - self._background_job = None +def verify_metrics(doc): + """Checks whether the output from $operatiomMetrics has the schema we expect.""" - def before_suite(self, test_report): - """Start the background thread.""" - self._background_job = _BackgroundJob("AggregateResourceConsumptionMetricsInBackground") - self.logger.info("Starting the background aggregate metrics thread.") - self._background_job.start() + top_level_fields = [ + "docBytesWritten", "docUnitsWritten", "idxEntryBytesWritten", "idxEntryUnitsWritten", + "totalUnitsWritten", "cpuNanos", "db", "primaryMetrics", "secondaryMetrics" + ] + read_fields = [ + "docBytesRead", "docUnitsRead", "idxEntryBytesRead", "idxEntryUnitsRead", "keysSorted", + "docUnitsReturned" + ] - def after_suite(self, test_report, teardown_flag=None): - """Signal the background aggregate metrics thread to exit, and wait until it does.""" - if self._background_job is None: - return + for key in top_level_fields: + assert key in doc, ("The metrics output is missing the property: " + key) - self.logger.info("Stopping the background aggregate metrics thread.") - self._background_job.stop() + primary_metrics = doc["primaryMetrics"] + for key in read_fields: + assert key in primary_metrics, ( + "The metrics output is missing the property: primaryMetrics." + key) - def before_test(self, test, test_report): - """Instruct the background aggregate metrics thread to run while 'test' is also running.""" - if self._background_job is None: - return + secondary_metrics = doc["secondaryMetrics"] + for key in read_fields: + assert key in secondary_metrics, ( + "The metrics output is missing the property: secondaryMetrics." + key) - hook_test_case = _ContinuousDynamicJSTestCase.create_before_test( - test.logger, test, self, self._js_filename, self._shell_options) - hook_test_case.configure(self.fixture) - self.logger.info("Resuming the background aggregate metrics thread.") - self._background_job.resume(hook_test_case, test_report) - - def after_test(self, test, test_report): # noqa: D205,D400 - """Instruct the background aggregate metrics thread to stop running now that 'test' has - finished running. - """ - if self._background_job is None: - return +class AggregateResourceConsumptionMetricsInBackground(BGHook): + """A hook to run $operationMetrics stage in the background.""" - self.logger.info("Pausing the background aggregate metrics thread.") - self._background_job.pause() + def __init__(self, hook_logger, fixture): + """Initialize AggregateResourceConsumptionMetricsInBackground.""" - if self._background_job.exc_info is not None: - if isinstance(self._background_job.exc_info[1], errors.TestFailure): - # If the mongo shell process running the JavaScript file exited with a non-zero - # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's - # test execution to stop. - raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) - else: - self.logger.error( - "Encountered an error inside the background aggregate metrics thread.", - exc_info=self._background_job.exc_info) - raise self._background_job.exc_info[1] + description = "Run background $operationMetrics on all mongods while a test is running" + super().__init__(hook_logger, fixture, description, tests_per_cycle=None, + loop_delay_ms=1000) + + def run_action(self): + """Collects $operationMetrics on all non-arbiter nodes in the fixture.""" + for node_info in self.fixture.get_node_info(): + conn = pymongo.MongoClient(port=node_info.port) + # Filter out arbiters. + if "arbiterOnly" in conn.admin.command({"isMaster": 1}): + self.logger.info( + "Skipping background aggregation against test node: %s because it is an " + + "arbiter and has no data.", node_info.full_name) + return + + # Clear the metrics about 10% of the time. + clear_metrics = random.random() < 0.1 + self.logger.info("Running $operationMetrics with {clearMetrics: %s} on host: %s", + clear_metrics, node_info.full_name) + with conn.admin.aggregate( + [{"$operationMetrics": {"clearMetrics": clear_metrics}}]) as cursor: + for doc in cursor: + try: + verify_metrics(doc) + except: + self.logger.info( + "caught exception while verifying that all expected fields are in the" + + " metrics output: ", doc) + raise diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py index 92dbb5437a4..cd775aabd4a 100644 --- a/buildscripts/resmokelib/testing/hooks/background_job.py +++ b/buildscripts/resmokelib/testing/hooks/background_job.py @@ -3,6 +3,7 @@ import sys import threading +from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing.hooks import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/bghook.py b/buildscripts/resmokelib/testing/hooks/bghook.py index b61f37b4e2d..1b3fed3b104 100644 --- a/buildscripts/resmokelib/testing/hooks/bghook.py +++ b/buildscripts/resmokelib/testing/hooks/bghook.py @@ -13,11 +13,13 @@ class BGJob(threading.Thread): BGJob will call 'run_action' without any delay and expects the 'run_action' function to add some form of delay. """ - def __init__(self, hook): + def __init__(self, hook, loop_delay_ms=None): """Initialize the background job.""" threading.Thread.__init__(self, name=f"BGJob-{hook.__class__.__name__}") + self._loop_delay_ms = loop_delay_ms self.daemon = True self._hook = hook + self._interrupt_event = threading.Event() self.__is_alive = True self.err = None @@ -29,6 +31,14 @@ class BGJob(threading.Thread): try: self._hook.run_action() + if self._loop_delay_ms is not None: + # The configured loop delay asked us to wait before running the action again. Do + # that wait, but listen to see if we finish running the test or are killed in + # the meantime. + interrupted = self._interrupt_event.wait(self._loop_delay_ms / 1000.0) + if interrupted: + self._hook.logger.info("interrupted") + break except Exception as err: # pylint: disable=broad-except self._hook.logger.error("Background thread caught exception: %s.", err) self.err = err @@ -37,6 +47,7 @@ class BGJob(threading.Thread): def kill(self): """Kill the background job.""" self.__is_alive = False + self._interrupt_event.set() class BGHook(interface.Hook): @@ -46,8 +57,13 @@ class BGHook(interface.Hook): # By default, we continuously run the background hook for the duration of the suite. DEFAULT_TESTS_PER_CYCLE = math.inf - def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None): - """Initialize the background hook.""" + def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None, loop_delay_ms=None): + """ + Initialize the background hook. + + 'tests_per_cycle' or 'loop_delay_ms' can be used to configure how often the background job + is restarted, and how often run_action() is called, respectively. + """ interface.Hook.__init__(self, hook_logger, fixture, desc) self.logger = hook_logger @@ -57,15 +73,21 @@ class BGHook(interface.Hook): self._test_num = 0 # The number of tests we execute before restarting the background hook. self._tests_per_cycle = self.DEFAULT_TESTS_PER_CYCLE if tests_per_cycle is None else tests_per_cycle + self._loop_delay_ms = loop_delay_ms def run_action(self): - """Perform an action. This function will be called continuously in the BgJob.""" + """ + Perform an action. This function will be called continuously in the BgJob. + + If a sleep_delay_ms was given, that many milliseconds of sleep will happen between each + invocation. + """ raise NotImplementedError def before_suite(self, test_report): """Start the background thread.""" self.logger.info("Starting the background thread.") - self._background_job = BGJob(self) + self._background_job = BGJob(self, self._loop_delay_ms) self._background_job.start() def after_suite(self, test_report, teardown_flag=None): @@ -86,7 +108,7 @@ class BGHook(interface.Hook): return self.logger.info("Restarting the background thread.") - self._background_job = BGJob(self) + self._background_job = BGJob(self, self._loop_delay_ms) self._background_job.start() def after_test(self, test, test_report): diff --git a/buildscripts/resmokelib/testing/hooks/run_query_stats.py b/buildscripts/resmokelib/testing/hooks/run_query_stats.py new file mode 100644 index 00000000000..736c5e261c7 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/run_query_stats.py @@ -0,0 +1,83 @@ +""" +Test hook for verifying $queryStats collects expected metrics and can redact query shapes. + +This runs in the background as other tests are ongoing. +""" + +from bson import binary +import pymongo.errors +from buildscripts.resmokelib.testing.hooks.interface import Hook + +QUERY_STATS_NOT_ENABLED_CODES = [224, 7373500, 6579000] + + +class RunQueryStats(Hook): + """Runs $queryStats after every test, and clears the query stats store before every test.""" + + IS_BACKGROUND = False + + def __init__(self, hook_logger, fixture, allow_feature_not_supported=False): + """Initialize the RunQueryStats hook. + + Args: + hook_logger: the logger instance for this hook. + fixture: the target fixture (replica sets or a sharded cluster). + allow_feature_not_supported: absorb 'QueryFeatureNotAllowed' errors when calling + $queryStats. This is to support fuzzer suites that may manipulate the FCV. + """ + description = "Read query stats data after each test." + super().__init__(hook_logger, fixture, description) + self.client = self.fixture.mongo_client() + self.hmac_key = binary.Binary(("0" * 32).encode('utf-8'), 8) + self.allow_feature_not_supported = allow_feature_not_supported + + def verify_query_stats(self, querystats_spec): + """Verify a $queryStats call has all the right properties.""" + query_stats_pipeline = [ + {"$queryStats": querystats_spec}, + # SERVER-90921: The pymongo version we use on this branch has trouble parsing invalid + # DBRefs, which can be produced by the 'key' field. We'll overwrite that field with a + # dummy one, since the contents aren't important for this test/check. + {"$set": {"key": "Redacted due to issues with DBRefs"}} + ] + try: + with self.client.admin.aggregate(query_stats_pipeline) as cursor: + nreturned = 0 + for operation in cursor: + assert "key" in operation + assert "metrics" in operation + assert "asOf" in operation + nreturned += 1 + self.logger.info("Found %d query stats entries.", nreturned) + except pymongo.errors.OperationFailure as err: + if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: + self.logger.info("Encountered an error while running $queryStats. " + "$queryStats will not be run for this test.") + else: + raise err + + def after_test(self, test, test_report): + """After the test, make sure we can ingest the query stats, with and without hmac.""" + self.verify_query_stats({}) + self.verify_query_stats( + {"transformIdentifiers": {"algorithm": "hmac-sha-256", "hmacKey": self.hmac_key}}) + + # Log the number of evictions we encountered. + server_status = self.client.admin.command({"serverStatus": 1}) + num_evicted_entries = server_status["metrics"]["queryStats"]["numEvicted"] + if num_evicted_entries > 0: + self.logger.info("Evicted %d query stats entries during test execution.", + num_evicted_entries) + + def before_test(self, test, test_report): + """Before the test, reset the contents of the query stats store.""" + try: + # Clear out all existing entries, then reset the size cap. + self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="0%") + self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="1%") + except pymongo.errors.OperationFailure as err: + if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: + self.logger.info("Encountered an error while configuring the query stats store. " + "Query stats will not be collected for this test.") + else: + raise err diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 7658a415fa6..81202daceb8 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -85,7 +85,7 @@ def get_task_name_without_suffix(task_name, variant_name): """Return evergreen task name without suffix added to the generated task. Remove evergreen variant name, numerical suffix and underscores between them from evergreen task name. - Example: "noPassthrough_0_enterprise-rhel-80-64-bit-dynamic-required" -> "noPassthrough" + Example: "noPassthrough_0_enterprise-rhel-8-64-bit-dynamic-required" -> "noPassthrough" """ task_name = task_name if task_name else "" return re.sub(fr"(_[0-9]+)?(_{variant_name})?$", "", task_name) diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index f39666a0c37..bedee40d186 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -35,7 +35,7 @@ from buildscripts.task_generation.task_types.resmoke_tasks import ResmokeGenTask from buildscripts.util.cmdutils import enable_logging from buildscripts.util.fileops import read_yaml_file from buildscripts.burn_in_tests import DEFAULT_REPO_LOCATIONS, create_task_list_for_tests, \ - TaskInfo + TaskToBurnInInfo from buildscripts.ciconfig.evergreen import ( EvergreenProjectConfig, Task, @@ -172,7 +172,7 @@ class TaskConfigService: task_vars = task_def.get("vars", {}) break - task_vars.update({"suite": task.get_suite_name()}) + task_vars.update({"suite": task.get_suite_names()[0]}) task_name = task.name[:-4] if task.name.endswith("_gen") else task.name return { @@ -182,7 +182,7 @@ class TaskConfigService: "large_distro_name": build_variant_config.expansion("large_distro_name"), } - def get_task_configs_for_test_mappings(self, tests_by_task: Dict[str, TaskInfo], + def get_task_configs_for_test_mappings(self, tests_by_task: Dict[str, TaskToBurnInInfo], build_variant_config: Variant) -> Dict[str, dict]: """ For test mappings, generate a dict containing task names and their config settings. @@ -196,7 +196,8 @@ class TaskConfigService: task = _find_task(build_variant_config, task_name) if task and not _exclude_task(task): evg_task_config = self.get_evg_task_config(task, build_variant_config) - evg_task_config.update({"selected_tests_to_run": set(test_list_info.tests)}) + evg_task_config.update( + {"selected_tests_to_run": set(test_list_info.collect_suite_tests())}) evg_task_configs[task.name] = evg_task_config return evg_task_configs diff --git a/buildscripts/setup_spawnhost_coredump b/buildscripts/setup_spawnhost_coredump index 865dae12282..11191d53c4a 100755 --- a/buildscripts/setup_spawnhost_coredump +++ b/buildscripts/setup_spawnhost_coredump @@ -61,7 +61,7 @@ else # Write this file that gets cat'ed on login to communicate to users logging in if this setup script is still running. echo '+-----------------------------------------------------------------------------------+' > ~/.setup_spawnhost_coredump_progress - echo '| The setup script is still setting up data files for inspection on a [${machine}] host. |' >> ~/.setup_spawnhost_coredump_progress + echo "| The setup script is still setting up data files for inspection on a [${machine}] host. |" >> ~/.setup_spawnhost_coredump_progress echo '+-----------------------------------------------------------------------------------+' >> ~/.setup_spawnhost_coredump_progress cat >> ~/.profile <<EOF @@ -157,3 +157,44 @@ EOF # paths/environment variables will be set as intended. wall "The setup_spawnhost_coredump script has completed, please relogin to ensure the right environment variables are set." fi + +# Send a Slack notification as the very last thing the setup_spawnhost_coredump script does. +# This way a Server engineer can temporarily forget about the Evergreen host they spawned until the +# paths and environment variables are configured as intended for when they first connect. +if [[ "${machine}" = "Cygwin" ]]; then + # The setup_spawnhost_coredump script runs as the mci-exec user on Windows hosts. However, + # Server engineers log in as the Administrator user. + ssh_user="Administrator" + # The Evergreen binary only expects a Windows path. The rest of Cygwin is flexible about it + # being a Cygwin path or a Windows path so we do the conversion here. + evg_credentials_pathname=$(cygpath -w ~Administrator/.evergreen.yml) + evg_binary_pathname=~Administrator/cli_bin/evergreen +else + ssh_user=$(whoami) + evg_credentials_pathname=~/.evergreen.yml + evg_binary_pathname=evergreen +fi + +slack_user=$(awk '{if ($1 == "user:") print $2}' "$evg_credentials_pathname") +# Refer to the https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html +# documentation for more information on the AWS instance metadata endpoints. +aws_metadata_svc="http://169.254.169.254" +aws_token=$(curl -s -X PUT "$aws_metadata_svc/latest/api/token" -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') +ssh_host=$(curl -s -H "X-aws-ec2-metadata-token: $aws_token" "$aws_metadata_svc/latest/meta-data/public-hostname") +if [[ "${machine}" = "Cygwin" ]]; then + slack_message=$(printf "The setup_spawnhost_coredump script has finished setting things up. \ +Please use Windows Remote Desktop with\n\ +1. PC name: $ssh_host\n\ +2. User account: $ssh_user\n\ +3. The RDP password configured under the edit dialog at https://spruce.mongodb.com/spawn/host\n\ +to log in.") +else + slack_message="The setup_spawnhost_coredump script has finished setting things up. Please run "'```'"ssh $ssh_user@$ssh_host"'```'" to log in." +fi + +# The Evergreen spawn host is expected to be provisioned with the user's .evergreen.yml credentials. +# But in case something unexpected happens we don't want the setup_spawnhost_coredump script itself +# to error. +if [[ -n "${slack_user}" ]]; then + "$evg_binary_pathname" --config "$evg_credentials_pathname" notify slack -t "@$slack_user" -m "$slack_message" +fi diff --git a/buildscripts/sync_repo_with_copybara.py b/buildscripts/sync_repo_with_copybara.py index b94c3e8ba65..889f8fe8d7f 100644 --- a/buildscripts/sync_repo_with_copybara.py +++ b/buildscripts/sync_repo_with_copybara.py @@ -1,5 +1,6 @@ """Module for syncing a repo with Copybara and setting up configurations.""" import argparse +import fileinput import subprocess import os import sys @@ -112,25 +113,56 @@ def main(): # Read configurations expansions = read_config_file(args.expansion_file) - access_token_copybara_syncer = get_installation_access_token( - expansions["app_id_copybara_syncer"], expansions["private_key_copybara_syncer"], - expansions["installation_id_copybara_syncer"]) + token_mongodb_mongo = get_installation_access_token( + expansions["app_id_copybara_syncer"], + expansions["private_key_copybara_syncer"], + expansions["installation_id_copybara_syncer"], + ) + token_10gen_mongo = get_installation_access_token( + expansions["app_id_copybara_syncer_10gen"], + expansions["private_key_copybara_syncer_10gen"], + expansions["installation_id_copybara_syncer_10gen"], + ) + + tokens_map = { + "https://github.com/mongodb/mongo.git": token_mongodb_mongo, + "https://github.com/10gen/mongo.git": token_10gen_mongo, + } # Create the mongodb-bot.gitconfig file as necessary. create_mongodb_bot_gitconfig() current_dir = os.getcwd() - git_destination_url_with_token = f"https://x-access-token:{access_token_copybara_syncer}@github.com/mongodb/mongo.git" + config_file = f"{current_dir}/copy.bara.sky" + + # Overwrite repo urls in copybara config in-place + with fileinput.FileInput(config_file, inplace=True) as file: + for line in file: + token = None + for repo, value in tokens_map.items(): + if repo in line: + token = value + + if token: + print( + line.replace( + "https://github.com", + f"https://x-access-token:{token}@github.com", + ), + end="", + ) + else: + print(line, end="") # Set up the Docker command and execute it docker_cmd = [ "docker run", "-v ~/.ssh:/root/.ssh", "-v ~/mongodb-bot.gitconfig:/root/.gitconfig", - f'-v "{current_dir}/copybara.sky":/usr/src/app/copy.bara.sky', + f'-v "{config_file}":/usr/src/app/copy.bara.sky', "-e COPYBARA_CONFIG='copy.bara.sky'", "-e COPYBARA_SUBCOMMAND='migrate'", - f"-e COPYBARA_OPTIONS='-v --git-destination-url={git_destination_url_with_token}'", + "-e COPYBARA_OPTIONS='-v'", "copybara copybara", ] diff --git a/buildscripts/testmatrix/getdisplaytaskname.py b/buildscripts/testmatrix/getdisplaytaskname.py index c0a08b422bc..e8ff49d6a39 100644 --- a/buildscripts/testmatrix/getdisplaytaskname.py +++ b/buildscripts/testmatrix/getdisplaytaskname.py @@ -1,7 +1,7 @@ """ Get the display task name from the execution task and the variant. -Get an execution task name like this: multiversion_auth_0_enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required +Get an execution task name like this: multiversion_auth_0_enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required Into a display task name like this: multiversion_auth """ diff --git a/buildscripts/tests/burn_in_tests_end2end/__init__.py b/buildscripts/tests/burn_in_tests_end2end/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/burn_in_tests_end2end/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py b/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py new file mode 100644 index 00000000000..cea56d16e9d --- /dev/null +++ b/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py @@ -0,0 +1,42 @@ +"""E2E tests for the buildscripts/burn_in_tests.py.""" + +import os +import subprocess +import sys +import unittest + +import yaml + +import buildscripts.burn_in_tests as under_test + +# pylint: disable=missing-docstring,protected-access,invalid-name,subprocess-run-check,broad-except + + +class TestBurnInTestsEnd2End(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([ + sys.executable, + "buildscripts/burn_in_tests.py", + "generate-test-membership-map-file-for-ci", + ]) + + @classmethod + def tearDownClass(cls): + if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE): + os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE) + + def test_valid_yaml_output(self): + process = subprocess.run([ + sys.executable, + "buildscripts/burn_in_tests.py", + "run", + "--yaml", + ], text=True, capture_output=True) + output = process.stdout + self.assertEqual(0, process.returncode) + + try: + yaml.safe_load(output) + except Exception: + self.fail(msg="burn_in_tests.py does not output valid yaml.") diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml index 60fd7628959..9a03764e5cd 100644 --- a/buildscripts/tests/ciconfig/evergreen.yml +++ b/buildscripts/tests/ciconfig/evergreen.yml @@ -141,6 +141,22 @@ tasks: vars: resmoke_args: "--storageEngine=wiredTiger" +- name: resmoke_multiversion_task_gen + depends_on: + - name: compile + commands: + - func: "initialize multiversion tasks" + vars: + multiversion_sanity_check_last_continuous_new_new_old: last_continuous + multiversion_sanity_check_last_continuous_new_old_new: last_continuous + multiversion_sanity_check_last_continuous_old_new_new: last_continuous + multiversion_sanity_check_last_lts_new_new_old: last_lts + multiversion_sanity_check_last_lts_new_old_new: last_lts + multiversion_sanity_check_last_lts_old_new_new: last_lts + - func: "generate resmoke tasks" + vars: + resmoke_args: "--storageEngine=wiredTiger" + modules: - name: render-module @@ -183,6 +199,7 @@ buildvariants: - debian-stretch tasks: - name: resmoke_task + - name: resmoke_multiversion_task_gen - name: amazon display_name: "! Amazon" run_on: diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index 32a8cd330f3..20a25d35801 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -6,7 +6,7 @@ import unittest import buildscripts.ciconfig.evergreen as _evergreen -# pylint: disable=missing-docstring,protected-access +# pylint: disable=missing-docstring,protected-access,invalid-name TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "evergreen.yml") @@ -24,14 +24,15 @@ class TestEvergreenProjectConfig(unittest.TestCase): _evergreen.parse_evergreen_file(invalid_path, evergreen_binary=None) def test_list_tasks(self): - self.assertEqual(6, len(self.conf.tasks)) - self.assertEqual(6, len(self.conf.task_names)) + self.assertEqual(7, len(self.conf.tasks)) + self.assertEqual(7, len(self.conf.task_names)) self.assertIn("compile", self.conf.task_names) self.assertIn("passing_test", self.conf.task_names) self.assertIn("failing_test", self.conf.task_names) self.assertIn("timeout_test", self.conf.task_names) self.assertIn("no_lifecycle_task", self.conf.task_names) self.assertIn("resmoke_task", self.conf.task_names) + self.assertIn("resmoke_multiversion_task_gen", self.conf.task_names) def test_list_task_groups(self): self.assertEqual(1, len(self.conf.task_groups)) @@ -83,55 +84,93 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual([], task.depends_on) self.assertEqual(task_dict, task.raw) - def test_resmoke_args(self): + def test_suite_to_resmoke_args_map_for_non_gen_task(self): suite_and_task = "jstestfuzz" - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--arg=val"}}] + task_commands = [{ + "func": "run tests", + "vars": {"resmoke_args": "--arg=val"}, + }] task_dict = {"name": suite_and_task, "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual(f"--suites={suite_and_task} --arg=val", task.resmoke_args) + self.assertEqual({suite_and_task: f"--suites={suite_and_task} --arg=val"}, + task.suite_to_resmoke_args_map) - def test_is_run_tests_task(self): - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] - task_dict = {"name": "jsCore", "commands": task_commands} + def test_suite_to_resmoke_args_map_for_gen_task(self): + suite = "jsCore" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"resmoke_args": "--installDir=/bin"}, + }] + task_dict = {"name": f"{suite}_gen", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertTrue(task.is_run_tests_task) - self.assertFalse(task.is_generate_resmoke_task) + self.assertEqual({suite: f"--suites={suite} --installDir=/bin"}, + task.suite_to_resmoke_args_map) - def test_run_tests_command(self): - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] + def test_suite_to_resmoke_args_map_for_gen_task_with_suite(self): + suite = "core" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"suite": suite, "resmoke_args": "--installDir=/bin"}, + }] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertDictEqual(task_commands[0], task.run_tests_command) - - def test_run_tests_multiversion(self): - require_multiversion_setup = True - task_commands = [{"func": "do multiversion setup"}, - {"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] - task_dict = {"name": "jsCore", "commands": task_commands, "tags": ["multiversion"]} + self.assertEqual({suite: f"--suites={suite} --installDir=/bin"}, + task.suite_to_resmoke_args_map) + + def test_suite_to_resmoke_args_map_for_initialize_multiversion_tasks_task(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + { + "func": "generate resmoke tasks", + "vars": {"resmoke_args": "--installDir=/bin"}, + }, + ] + task_dict = {"name": "multiversion_sanity_check_gen", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual(task.multiversion_setup_command, {"func": "do multiversion setup"}) - self.assertEqual(require_multiversion_setup, task.require_multiversion_setup()) + self.assertEqual({ + "multiversion_sanity_check_last_continuous_new_new_old": + "--suites=multiversion_sanity_check_last_continuous_new_new_old --installDir=/bin", + "multiversion_sanity_check_last_continuous_new_old_new": + "--suites=multiversion_sanity_check_last_continuous_new_old_new --installDir=/bin", + "multiversion_sanity_check_last_continuous_old_new_new": + "--suites=multiversion_sanity_check_last_continuous_old_new_new --installDir=/bin", + "multiversion_sanity_check_last_lts_new_new_old": + "--suites=multiversion_sanity_check_last_lts_new_new_old --installDir=/bin", + "multiversion_sanity_check_last_lts_new_old_new": + "--suites=multiversion_sanity_check_last_lts_new_old_new --installDir=/bin", + "multiversion_sanity_check_last_lts_old_new_new": + "--suites=multiversion_sanity_check_last_lts_old_new_new --installDir=/bin", + }, task.suite_to_resmoke_args_map) - def test_run_tests_no_multiversion(self): + def test_is_run_tests_task(self): task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertFalse(task.require_multiversion_setup()) - self.assertIsNone(task.multiversion_setup_command) + self.assertTrue(task.is_run_tests_task) + self.assertFalse(task.is_generate_resmoke_task) + self.assertFalse(task.is_initialize_multiversion_tasks_task) - def test_resmoke_args_gen(self): - task_commands = [{ - "func": "generate resmoke tasks", "vars": {"resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore_gen", "commands": task_commands} + def test_run_tests_command(self): + task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] + task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual("--suites=jsCore --installDir=/bin", task.resmoke_args) + self.assertDictEqual(task_commands[0], task.run_tests_command) def test_is_generate_resmoke_task(self): task_name = "core" @@ -144,6 +183,7 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue(task.is_generate_resmoke_task) self.assertFalse(task.is_run_tests_task) + self.assertFalse(task.is_initialize_multiversion_tasks_task) def test_generate_resmoke_tasks_command(self): task_commands = [{ @@ -155,17 +195,74 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual("jsCore", task.generated_task_name) - def test_resmoke_args_gen_with_suite(self): - task_name = "jsCore" - suite_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "suite": suite_name, "resmoke_args": "--installDir=/bin"} - }] + def test_is_initialize_multiversion_tasks_task(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ] + task = _evergreen.Task({ + "name": "multiversion_sanity_check_gen", + "commands": task_commands, + }) + + self.assertTrue(task.is_initialize_multiversion_tasks_task) + self.assertTrue(task.is_generate_resmoke_task) + self.assertFalse(task.is_run_tests_task) + + def test_initialize_multiversion_tasks_command(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ] + task = _evergreen.Task({ + "name": "multiversion_sanity_check_gen", + "commands": task_commands, + }) + + self.assertDictEqual(task_commands[0], task.initialize_multiversion_tasks_command) + self.assertEqual("multiversion_sanity_check", task.generated_task_name) + + def test_get_resmoke_command_vars_from_run_tests_command(self): + resmoke_command_vars = {"suite": "core"} + task_commands = [{"func": "run tests", "vars": resmoke_command_vars}] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual("--suites=core --installDir=/bin", task.resmoke_args) + self.assertEqual(resmoke_command_vars, task.get_resmoke_command_vars()) + + def test_get_resmoke_command_vars_from_generate_resmoke_tasks_command(self): + resmoke_command_vars = {"suite": "core"} + task_commands = [{"func": "generate resmoke tasks", "vars": resmoke_command_vars}] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertEqual(resmoke_command_vars, task.get_resmoke_command_vars()) + + def test_get_resmoke_command_vars_from_non_resmoke_task(self): + task_dict = {"name": "compile", "commands": []} + task = _evergreen.Task(task_dict) + + self.assertEqual({}, task.get_resmoke_command_vars()) def test_tags_with_no_tags(self): task_dict = { @@ -208,49 +305,61 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual("jsCore", task.generated_task_name) - def test_gen_resmoke_multiversion(self): - require_multiversion_setup = True - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore", "commands": task_commands, "tags": ["multiversion"]} - task = _evergreen.Task(task_dict) - - self.assertEqual(require_multiversion_setup, task.require_multiversion_setup()) + def test_get_suite_names_from_non_gen_task_name(self): + task = _evergreen.Task({ + "name": "task_name", + "commands": [{"func": "run tests"}], + }) - def test_gen_resmoke_no_multiversion(self): - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["task_name"], task.get_suite_names()) - self.assertFalse(task.require_multiversion_setup()) + def test_get_suite_names_from_non_gen_task_suite_var(self): + task = _evergreen.Task({ + "name": "task_name", + "commands": [{ + "func": "run tests", + "vars": {"suite": "suite_var"}, + }], + }) - def test_get_vars_suite_name_generate_resmoke_tasks(self): - task_name = "jsCore" - suite_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "suite": suite_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": task_name, "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["suite_var"], task.get_suite_names()) - self.assertEqual(suite_name, task.get_suite_name()) + def test_get_suite_names_from_gen_task_name(self): + task = _evergreen.Task({ + "name": "task_name_gen", + "commands": [{"func": "generate resmoke tasks"}], + }) - def test_get_suite_name_default_to_task_name(self): - task_name = "concurrency_gen" - no_gen_task_name = "concurrency" - task_commands = [{"func": "generate resmoke tasks"}] - task_dict = {"name": task_name, "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["task_name"], task.get_suite_names()) - self.assertEqual(no_gen_task_name, task.get_suite_name()) + def test_get_suite_names_from_gen_task_suite_var(self): + task = _evergreen.Task({ + "name": "task_name_gen", + "commands": [{ + "func": "generate resmoke tasks", + "vars": {"suite": "suite_var"}, + }], + }) + + self.assertEqual(["suite_var"], task.get_suite_names()) + + def test_get_suite_names_from_init_multiversion_task(self): + task = _evergreen.Task({ + "name": + "task_name_multiversion_gen", + "commands": [ + { + "func": "initialize multiversion tasks", + "vars": { + "suite_last_continuous": "last_continuous", + "suite_last_lts": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ], + }) + + self.assertEqual(["suite_last_continuous", "suite_last_lts"], task.get_suite_names()) def test_generate_task_name_non_gen_tasks(self): task_name = "jsCore" @@ -359,10 +468,10 @@ class TestVariant(unittest.TestCase): def test_distro_names(self): variant_ubuntu = self.conf.get_variant("ubuntu") - self.assertEqual(set(["ubuntu1404-test", "pdp-11"]), variant_ubuntu.distro_names) + self.assertEqual({"ubuntu1404-test", "pdp-11"}, variant_ubuntu.distro_names) variant_osx = self.conf.get_variant("osx-108") - self.assertEqual(set(["localtestdistro"]), variant_osx.distro_names) + self.assertEqual({"localtestdistro"}, variant_osx.distro_names) def test_test_flags(self): variant_ubuntu = self.conf.get_variant("ubuntu") @@ -389,21 +498,41 @@ class TestVariant(unittest.TestCase): self.assertEqual(variant_ubuntu, task.variant) self.assertIn(task_name, variant_ubuntu.task_names) - # Check combined_resmoke_args when test_flags is set on the variant. + # Check combined_suite_to_resmoke_args_map when test_flags is set on the variant. resmoke_task = variant_ubuntu.get_task("resmoke_task") - self.assertEqual("--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu", - resmoke_task.combined_resmoke_args) + self.assertEqual({ + "resmoke_task": + "--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu" + }, resmoke_task.combined_suite_to_resmoke_args_map) - # Check combined_resmoke_args when the task doesn't have resmoke_args. + # Check combined_suite_to_resmoke_args_map when the task doesn't have resmoke_args. passing_task = variant_ubuntu.get_task("passing_test") - self.assertEqual("--suites=passing_test --param=value --ubuntu", - passing_task.combined_resmoke_args) + self.assertEqual({"passing_test": "--suites=passing_test --param=value --ubuntu"}, + passing_task.combined_suite_to_resmoke_args_map) - # Check combined_resmoke_args when test_flags is not set on the variant. + # Check combined_suite_to_resmoke_args_map when test_flags is not set on the variant. variant_debian = self.conf.get_variant("debian") resmoke_task = variant_debian.get_task("resmoke_task") - self.assertEqual("--suites=resmoke_task --storageEngine=wiredTiger", - resmoke_task.combined_resmoke_args) + self.assertEqual({"resmoke_task": "--suites=resmoke_task --storageEngine=wiredTiger"}, + resmoke_task.combined_suite_to_resmoke_args_map) + + # Check combined_suite_to_resmoke_args_map for "initialize multiversion tasks" task. + variant_debian = self.conf.get_variant("debian") + resmoke_task = variant_debian.get_task("resmoke_multiversion_task_gen") + self.assertEqual({ + "multiversion_sanity_check_last_continuous_new_new_old": + "--suites=multiversion_sanity_check_last_continuous_new_new_old --storageEngine=wiredTiger", + "multiversion_sanity_check_last_continuous_new_old_new": + "--suites=multiversion_sanity_check_last_continuous_new_old_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_continuous_old_new_new": + "--suites=multiversion_sanity_check_last_continuous_old_new_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_new_new_old": + "--suites=multiversion_sanity_check_last_lts_new_new_old --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_new_old_new": + "--suites=multiversion_sanity_check_last_lts_new_old_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_old_new_new": + "--suites=multiversion_sanity_check_last_lts_old_new_new --storageEngine=wiredTiger", + }, resmoke_task.combined_suite_to_resmoke_args_map) # Check for tasks included in task_groups variant_amazon = self.conf.get_variant("amazon") diff --git a/buildscripts/tests/resmoke_validation/test_jstest_tags.py b/buildscripts/tests/resmoke_validation/test_jstest_tags.py new file mode 100644 index 00000000000..515fe7038c4 --- /dev/null +++ b/buildscripts/tests/resmoke_validation/test_jstest_tags.py @@ -0,0 +1,65 @@ +# pylint: disable=missing-docstring +import glob +import json +import unittest +from collections import defaultdict +from typing import Optional + +from buildscripts.resmokelib.multiversionconstants import ( + REQUIRES_FCV_TAG_LATEST, + REQUIRES_FCV_TAGS_LESS_THAN_LATEST, +) +from buildscripts.resmokelib.utils import jscomment + + +class JstestTagRule: + def __init__(self, failure_message): + self.failure_message = failure_message + self.failures = defaultdict(list) + + def check(self, file: str, tag: str) -> None: + if self._tag_failed(file, tag): + self.failures[file].append(tag) + + def _tag_failed(self, file: str, tag: str) -> bool: + raise NotImplementedError() + + def make_failure_message(self) -> Optional[str]: + if self.failures: + pretty_failures = json.dumps(self.failures, indent=4) + return f"{self.failure_message}:\n{pretty_failures}" + return None + + +class RequiresFcvTagRule(JstestTagRule): + def __init__(self): + super().__init__( + failure_message="The following tags reference FCV version that is not available") + self.allowed_tags = [*REQUIRES_FCV_TAGS_LESS_THAN_LATEST, REQUIRES_FCV_TAG_LATEST] + + def _tag_failed(self, file: str, tag: str) -> bool: + return tag.startswith("requires_fcv_") and tag not in self.allowed_tags + + +class TestJstestTags(unittest.TestCase): + def test_jstest_tags(self): + globs = ["src/mongo/db/modules/enterprise/jstests/**/*.js", "jstests/**/*.js"] + + tag_rules = [ + RequiresFcvTagRule(), + ] + + for pattern in globs: + for file in glob.glob(pattern, recursive=True): + for tag in jscomment.get_tags(file): + for tag_rule in tag_rules: + tag_rule.check(file, tag) + + full_failure_message = "" + for tag_rule in tag_rules: + failure_message = tag_rule.make_failure_message() + if failure_message: + full_failure_message = f"{full_failure_message}\n{failure_message}" + + if full_failure_message.strip(): + self.fail(full_failure_message.strip()) diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py deleted file mode 100644 index 0cb8554e1ad..00000000000 --- a/buildscripts/tests/test_burn_in_tags.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Unit tests for the burn_in_tags.py script.""" -from collections import defaultdict -import json -import os -import sys -import unittest -from unittest.mock import MagicMock, patch - -from shrub.v2 import ShrubProject - -import buildscripts.ciconfig.evergreen as _evergreen -from buildscripts.burn_in_tests import TaskInfo -from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns -from buildscripts.ciconfig.evergreen import EvergreenProjectConfig - -import buildscripts.burn_in_tags as under_test -from buildscripts.util.teststats import HistoricalTestInformation - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,too-many-arguments - -EMPTY_PROJECT = { - "buildvariants": [], - "tasks": [], -} -TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "test_burn_in_tags_evergreen.yml") - -NS = "buildscripts.burn_in_tags" - - -def ns(relative_name): # pylint: disable-invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def get_expansions_data(): - return { - "branch_name": "fake_branch", - "build_variant": "enterprise-rhel-80-64-bit-suggested", - "check_evergreen": 2, - "distro_id": "rhel80-small", - "is_patch": "true", - "max_revisions": 25, - "repeat_tests_max": 1000, - "repeat_tests_min": 2, - "repeat_tests_secs": 600, - "revision": "fake_sha", - "project": "fake_project", - "task_id": "task id", - } # yapf: disable - - -def get_evergreen_config() -> EvergreenProjectConfig: - return _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) - - -class TestCreateEvgBuildVariantMap(unittest.TestCase): - def test_create_evg_buildvariant_map(self): - expansions_file_data = { - "build_variant": "variant1", "burn_in_tag_include_build_variants": "variant2 variant3" - } - buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data) - - expected_buildvariant_map = { - "variant2": "variant2-required", "variant3": "variant3-required" - } - self.assertEqual(buildvariant_map, expected_buildvariant_map) - - -class TestGenerateEvgBuildVariants(unittest.TestCase): - def test_generate_evg_buildvariant_one_base_variant(self): - evg_conf_mock = get_evergreen_config() - base_variant = "enterprise-rhel-80-64-bit-inmem" - generated_variant = "enterprise-rhel-80-64-bit-inmem-required" - burn_in_tags_gen_variant = "enterprise-rhel-80-64-bit" - variant = evg_conf_mock.get_variant(base_variant) - - build_variant = under_test._generate_evg_build_variant(variant, generated_variant, - burn_in_tags_gen_variant) - - generated_build_variant = build_variant.as_dict() - self.assertEqual(generated_build_variant["name"], generated_variant) - self.assertNotIn('modules', generated_build_variant) - generated_expansions = generated_build_variant["expansions"] - burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass") - self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant) - self.assertEqual(generated_expansions, variant.expansions) - - -class TestGenerateEvgTasks(unittest.TestCase): - @patch(ns("create_tests_by_task")) - def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock): - evg_conf_mock = get_evergreen_config() - create_tests_by_task_mock.return_value = {} - expansions_file_data = get_expansions_data() - buildvariant_map = { - "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", - "enterprise-rhel-80-64-bit-majority-read-concern-off": - "enterprise-rhel-80-64-bit-majority-read-concern-off-required", - } # yapf: disable - shrub_config = ShrubProject() - evergreen_api = MagicMock() - repo = MagicMock(working_dir=os.getcwd()) - under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, - buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') - - self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT) - - @patch(ns("create_tests_by_task")) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_generate_evg_tasks_one_test_changed(self, get_stats_from_s3_mock, - create_tests_by_task_mock): - evg_conf_mock = get_evergreen_config() - create_tests_by_task_mock.return_value = { - "aggregation_mongos_passthrough": TaskInfo( - display_task_name="aggregation_mongos_passthrough", - suite="aggregation_mongos_passthrough", - resmoke_args="--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger", - tests=["jstests/aggregation/ifnull.js"], - require_multiversion_setup=False, - distro="", - build_variant="enterprise-rhel-80-64-bit-inmem" - ) - } # yapf: disable - expansions_file_data = get_expansions_data() - buildvariant_map = { - "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", - "enterprise-rhel-80-64-bit-majority-read-concern-off": - "enterprise-rhel-80-64-bit-majority-read-concern-off-required", - } # yapf: disable - shrub_config = ShrubProject.empty() - evergreen_api = MagicMock() - repo = MagicMock(working_dir=os.getcwd()) - get_stats_from_s3_mock.return_value = [ - HistoricalTestInformation( - test_name="dir/test2.js", - num_pass=1, - num_fail=0, - avg_duration_pass=10, - ) - ] - under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, - buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') - - generated_config = shrub_config.as_dict() - self.assertEqual(len(generated_config["buildvariants"]), 2) - first_generated_build_variant = generated_config["buildvariants"][0] - self.assertIn(first_generated_build_variant["name"], buildvariant_map.values()) - self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests") - self.assertEqual( - first_generated_build_variant["display_tasks"][0]["execution_tasks"][0], - f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}") - - -EXPANSIONS_FILE_DATA = { - "build_variant": "enterprise-rhel-80-64-bit", - "revision": "badf00d000000000000000000000000000000000", "max_revisions": "1000", - "branch_name": "mongodb-mongo-master", "is_patch": "false", "distro_id": "rhel62-small", - "repeat_tests_min": "2", "repeat_tests_max": "1000", "repeat_tests_secs": "600", "project": - "mongodb-mongo-master", "task_id": "task id" -} - -CREATE_EVG_BUILD_VARIANT_MAP = { - 'enterprise-rhel-80-64-bit-majority-read-concern-off': - 'enterprise-rhel-80-64-bit-majority-read-concern-off-required', - 'enterprise-rhel-80-64-bit-inmem': - 'enterprise-rhel-80-64-bit-inmem-required' -} - -CREATE_TEST_MEMBERSHIP_MAP = { - "jstests/aggregation/accumulators/accumulator_js.js": [ - "aggregation", "aggregation_auth", "aggregation_disabled_optimization", "aggregation_ese", - "aggregation_ese_gcm", "aggregation_facet_unwind_passthrough", - "aggregation_mongos_passthrough", "aggregation_one_shard_sharded_collections", - "aggregation_read_concern_majority_passthrough", "aggregation_secondary_reads", - "aggregation_sharded_collections_passthrough" - ], "jstests/core/create_collection.js": [ - "core", "core_auth", "core_ese", "core_ese_gcm", "core_minimum_batch_size", "core_op_query", - "cwrwc_passthrough", "cwrwc_rc_majority_passthrough", "cwrwc_wc_majority_passthrough", - "logical_session_cache_replication_100ms_refresh_jscore_passthrough", - "logical_session_cache_replication_10sec_refresh_jscore_passthrough", - "logical_session_cache_replication_1sec_refresh_jscore_passthrough", - "logical_session_cache_replication_default_refresh_jscore_passthrough", - "logical_session_cache_standalone_100ms_refresh_jscore_passthrough", - "logical_session_cache_standalone_10sec_refresh_jscore_passthrough", - "logical_session_cache_standalone_1sec_refresh_jscore_passthrough", - "logical_session_cache_standalone_default_refresh_jscore_passthrough", - "read_concern_linearizable_passthrough", "read_concern_majority_passthrough", - "causally_consistent_read_concern_snapshot_passthrough", - "replica_sets_initsync_jscore_passthrough", "replica_sets_fcbis_jscore_passthrough", - "replica_sets_initsync_static_jscore_passthrough", "replica_sets_jscore_passthrough", - "replica_sets_kill_primary_jscore_passthrough", - "replica_sets_kill_secondaries_jscore_passthrough", - "replica_sets_reconfig_jscore_passthrough", - "replica_sets_terminate_primary_jscore_passthrough", "retryable_writes_jscore_passthrough", - "retryable_writes_jscore_stepdown_passthrough", "secondary_reads_passthrough", - "session_jscore_passthrough", "write_concern_majority_passthrough" - ] -} - - -class TestAcceptance(unittest.TestCase): - @patch(ns("write_file_to_dir")) - @patch(ns("_create_evg_build_variant_map")) - @patch(ns("EvergreenFileChangeDetector")) - def test_no_tests_run_if_none_changed(self, find_changed_tests_mock, - create_evg_build_variant_map_mock, write_to_file_mock): - """ - Given a git repository with no changes, - When burn_in_tags is run, - Then no tests are discovered to run. - """ - repos = [MagicMock(working_dir=os.getcwd())] - evg_conf_mock = MagicMock() - find_changed_tests_mock.return_value.find_changed_tests.return_value = {} - - create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP - - under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf_mock, MagicMock(), repos, - 'install_dir/bin') - - write_to_file_mock.assert_called_once() - shrub_config = write_to_file_mock.call_args[0][2] - self.assertEqual(EMPTY_PROJECT, json.loads(shrub_config)) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("write_file_to_dir")) - @patch(ns("_create_evg_build_variant_map")) - @patch(ns("EvergreenFileChangeDetector")) - @patch(burn_in_tests_ns("create_test_membership_map")) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_tests_generated_if_a_file_changed( - self, get_stats_from_s3_mock, create_test_membership_map_mock, find_changed_tests_mock, - create_evg_build_variant_map_mock, write_to_file_mock): - """ - Given a git repository with changes, - When burn_in_tags is run, - Then some tags are discovered to run. - """ - create_test_membership_map_mock.return_value = defaultdict(list, CREATE_TEST_MEMBERSHIP_MAP) - - repos = [MagicMock(working_dir=os.getcwd())] - evg_conf = get_evergreen_config() - create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP - find_changed_tests_mock.return_value.find_changed_tests.return_value = { - 'jstests/slow1/large_role_chain.js', - 'jstests/aggregation/accumulators/accumulator_js.js' - } - get_stats_from_s3_mock.return_value = [] - - under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin') - - write_to_file_mock.assert_called_once() - written_config = write_to_file_mock.call_args[0][2] - written_config_map = json.loads(written_config) - - n_tasks = len(written_config_map["tasks"]) - # Ensure we are generating at least one task for the test. - self.assertGreaterEqual(n_tasks, 1) - - written_build_variants = written_config_map["buildvariants"] - written_build_variants_name = [variant['name'] for variant in written_build_variants] - self.assertEqual( - set(CREATE_EVG_BUILD_VARIANT_MAP.values()), set(written_build_variants_name)) - - tasks = written_config_map["tasks"] - self.assertGreaterEqual(len(tasks), len(CREATE_EVG_BUILD_VARIANT_MAP)) - - self.assertTrue( - all( - len(display_tasks) == 1 for display_tasks in - [build_variant["display_tasks"] for build_variant in written_build_variants])) diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml deleted file mode 100644 index 685f79e9e0e..00000000000 --- a/buildscripts/tests/test_burn_in_tags_evergreen.yml +++ /dev/null @@ -1,134 +0,0 @@ -functions: - "fetch source": - - command: git.get_project - params: - directory: src - - command: shell.exec - params: - working_dir: src - script: | - echo "this is a 2nd command in the function!" - ls - -tasks: -- name: compile - depends_on: [] - commands: - - func: "fetch source" -- name: burn_in_tags_gen - depends_on: [] - commands: - - func: "fake command" -- name: compile_all_run_unittests_TG - depends_on: [] - commands: - - func: "fake command" -- name: clang_tidy_TG - depends_on: [] - commands: - - func: "fake command" -- name: stitch_support_lib_build_and_archive - depends_on: [] - commands: - - func: "fake command" -- name: lint_pylinters - depends_on: [] - commands: - - func: "fake command" -- name: lint_clang_format - depends_on: [] - commands: - - func: "fake command" -- name: burn_in_tests_gen - depends_on: [] - commands: - - func: "fake command" -- name: aggregation_multiversion_fuzzer_gen - depends_on: [] - commands: - - func: "generate resmoke tasks" -- name: aggregation_expression_multiversion_fuzzer_gen - depends_on: [] - commands: - - func: "generate resmoke tasks" -- name: aggregation - depends_on: - - name: compile - commands: - - func: run tests - vars: - resmoke_args: --suites=aggregation --storageEngine=wiredTiger - -buildvariants: -- name: enterprise-rhel-80-64-bit - display_name: "! Enterprise RHEL 8.0" - expansions: - multiversion_platform: rhel80 - burn_in_tag_include_build_variants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: lint_pylinters - - name: burn_in_tests_gen - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation - - name: burn_in_tags_gen -- name: buildvariant-without-burn-in-tag-buildvariants - display_name: "Buildvariant without burn in tag buildvariants expansion" - expansions: - multiversion_platform: rhel80 - tasks: - - name: burn_in_tags_gen -- name: enterprise-rhel-80-64-bit-majority-read-concern-off - display_name: "Enterprise RHEL 8.0 (majority read concern off)" - run_on: - - rhel80-small - expansions: &enterprise-rhel-80-64-bit-majority-read-concern-off-expansions - multiversion_edition: enterprise - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation -- name: enterprise-rhel-80-64-bit-inmem - display_name: Enterprise RHEL 8.0 (inMemory) - run_on: - - rhel80-small - expansions: &enterprise-rhel-80-64-bit-inmem-expansions - test_flags: >- - --majorityReadConcern=off - --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime - compile_flags: >- - -j$(grep -c ^processor /proc/cpuinfo) - --ssl - --release - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - MONGO_DISTMOD=rhel80 - multiversion_platform: rhel80 - multiversion_edition: enterprise - scons_cache_scope: shared - tooltags: "ssl sasl gssapi" - large_distro_name: rhel80-large - tasks: - - name: compile -- name: enterprise-rhel-80-64-bit-inmem - display_name: Enterprise RHEL 8.0 (inMemory) - expansions: - additional_targets: archive-mongocryptd archive-mongocryptd-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - large_distro_name: rhel80-large - multiversion_edition: enterprise - multiversion_platform: rhel80 - scons_cache_scope: shared - test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling - run_on: - - rhel80-small - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 369131eff60..a5b9ba625d3 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,18 +4,19 @@ from __future__ import absolute_import import collections import datetime -from io import StringIO import os -import sys import subprocess +import sys import unittest +from io import StringIO -from mock import Mock, patch, MagicMock import yaml +from mock import MagicMock, Mock, patch import buildscripts.burn_in_tests as under_test -from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask import buildscripts.resmokelib.parser as _parser +from buildscripts.ciconfig.evergreen import parse_evergreen_file + _parser.set_run_options() # pylint: disable=missing-docstring,protected-access,too-many-lines,no-self-use @@ -23,10 +24,16 @@ _parser.set_run_options() def create_tests_by_task_mock(n_tasks, n_tests): return { - f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ - f"jstests/tests_{j}" for j in range(n_tests) - ], require_multiversion_setup=False, distro=f"distro_{i}", suite=f"suite_{i}", - build_variant="dummy_variant") + f"task_{i}_gen": under_test.TaskToBurnInInfo( + display_task_name=f"task_{i}", + suites=[ + under_test.SuiteToBurnInInfo( + name=f"suite_{i}", + resmoke_args="", + tests=[f"jstests/tests_{j}" for j in range(n_tests)], + ), + ], + ) for i in range(n_tasks) } @@ -294,9 +301,8 @@ def create_variant_task_mock(task_name, suite_name, distro="distro"): variant_task = MagicMock() variant_task.name = task_name variant_task.generated_task_name = task_name - variant_task.get_suite_name.return_value = suite_name - variant_task.resmoke_args = f"--suites={suite_name}" - variant_task.require_multiversion_setup.return_value = False + variant_task.get_suite_names.return_value = [suite_name] + variant_task.combined_suite_to_resmoke_args_map = {suite_name: f"--suites={suite_name}"} variant_task.run_on = [distro] return variant_task @@ -304,89 +310,9 @@ def create_variant_task_mock(task_name, suite_name, distro="distro"): class TestTaskInfo(unittest.TestCase): def test_non_generated_task(self): suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" evg_conf_mock = MagicMock() evg_conf_mock.get_task.return_value.is_generate_resmoke_task = False - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_no_large_on_task(self): - suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {}} - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_no_large_on_build_variant(self): - suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {"use_large_distro": True}} - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_large_distro(self): - suite_name = "suite_1" - distro_name = "distro_1" - large_distro_name = "large_distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {"use_large_distro": True}} - evg_conf_mock.get_variant.return_value.raw = { - "expansions": { - "large_distro_name": large_distro_name - } - } # yapf: disable - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) + task_mock = create_variant_task_mock("task 1", suite_name) test_list = [f"test{i}.js" for i in range(3)] tests_by_suite = { suite_name: test_list, @@ -394,13 +320,11 @@ class TestTaskInfo(unittest.TestCase): "suite 3": [f"test{i}.js" for i in range(2)], } - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) + task_info = under_test.TaskToBurnInInfo.from_task(task_mock, tests_by_suite) - self.assertIn(suite_name, task_info.resmoke_args) + self.assertIn(suite_name, task_info.suites[0].name) for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(large_distro_name, task_info.distro) + self.assertIn(test, task_info.suites[0].tests) class TestCreateTaskList(unittest.TestCase): @@ -444,11 +368,9 @@ class TestCreateTaskList(unittest.TestCase): self.assertIn("task 1", task_list) task_info = task_list["task 1"] - self.assertIn("suite_1", task_info.resmoke_args) + self.assertIn("suite_1", task_info.suites[0].resmoke_args) for i in range(3): - self.assertIn(f"test{i}.js", task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual("distro 1", task_info.distro) + self.assertIn(f"test{i}.js", task_info.suites[0].tests) def test_create_task_list_with_excludes(self): variant = "variant name" @@ -573,4 +495,4 @@ class TestYamlBurnInExecutor(unittest.TestCase): yaml_raw = stdout.getvalue() results = yaml.safe_load(yaml_raw) self.assertEqual(n_tasks, len(results["discovered_tasks"])) - self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) + self.assertEqual(n_tests, len(results["discovered_tasks"][0]["suites"][0]["test_list"])) diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py deleted file mode 100644 index 3ea78039930..00000000000 --- a/buildscripts/tests/test_evergreen_burn_in_tests.py +++ /dev/null @@ -1,399 +0,0 @@ -"""Unit tests for buildscripts/burn_in_tests.py.""" - -from __future__ import absolute_import - -import json -import os -import sys -import unittest -from datetime import datetime, timedelta -from math import ceil - -import requests -from mock import patch, MagicMock -from shrub.v2 import BuildVariant, ShrubProject -from evergreen.api import EvergreenApi - -import buildscripts.evergreen_burn_in_tests as under_test -from buildscripts.ciconfig.evergreen import parse_evergreen_file -import buildscripts.resmokelib.parser as _parser -import buildscripts.resmokelib.config as _config -import buildscripts.util.teststats as teststats_utils -_parser.set_run_options() - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access - -NS = "buildscripts.evergreen_burn_in_tests" - - -def ns(relative_name): # pylint: disable=invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def mock_a_file(filename): - change = MagicMock(a_path=filename) - return change - - -def mock_git_diff(change_list): - diff = MagicMock() - diff.iter_change_type.return_value = change_list - return diff - - -def mock_changed_git_files(add_files): - repo = MagicMock() - repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) - repo.working_dir = "." - return repo - - -def get_evergreen_config(config_file_path): - evergreen_home = os.path.expanduser(os.path.join("~", "evergreen")) - if os.path.exists(evergreen_home): - return parse_evergreen_file(config_file_path, evergreen_home) - return parse_evergreen_file(config_file_path) - - -class TestAcceptance(unittest.TestCase): - def tearDown(self): - _parser.set_run_options() - - @patch(ns("write_file")) - def test_no_tests_run_if_none_changed(self, write_json_mock): - """ - Given a git repository with no changes, - When burn_in_tests is run, - Then no tests are discovered to run. - """ - variant = "build_variant" - repos = [mock_changed_git_files([])] - repeat_config = under_test.RepeatConfig() - gen_config = under_test.GenerateConfig( - variant, - "project", - ) # yapf: disable - mock_evg_conf = MagicMock() - mock_evg_conf.get_task_names_by_tag.return_value = set() - mock_evg_api = MagicMock() - - under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, - mock_evg_conf, repos, "testfile.json", "install-dir/bin") - - write_json_mock.assert_called_once() - written_config = json.loads(write_json_mock.call_args[0][1]) - display_task = written_config["buildvariants"][0]["display_tasks"][0] - self.assertEqual(1, len(display_task["execution_tasks"])) - self.assertEqual(under_test.BURN_IN_TESTS_GEN_TASK, display_task["execution_tasks"][0]) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("write_file")) - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_tests_generated_if_a_file_changed(self, get_stats_from_s3_mock, write_json_mock): - """ - Given a git repository with changes, - When burn_in_tests is run, - Then tests are discovered to run. - """ - # Note: this test is using actual tests and suites. So changes to those suites could - # introduce failures and require this test to be updated. - # You can see the test file it is using below. This test is used in the 'auth' and - # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. - variant = "enterprise-rhel-80-64-bit-inmem" - repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] - repeat_config = under_test.RepeatConfig() - gen_config = under_test.GenerateConfig( - variant, - "project", - ) # yapf: disable - mock_evg_conf = get_evergreen_config("etc/evergreen.yml") - mock_evg_api = MagicMock() - get_stats_from_s3_mock.return_value = [] - - under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, - mock_evg_conf, repos, "testfile.json", 'install-dir/bin') - - write_json_mock.assert_called_once() - written_config = json.loads(write_json_mock.call_args[0][1]) - n_tasks = len(written_config["tasks"]) - # Ensure we are generating at least one task for the test. - self.assertGreaterEqual(n_tasks, 1) - - written_build_variant = written_config["buildvariants"][0] - self.assertEqual(variant, written_build_variant["name"]) - self.assertEqual(n_tasks, len(written_build_variant["tasks"])) - - display_task = written_build_variant["display_tasks"][0] - # The display task should contain all the generated tasks as well as 1 extra task for - # the burn_in_test_gen task. - self.assertEqual(n_tasks + 1, len(display_task["execution_tasks"])) - - -class TestGenerateConfig(unittest.TestCase): - def test_run_build_variant_with_no_run_build_variant(self): - gen_config = under_test.GenerateConfig("build_variant", "project") - - self.assertEqual(gen_config.build_variant, gen_config.run_build_variant) - - def test_run_build_variant_with_run_build_variant(self): - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - - self.assertNotEqual(gen_config.build_variant, gen_config.run_build_variant) - self.assertEqual(gen_config.run_build_variant, "run_build_variant") - - def test_validate_non_existing_build_variant(self): - evg_conf_mock = MagicMock() - evg_conf_mock.get_variant.return_value = None - - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - - with self.assertRaises(ValueError): - gen_config.validate(evg_conf_mock) - - def test_validate_existing_build_variant(self): - evg_conf_mock = MagicMock() - - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - gen_config.validate(evg_conf_mock) - - def test_validate_non_existing_run_build_variant(self): - evg_conf_mock = MagicMock() - - gen_config = under_test.GenerateConfig("build_variant", "project") - gen_config.validate(evg_conf_mock) - - -class TestParseAvgTestRuntime(unittest.TestCase): - def test__parse_avg_test_runtime(self): - task_avg_test_runtime_stats = [ - teststats_utils.TestRuntime(test_name="dir/test1.js", runtime=30.2), - teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1) - ] - result = under_test._parse_avg_test_runtime("dir/test2.js", task_avg_test_runtime_stats) - self.assertEqual(result, 455.1) - - -class TestCalculateTimeout(unittest.TestCase): - def test__calculate_timeout(self): - avg_test_runtime = 455.1 - expected_result = ceil(avg_test_runtime * under_test.AVG_TEST_TIME_MULTIPLIER) - self.assertEqual(expected_result, under_test._calculate_timeout(avg_test_runtime)) - - def test__calculate_timeout_avg_is_less_than_min(self): - avg_test_runtime = 10 - self.assertEqual(under_test.MIN_AVG_TEST_TIME_SEC, - under_test._calculate_timeout(avg_test_runtime)) - - -class TestCalculateExecTimeout(unittest.TestCase): - def test__calculate_exec_timeout(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - avg_test_runtime = 455.1 - - exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) - - self.assertEqual(1771, exec_timeout) - - def test_average_timeout_greater_than_execution_time(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600, repeat_tests_min=2) - avg_test_runtime = 750 - - exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) - - # The timeout needs to be greater than the number of the test * the minimum number of runs. - minimum_expected_timeout = avg_test_runtime * repeat_config.repeat_tests_min - - self.assertGreater(exec_timeout, minimum_expected_timeout) - - -class TestGenerateTimeouts(unittest.TestCase): - def test__generate_timeouts(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)] - test_name = "dir/test2.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertEqual(timeout_info.exec_timeout, 1771) - self.assertEqual(timeout_info.timeout, 1366) - - def test__generate_timeouts_no_results(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [] - test_name = "dir/new_test.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertIsNone(timeout_info.cmd) - - def test__generate_timeouts_avg_runtime_is_zero(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [ - teststats_utils.TestRuntime(test_name="dir/test_with_zero_runtime.js", runtime=0) - ] - test_name = "dir/test_with_zero_runtime.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertIsNone(timeout_info.cmd) - - -class TestGetTaskRuntimeHistory(unittest.TestCase): - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_get_task_runtime_history(self, get_stats_from_s3_mock): - test_stats = [ - teststats_utils.HistoricalTestInformation( - test_name="dir/test2.js", - num_pass=1, - num_fail=0, - avg_duration_pass=10.1, - ) - ] - get_stats_from_s3_mock.return_value = test_stats - mock_gen_config = MagicMock(project="project1", build_variant="variant1") - - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) - result = executor.get_task_runtime_history("task1") - - self.assertEqual(result, [("dir/test2.js", 10.1)]) - - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_get_task_runtime_history_when_s3_has_no_data(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] - mock_gen_config = MagicMock(project="project1", build_variant="variant1") - - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) - result = executor.get_task_runtime_history("task1") - - self.assertEqual(result, []) - - -TESTS_BY_TASK = { - "task1": { - "resmoke_args": "--suites=suite1", - "tests": ["jstests/test1.js", "jstests/test2.js"]}, - "task2": { - "resmoke_args": "--suites=suite1", - "tests": ["jstests/test1.js", "jstests/test3.js"]}, - "task3": { - "resmoke_args": "--suites=suite3", - "tests": ["jstests/test4.js", "jstests/test5.js"]}, - "task4": { - "resmoke_args": "--suites=suite4", "tests": []}, -} # yapf: disable - - -def create_tests_by_task_mock(n_tasks, n_tests): - return { - f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ - f"jstests/tests_{j}" for j in range(n_tests) - ], require_multiversion_setup=False, distro=f"distro_{i}", build_variant="variant", - suite=f"suite_{i}") - for i in range(n_tasks) - } - - -class TestCreateGenerateTasksConfig(unittest.TestCase): - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_no_tasks_given(self): - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant") - repeat_config = MagicMock() - mock_evg_api = MagicMock() - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) - executor.generate_tasks_for_variant({}, build_variant) - - evg_config_dict = build_variant.as_dict() - self.assertEqual(0, len(evg_config_dict["tasks"])) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_one_task_one_test(self, get_stats_from_s3_mock): - n_tasks = 1 - n_tests = 1 - resmoke_options = "options for resmoke" - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant", distro=None) - repeat_config = MagicMock() - repeat_config.generate_resmoke_options.return_value = resmoke_options - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - get_stats_from_s3_mock.return_value = [] - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) - executor.generate_tasks_for_variant(tests_by_task, build_variant) - - shrub_config = ShrubProject.empty().add_build_variant(build_variant) - evg_config_dict = shrub_config.as_dict() - tasks = evg_config_dict["tasks"] - self.assertEqual(n_tasks * n_tests, len(tasks)) - cmd = tasks[0]["commands"] - self.assertIn(resmoke_options, cmd[2]["vars"]["resmoke_args"]) - self.assertEqual("suite_0", cmd[2]["vars"]["suite"]) - self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"]) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_n_task_m_test(self, get_stats_from_s3_mock): - n_tasks = 3 - n_tests = 5 - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant", distro=None) - repeat_config = MagicMock() - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - get_stats_from_s3_mock.return_value = [] - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) - executor.generate_tasks_for_variant(tests_by_task, build_variant) - - evg_config_dict = build_variant.as_dict() - self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"])) - - -class TestCreateGenerateTasksFile(unittest.TestCase): - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("sys.exit")) - @patch(ns("validate_task_generation_limit")) - def test_cap_on_task_generate(self, validate_mock, exit_mock): - gen_config = MagicMock(require_multiversion_setup=False) - repeat_config = MagicMock() - tests_by_task = MagicMock() - - validate_mock.return_value = False - - exit_mock.side_effect = ValueError("exiting") - with self.assertRaises(ValueError): - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, "gen_file.json") - executor.execute(tests_by_task) - - exit_mock.assert_called_once() - - -class TestFindChangedTests(unittest.TestCase): - def test_manual_tests_should_be_specifiable_via_env_vars(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_repo = MagicMock() - mock_env = { - "BURN_IN_TESTS": "jstests/auth/auth1.js,jstests/core/where1.js", - } - change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) - - test_set = change_detector.find_changed_tests([mock_repo]) - - self.assertEqual(2, len(test_set)) - self.assertIn("jstests/auth/auth1.js", test_set) - self.assertIn("jstests/core/where1.js", test_set) - - def test_empty_env_should_not_add_extra_tests(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_repo = MagicMock() - mock_env = {} - change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) - - test_set = change_detector.find_changed_tests([mock_repo]) - - self.assertEqual(set(), test_set) diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py deleted file mode 100644 index 7ece404ab36..00000000000 --- a/buildscripts/tests/test_evergreen_gen_build_variant.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Unit tests for the generate_resmoke_suite script.""" -import unittest - -from mock import MagicMock - -from buildscripts import evergreen_gen_build_variant as under_test -from buildscripts.ciconfig.evergreen import Variant, Task - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access -# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter - - -def build_mock_build_variant(expansions=None, task_list=None): - task_spec_list = [{"name": task.name} for task in task_list] if task_list else [] - config = { - "tasks": task_spec_list, - } - if expansions: - config["expansions"] = expansions - - if task_list is None: - task_list = [] - task_map = {task.name: task for task in task_list} - - return Variant(config, task_map, {}) - - -def build_mock_task(name, run_vars=None, depends_on=None): - config = { - "name": - name, "commands": [ - {"func": "do setup"}, - { - "func": "generate resmoke tasks", - "vars": run_vars if run_vars else {}, - }, - ] - } - - if depends_on is not None: - config["depends_on"] = depends_on - return Task(config) - - -def build_mock_project_config(variant=None, task_defs=None): - mock_project = MagicMock() - if variant: - mock_project.get_variant.return_value = variant - - if task_defs: - mock_project.get_task.side_effect = task_defs - - return mock_project - - -def build_mock_expansions(): - mock_expansions = MagicMock() - mock_expansions.config_location.return_value = "/path/to/config" - mock_expansions.get_max_sub_suites.return_value = 998 - mock_expansions.task_name = "generating_task" - return mock_expansions - - -def build_mock_evg_api(build_task_list): - mock_evg_api = MagicMock() - mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list - return mock_evg_api - - -def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None): - if build_expansions is None: - build_expansions = {} - if task_def_list is None: - task_def_list = [] - if build_task_list is None: - build_task_list = [] - - mock_build_variant = build_mock_build_variant(build_expansions, task_def_list) - mock_project = build_mock_project_config(mock_build_variant, task_def_list) - mock_evg_expansions = build_mock_expansions() - mock_evg_api = build_mock_evg_api(build_task_list) - - return under_test.GenerateBuildVariantOrchestrator( - gen_task_validation=MagicMock(), - gen_task_options=MagicMock(), - evg_project_config=mock_project, - evg_expansions=mock_evg_expansions, - evg_api=mock_evg_api, - ) - - -class TestEvgExpansions(unittest.TestCase): - def test_get_max_sub_suites_should_use_patch_value_in_patches(self): - evg_expansions = under_test.EvgExpansions( - is_patch=True, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), evg_expansions.max_sub_suites) - - def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self): - evg_expansions = under_test.EvgExpansions( - is_patch=False, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), - evg_expansions.mainline_max_sub_suites) - - def test_get_max_sub_suites_should_use_mainline_value_if_patch_status_unknown(self): - evg_expansions = under_test.EvgExpansions( - is_patch=None, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), - evg_expansions.mainline_max_sub_suites) - - -class TestTranslateRunVar(unittest.TestCase): - def test_normal_value_should_be_returned(self): - run_var = "some value" - mock_build_variant = build_mock_build_variant() - self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_should_be_returned_from_build_variant(self): - run_var = "${my_expansion}" - value = "my value" - mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) - self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_not_found_should_return_none(self): - run_var = "${my_expansion}" - mock_build_variant = build_mock_build_variant(expansions={}) - self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_not_found_should_return_default(self): - run_var = "${my_expansion|default}" - mock_build_variant = build_mock_build_variant(expansions={}) - self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_should_be_returned_from_build_variant_even_with_default(self): - run_var = "${my_expansion|default}" - value = "my value" - mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) - self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) - - -class TestTaskDefToSplitParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "resmoke_args": "run tests", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") - - self.assertEqual("build_variant", split_param.build_variant) - self.assertEqual("my_task", split_param.task_name) - self.assertEqual("my_task", split_param.suite_name) - self.assertEqual("my_task", split_param.filename) - - def test_params_should_allow_suite_to_be_overridden(self): - run_vars = { - "resmoke_args": "run tests", - "suite": "the suite", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") - - self.assertEqual("build_variant", split_param.build_variant) - self.assertEqual("my_task", split_param.task_name) - self.assertEqual("the suite", split_param.suite_name) - self.assertEqual("the suite", split_param.filename) - - -class TestDetermineTaskDependencies(unittest.TestCase): - def test_running_task_should_not_be_included_in_depends_on(self): - run_vars = { - "resmoke_args": "run tests", - } - depends_on = [ - {"name": "compile"}, - {"name": "build_variant_gen"}, - ] - mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def) - - self.assertIn("compile", dependencies) - self.assertNotIn("generating_task", dependencies) - - -class TestTaskDefToGenParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "resmoke_args": "run tests", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") - - self.assertFalse(gen_params.require_multiversion_setup) - self.assertEqual("run tests", gen_params.resmoke_args) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - gen_params.config_location) - self.assertIsNone(gen_params.large_distro_name) - self.assertFalse(gen_params.use_large_distro) - - def test_params_should_be_overwritable(self): - run_vars = { - "resmoke_args": "run tests", - "use_large_distro": "true", - } - mock_task_def = build_mock_task("my_task", run_vars) - build_expansions = {"large_distro_name": "my large distro"} - mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, - task_def_list=[mock_task_def]) - gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") - - self.assertFalse(gen_params.require_multiversion_setup) - self.assertEqual("run tests", gen_params.resmoke_args) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - gen_params.config_location) - self.assertEqual("my large distro", gen_params.large_distro_name) - self.assertTrue(gen_params.use_large_distro) - - -class TestTaskDefToFuzzerParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "num_files": "5", - "num_tasks": "3", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual("my_fuzzer", fuzzer_params.task_name) - self.assertEqual(5, fuzzer_params.num_files) - self.assertEqual(3, fuzzer_params.num_tasks) - self.assertEqual("jstestfuzz", fuzzer_params.npm_command) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - fuzzer_params.config_location) - self.assertIsNone(fuzzer_params.large_distro_name) - self.assertFalse(fuzzer_params.use_large_distro) - - def test_num_tasks_respects_max_sub_suites(self): - run_vars = { - "num_files": "5", - "num_tasks": "3", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - mock_orchestrator.evg_expansions.get_max_sub_suites.return_value = 1 - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual(5, fuzzer_params.num_files) - self.assertEqual(1, fuzzer_params.num_tasks) - - def test_params_should_be_overwritable(self): - run_vars = { - "num_files": "${file_count|8}", - "num_tasks": "3", - "use_large_distro": "true", - "npm_command": "aggfuzzer", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - build_expansions = {"large_distro_name": "my large distro"} - mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, - task_def_list=[mock_task_def]) - - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual("my_fuzzer", fuzzer_params.task_name) - self.assertEqual(8, fuzzer_params.num_files) - self.assertEqual(3, fuzzer_params.num_tasks) - self.assertEqual("aggfuzzer", fuzzer_params.npm_command) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - fuzzer_params.config_location) - self.assertEqual("my large distro", fuzzer_params.large_distro_name) - self.assertTrue(fuzzer_params.use_large_distro) - - -class TestGenerateBuildVariant(unittest.TestCase): - def test_a_whole_build_variant(self): - gen_run_vars = { - "resmoke_args": "run tests", - } - mv_gen_run_vars = { - "resmoke_args": "run tests", - "suite": "some suite", - } - fuzz_run_vars = { - "num_files": "5", - "num_tasks": "3", - "is_jstestfuzz": "true", - } - mv_fuzz_run_vars = { - "num_files": "5", - "num_tasks": "3", - "is_jstestfuzz": "true", - "suite": "aggfuzzer", - } - mock_task_defs = [ - build_mock_task("my_gen_task", gen_run_vars), - build_mock_task("my_fuzzer_task", fuzz_run_vars), - build_mock_task("my_mv_fuzzer_task", mv_fuzz_run_vars), - build_mock_task("my_mv_gen_task", mv_gen_run_vars), - ] - mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs) - builder = MagicMock() - - builder = mock_orchestrator.generate_build_variant(builder, "build variant") - - self.assertEqual(builder.generate_suite.call_count, 2) - self.assertEqual(builder.generate_fuzzer.call_count, 2) - - -class TestAdjustTaskPriority(unittest.TestCase): - def test_task_is_updates(self): - starting_priority = 42 - task_id = "task 314" - mock_task = MagicMock(task_id=task_id, priority=starting_priority) - mock_orchestrator = build_mock_orchestrator() - - mock_orchestrator.adjust_task_priority(mock_task) - - mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, - priority=starting_priority + 1) - - def test_task_should_only_reach_99(self): - starting_priority = 99 - task_id = "task 314" - mock_task = MagicMock(task_id=task_id, priority=starting_priority) - mock_orchestrator = build_mock_orchestrator() - - mock_orchestrator.adjust_task_priority(mock_task) - - mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, - priority=starting_priority) - - -class TestAdjustGenTasksPriority(unittest.TestCase): - def test_gen_tasks_in_task_list_are_adjusted(self): - gen_tasks = {"task_3", "task_8", "task_13"} - n_build_tasks = 25 - mock_task_list = [ - MagicMock(build_variant='dummy_variant', display_name=f"task_{i}", priority=0) - for i in range(n_build_tasks) - ] - mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list) - - n_tasks_adjusted = mock_orchestrator.adjust_gen_tasks_priority(gen_tasks) - - self.assertEqual(len(gen_tasks), n_tasks_adjusted) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 819d963ac48..c59952a8312 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -11,7 +11,7 @@ from evergreen import EvergreenApi # pylint: disable=wrong-import-position import buildscripts.ciconfig.evergreen as _evergreen -from buildscripts.burn_in_tests import TaskInfo +from buildscripts.burn_in_tests import TaskToBurnInInfo, SuiteToBurnInInfo from buildscripts.patch_builds.selected_tests.selected_tests_client import SelectedTestsClient, \ TestMappingsResponse, TestMapping, TestFileInstance, TaskMappingsResponse, TaskMapInstance, \ TaskMapping @@ -135,7 +135,7 @@ class TestAcceptance(unittest.TestCase): # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file - self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate) + self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list @@ -147,7 +147,7 @@ class TestAcceptance(unittest.TestCase): # jstests/auth/auth1.js belongs to two suites, auth and auth_audit, rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @@ -168,7 +168,7 @@ class TestAcceptance(unittest.TestCase): mock_task_mapping = TaskMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, - tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)]) + tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-8", flip_count=5)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse( task_mappings=[mock_task_mapping]) @@ -194,7 +194,7 @@ class TestAcceptance(unittest.TestCase): build_variants_with_generated_tasks = generated_evg_config["buildvariants"] rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5) @@ -274,27 +274,29 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): exclude_task_mock.return_value = False tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core_auth", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core_auth", - build_variant="dummy_variant", ), "auth_gen": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 2", - tests=["jstests/auth/auth3.js"], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="auth", - build_variant="dummy_variant", + suites=[ + SuiteToBurnInInfo( + name="auth", + resmoke_args="", + tests=["jstests/auth/auth3.js"], + ), + ], ), } @@ -319,17 +321,18 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): exclude_task_mock.return_value = True tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core", - build_variant="dummy_variant", ), } @@ -344,17 +347,18 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): find_task_mock.return_value = None tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core", - build_variant="dummy_variant", ), } diff --git a/buildscripts/tests/test_validate_commit_message.py b/buildscripts/tests/test_validate_commit_message.py index db341a5add1..1bf0039188d 100644 --- a/buildscripts/tests/test_validate_commit_message.py +++ b/buildscripts/tests/test_validate_commit_message.py @@ -1,130 +1,31 @@ """Unit tests for the evergreen_task_timeout script.""" -import itertools import unittest -from typing import List, Optional -from unittest.mock import MagicMock -import buildscripts.validate_commit_message as under_test -from buildscripts.client.jiraclient import JiraClient, SecurityLevel -from evergreen import EvergreenApi +from buildscripts.validate_commit_message import main, STATUS_OK, STATUS_ERROR # pylint: disable=missing-docstring,no-self-use -INVALID_MESSAGES = [ - "", # You must provide a message - "RevertEVG-1", # revert and ticket must be formatted - "revert EVG-1", # revert must be capitalized - "This is not a valid message", # message must be valid - "Fix Lint", # Fix lint is strict in terms of caps -] - - -def create_mock_code_change(code_change_messages: List[str], branch_name: Optional[str] = None): - mock_code_change = MagicMock( - commit_messages=code_change_messages, - branch_name=branch_name if branch_name else "mongodb-mongo-master", - ) - return mock_code_change - - -def create_mock_patch(code_change_messages: List[str], branch_name: Optional[str] = None): - mock_code_change = create_mock_code_change(code_change_messages, branch_name) - mock_patch = MagicMock(module_code_changes=[mock_code_change]) - return mock_patch - - -def create_mock_evg_client(code_change_messages: List[str], - branch_name: Optional[str] = None) -> MagicMock: - mock_patch = create_mock_patch(code_change_messages, branch_name) - - mock_evg_client = MagicMock(spec_set=EvergreenApi) - mock_evg_client.patch_by_id.return_value = mock_patch - return mock_evg_client - - -def create_mock_jira_client(): - mock_jira = MagicMock(spec_set=JiraClient) - mock_jira.get_ticket_security_level.return_value = SecurityLevel.NONE - return mock_jira - - -def interleave_new_format(older): - """Create a new list containing a new and old format copy of each string.""" - newer = [ - f"Commit Queue Merge: '{old}' into 'mongodb/mongo:SERVER-45949-validate-message-format'" - for old in older - ] - return list(itertools.chain(*zip(older, newer))) - class ValidateCommitMessageTest(unittest.TestCase): - def test_valid_commits(self): + def test_valid(self): messages = [ - "Fix lint", - "EVG-1", # Test valid projects with various number lengths - "SERVER-20", - "WT-300", "SERVER-44338", - "Revert EVG-5", - "Revert SERVER-60", - "Revert WT-700", - "Revert 'SERVER-8000", - 'Revert "SERVER-90000', + "Revert \"SERVER-60", "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", - "Import tools: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", 'Revert "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4"', ] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_OK) - def test_private(self): - messages = ["XYZ-1"] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + self.assertTrue(all(main([message]) == STATUS_OK for message in messages)) - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_private_with_public(self): + def test_invalid(self): messages = [ - "Fix lint", - "EVG-1", # Test valid projects with various number lengths - "SERVER-20", - "XYZ-1", + "SERVER-", # missing number + "Revert SERVER-60", # missing quote before SERVER + "", # empty value + "nonsense", # nonsense values ] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_internal_ticket_to_public_repo_should_fail(self): - message = "SERVER-20" - mock_evg_api = create_mock_evg_client(interleave_new_format([message])) - mock_jira = create_mock_jira_client() - mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_internal_ticket_to_private_repo_should_succeed(self): - message = "SERVER-20" - mock_evg_api = create_mock_evg_client(interleave_new_format([message]), "private-repo") - mock_jira = create_mock_jira_client() - mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - is_valid = orchestrator.validate_commit_messages("version_id") + self.assertTrue(all(main([message]) == STATUS_ERROR for message in messages)) - self.assertEqual(is_valid, under_test.STATUS_OK) + def test_message_is_empty_list(self): + self.assertEqual(main([]), STATUS_ERROR) diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py index 4bd8dff252a..23e9c1f331f 100644 --- a/buildscripts/tests/timeouts/test_timeout_service.py +++ b/buildscripts/tests/timeouts/test_timeout_service.py @@ -126,7 +126,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) @patch(ns("HistoricTaskData.from_s3")) def test_enough_history_but_some_tests_with_zero_runtime_should_cause_custom_task_and_default_test_timeout( @@ -155,7 +155,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) @patch(ns("HistoricTaskData.from_s3")) def test_all_tests_with_runtime_history_should_use_custom_timeout(self, @@ -180,7 +180,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(1860, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) class TestGetTaskHookOverhead(unittest.TestCase): diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py index 7f3296ca1aa..637ee59321e 100644 --- a/buildscripts/tests/util/test_taskname.py +++ b/buildscripts/tests/util/test_taskname.py @@ -28,7 +28,7 @@ class TestRemoveGenSuffix(unittest.TestCase): class TestDetermineTaskBaseName(unittest.TestCase): def test_task_name_with_build_variant_should_strip_bv_and_sub_task_index(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = f"auth_23_{bv}" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -36,7 +36,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_should_strip_sub_task_index(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = "auth_314" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -44,7 +44,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_or_subtask_index_should_self(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = "auth" base_task_name = under_test.determine_task_base_name(task_name, bv) diff --git a/buildscripts/timeouts/timeout.py b/buildscripts/timeouts/timeout.py index e3faa940fc5..36f08db58df 100644 --- a/buildscripts/timeouts/timeout.py +++ b/buildscripts/timeouts/timeout.py @@ -9,7 +9,7 @@ from buildscripts.patch_builds.task_generation import TimeoutInfo LOGGER = structlog.getLogger(__name__) -AVG_TASK_SETUP_TIME = int(timedelta(minutes=2).total_seconds()) +AVG_TASK_SETUP_TIME = int(timedelta(minutes=5).total_seconds()) MIN_TIMEOUT_SECONDS = int(timedelta(minutes=5).total_seconds()) MAX_EXPECTED_TIMEOUT = int(timedelta(hours=48).total_seconds()) DEFAULT_SCALING_FACTOR = 3.0 diff --git a/buildscripts/validate_commit_message.py b/buildscripts/validate_commit_message.py index 114a799dd0c..da01fa75a62 100755 --- a/buildscripts/validate_commit_message.py +++ b/buildscripts/validate_commit_message.py @@ -28,270 +28,46 @@ # """Validate that the commit message is ok.""" import argparse -import logging -import os import re import sys -from typing import List, Optional - -from evergreen import EvergreenApi, RetryingEvergreenApi - -from buildscripts.client.jiraclient import JiraAuth, JiraClient, SecurityLevel - -JIRA_SERVER = "https://jira.mongodb.org" -EVG_CONFIG_FILE = "~/.evergreen.yml" -SERVER_TICKET_PREFIX = "SERVER-" -PUBLIC_PROJECT_PREFIX = "mongodb-mongo-" +import logging LOGGER = logging.getLogger(__name__) -ERROR_MSG = """ -################################################################################ -Encountered an invalid commit message. Please correct to the commit message to -continue. - -Commit message should start with a Public Jira ticket, an "Import" for wiredtiger -or tools, or a "Revert" message. - -{error_msg} on '{branch}': -'{commit_message}' -################################################################################ -""" - -COMMON_PUBLIC_PATTERN = r""" - ((?P<revert>Revert)\s+[\"\']?)? # Revert (optional) - ((?P<ticket>(?:EVG|SERVER|WT)-[0-9]+)[\"\']?\s*) # ticket identifier - (?P<body>(?:(?!\(cherry\spicked\sfrom).)*)? # To also capture the body - (?P<backport>\(cherry\spicked\sfrom.*)? # back port (optional) - """ -"""Common Public pattern format.""" - -COMMON_10GENREPO_COMMIT_QUEUE_PATTERN = r' ^\'(?P<repo>10gen/mongo)\'\s.*commit\squeue\smerge.*SERVER-[0-9]+' -"""Common commit queue format.""" - -COMMON_LINT_PATTERN = r"(?P<lint>Fix\slint)" -"""Common Lint pattern format.""" - -COMMON_IMPORT_PATTERN = r"(?P<imported>Import\s(wiredtiger|tools):\s.*)" -"""Common Import pattern format.""" - -COMMON_REVERT_IMPORT_PATTERN = (r"Revert\s+[\"\']?(?P<imported>Import\s(wiredtiger|tools):\s.*)") -"""Common revert Import pattern format.""" - -COMMON_PRIVATE_PATTERN = r""" - ((?P<revert>Revert)\s+[\"\']?)? # Revert (optional) - ((?P<ticket>[A-Z]+-[0-9]+)[\"\']?\s*) # ticket identifier - (?P<body>(?:(?!('\s(into\s'(([^/]+))/(([^:]+)):(([^']+))'))).)*)? # To also capture the body -""" -"""Common Private pattern format.""" - STATUS_OK = 0 STATUS_ERROR = 1 -def new_patch_description(pattern: str) -> str: - """ - Wrap the pattern to conform to the new commit queue patch description format. - - Add the commit queue prefix and suffix to the pattern. The format looks like: - - Commit Queue Merge: '<commit message>' into '<owner>/<repo>:<branch>' - - :param pattern: The pattern to wrap. - :return: A pattern to match the new format for the patch description. - """ - return (r"""^((?P<commitqueue>Commit\sQueue\sMerge:)\s')""" - f"{pattern}" - # r"""('\s(?P<into>into\s'((?P<owner>[^/]+))/((?P<repo>[^:]+)):((?P<branch>[^']+))'))""" - ) - - -def old_patch_description(pattern: str) -> str: - """ - Wrap the pattern to conform to the new commit queue patch description format. - - Just add a start anchor. The format looks like: - - <commit message> - - :param pattern: The pattern to wrap. - :return: A pattern to match the old format for the patch description. - """ - return r"^" f"{pattern}" - - -# NOTE: re.VERBOSE is for visibility / debugging. As such significant white space must be -# escaped (e.g ' ' to \s). -COMMON_PUBLIC_PATTERNS = [ - re.compile( - new_patch_description(COMMON_PUBLIC_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - old_patch_description(COMMON_PUBLIC_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), -] -"""common public patterns.""" - -VALID_PATTERNS = [ - re.compile( - new_patch_description(COMMON_LINT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - old_patch_description(COMMON_LINT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - new_patch_description(COMMON_IMPORT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - old_patch_description(COMMON_IMPORT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - new_patch_description(COMMON_REVERT_IMPORT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - old_patch_description(COMMON_REVERT_IMPORT_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), -] -"""valid public patterns.""" - -PRIVATE_PATTERNS = [ - re.compile( - new_patch_description(COMMON_PRIVATE_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - old_patch_description(COMMON_PRIVATE_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE, - ), - re.compile( - new_patch_description(COMMON_10GENREPO_COMMIT_QUEUE_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE), - re.compile( - old_patch_description(COMMON_10GENREPO_COMMIT_QUEUE_PATTERN), - re.MULTILINE | re.DOTALL | re.VERBOSE), -] -"""private patterns.""" - - -class CommitMessageValidationOrchestrator: - """An orchestrator to validate that commit messages are valid.""" - - def __init__(self, evg_api: EvergreenApi, jira_client: JiraClient) -> None: - """ - Initialize the orchestrator. - - :param evg_api: Evergreen API client. - :param jira_client: Client to Jira API. - """ - self.evg_api = evg_api - self.jira_client = jira_client - - def validate_ticket(self, ticket: str, project: str) -> bool: - """ - Check that the given Jira ticket has a proper security level. - - Commits targeting a public project should not have a defined security level (these are - public by default). - - :param ticket: Ticket to check. - :param project: Project commit is targeting. - :return: True if ticket is valid. - """ - if ticket.startswith(SERVER_TICKET_PREFIX) and project.startswith(PUBLIC_PROJECT_PREFIX): - security_level = self.jira_client.get_ticket_security_level(ticket) - return security_level == SecurityLevel.NONE - return True - - def validate_msg(self, message: str, project: str) -> bool: - """ - Check that the given message is valid. - - :param message: Commit message to validate. - :param project: Project commit is targeting. - :return: True if the message is valid. - """ - for pattern in COMMON_PUBLIC_PATTERNS: - match = pattern.match(message) - if not match: - continue - if not self.validate_ticket(match.group("ticket"), project): - print( - ERROR_MSG.format( - error_msg="Reference to a internal Jira Ticket", - branch=project, - commit_message=message, - )) - return False - return True - - valid_matches = [valid_pattern.match(message) for valid_pattern in VALID_PATTERNS] - if any(valid_matches): - return True - elif any(private_pattern.match(message) for private_pattern in PRIVATE_PATTERNS): - print( - ERROR_MSG.format( - error_msg="Reference to a private project", - branch=project, - commit_message=message, - )) - return False - else: - print( - ERROR_MSG.format( - error_msg="Commit without a ticket", - branch=project, - commit_message=message, - )) - return False - - def validate_commit_messages(self, version_id: str) -> int: - """ - Validate the commit messages for the given build. - - :param version_id: ID of version to validate. - :param evg_api: Evergreen API client. - :return: True if all commit messages were valid. - """ - found_error = False - code_changes = self.evg_api.patch_by_id(version_id).module_code_changes - for change in code_changes: - for message in change.commit_messages: - is_valid = self.validate_msg(message, change.branch_name) - found_error = found_error or not is_valid - - return STATUS_ERROR if found_error else STATUS_OK - - -def main(argv: Optional[List[str]] = None) -> int: +def main(argv=None): """Execute Main function to validate commit messages.""" parser = argparse.ArgumentParser( usage="Validate the commit message. " "It validates the latest message when no arguments are provided.") parser.add_argument( - "version_id", - metavar="version id", - help="The id of the version to validate", - ) - parser.add_argument( - "--evg-config-file", - default=EVG_CONFIG_FILE, - help="Path to evergreen configuration file containing auth information.", + "message", + metavar="commit message", + nargs="*", + help="The commit message to validate", ) args = parser.parse_args(argv) - evg_api = RetryingEvergreenApi.get_api(config_file=os.path.expanduser(args.evg_config_file)) - jira_auth = JiraAuth() - jira_client = JiraClient(JIRA_SERVER, jira_auth) - orchestrator = CommitMessageValidationOrchestrator(evg_api, jira_client) - return orchestrator.validate_commit_messages(args.version_id) + if not args.message: + LOGGER.error("Must specify non-empty value for --message") + return STATUS_ERROR + message = " ".join(args.message) + + # Valid values look like: + # 1. SERVER-\d+ + # 2. Revert "SERVER-\d+ + # 3. Import wiredtiger + # 4. Revert "Import wiredtiger + valid_pattern = re.compile(r'(Revert ")?(SERVER-[0-9]+|Import wiredtiger)') + + if valid_pattern.match(message): + return STATUS_OK + else: + LOGGER.error(f"Found a commit without a ticket\n{message}") # pylint: disable=logging-fstring-interpolation + return STATUS_ERROR if __name__ == "__main__": |