diff options
Diffstat (limited to 'buildscripts/resmokelib')
| -rw-r--r-- | buildscripts/resmokelib/config.py | 5 | ||||
| -rw-r--r-- | buildscripts/resmokelib/configure_resmoke.py | 25 | ||||
| -rw-r--r-- | buildscripts/resmokelib/core/programs.py | 37 | ||||
| -rw-r--r-- | buildscripts/resmokelib/hang_analyzer/dumper.py | 28 | ||||
| -rw-r--r-- | buildscripts/resmokelib/hang_analyzer/process.py | 13 | ||||
| -rw-r--r-- | buildscripts/resmokelib/logging/handlers.py | 23 | ||||
| -rw-r--r-- | buildscripts/resmokelib/logging/loggers.py | 32 | ||||
| -rw-r--r-- | buildscripts/resmokelib/multiversion/__init__.py | 3 | ||||
| -rw-r--r-- | buildscripts/resmokelib/multiversion/multiversion_service.py | 4 | ||||
| -rw-r--r-- | buildscripts/resmokelib/multiversionconstants.py | 14 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py | 112 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/hooks/background_job.py | 1 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/hooks/bghook.py | 34 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/hooks/run_query_stats.py | 83 | ||||
| -rw-r--r-- | buildscripts/resmokelib/utils/__init__.py | 2 |
15 files changed, 343 insertions, 73 deletions
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 039ff78dbc5..f3ab984756c 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -101,7 +101,7 @@ DEFAULTS = { "stagger_jobs": None, "majority_read_concern": "on", "storage_engine": "wiredTiger", - "enable_enterprise_tests": None, + "enable_enterprise_tests": "on", "storage_engine_cache_size_gb": None, "suite_files": "with_server", "tag_files": [], @@ -295,6 +295,9 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None +# if set, enables enterprise jstest to automatically be included +ENABLE_ENTERPRISE_TESTS = None + # URL to connect to the Evergreen service. EVERGREEN_URL = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index 35e79bbd72e..ed659d2f79b 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -1,5 +1,6 @@ """Configure the command line input for the resmoke 'run' subcommand.""" +import argparse import collections import configparser import datetime @@ -22,6 +23,7 @@ from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils from buildscripts.resmokelib import mongod_fuzzer_configs from buildscripts.resmokelib.suitesconfig import SuiteFinder +from buildscripts.util.read_config import read_config_file def validate_and_update_config(parser, args): @@ -520,3 +522,26 @@ def _update_symbolizer_secrets(): yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") + + +def detect_evergreen_config(parsed_args: argparse.Namespace, + expansions_file: str = "../expansions.yml"): + """Detect evergreen expansions.""" + if not os.path.exists(expansions_file): + return + + expansions = read_config_file(expansions_file) + + parsed_args.build_id = expansions.get("build_id", None) + parsed_args.distro_id = expansions.get("distro_id", None) + parsed_args.execution_number = expansions.get("execution", None) + parsed_args.project_name = expansions.get("project", None) + parsed_args.git_revision = expansions.get("revision", None) + parsed_args.revision_order_id = expansions.get("revision_order_id", None) + parsed_args.task_id = expansions.get("task_id", None) + parsed_args.task_name = expansions.get("task_name", None) + parsed_args.variant_name = expansions.get("build_variant", None) + parsed_args.version_id = expansions.get("version_id", None) + parsed_args.work_dir = expansions.get("workdir", None) + parsed_args.evg_project_config_path = expansions.get("evergreen_config_file_path", None) + parsed_args.requester = expansions.get("requester", None) diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index 2a22d177a29..fa9a2e98ebd 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -6,7 +6,9 @@ Handles all the nitty-gritty parameter conversion. import json import os import os.path +import re import stat +from packaging import version from buildscripts.resmokelib import config from buildscripts.resmokelib import utils @@ -43,6 +45,31 @@ def get_path_env_var(env_vars): return path +def get_binary_version(executable): + """Return the string for the binary version of the given executable.""" + + # pylint: disable=wrong-import-position + from buildscripts.resmokelib.multiversionconstants import LATEST_FCV + + split_executable = executable.split("-") + version_regex = re.compile(version.VERSION_PATTERN, re.VERBOSE | re.IGNORECASE) + if len(split_executable) > 1 and version_regex.match(split_executable[-1]): + return split_executable[-1] + return LATEST_FCV + + +def remove_set_parameter_if_before_version(set_parameters, parameter_name, bin_version, + required_bin_version): + """ + Used for removing a server parameter that does not exist prior to a specified version. + + Remove 'parameter_name' from the 'set_parameters' dictionary if 'bin_version' is older than + 'required_bin_version'. + """ + if version.parse(bin_version) < version.parse(required_bin_version): + set_parameters.pop(parameter_name, None) + + def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): """ Return a Process instance that starts mongod arguments constructed from 'mongod_options'. @@ -54,12 +81,17 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): @param mongod_options - A HistoryDict describing the various options to pass to the mongod. """ + bin_version = get_binary_version(executable) args = [executable] mongod_options = mongod_options.copy() if "port" not in mongod_options: mongod_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongod_options.get("set_parameters", {}) + remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", + bin_version, "6.0") + remove_set_parameter_if_before_version( + suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongod_options.pop("set_parameters") @@ -78,6 +110,7 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos_options=None): # pylint: disable=too-many-arguments """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" + bin_version = get_binary_version(executable) args = [executable] mongos_options = mongos_options.copy() @@ -85,6 +118,10 @@ def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos if "port" not in mongos_options: mongos_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongos_options.get("set_parameters", {}) + remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", + bin_version, "6.0") + remove_set_parameter_if_before_version( + suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongos_options.pop("set_parameters") diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 2c4d4b761d7..97788b0469b 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -5,12 +5,14 @@ import logging import os import sys import tempfile +from datetime import datetime from abc import ABCMeta, abstractmethod from collections import namedtuple from distutils import spawn # pylint: disable=no-name-in-module from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program from buildscripts.resmokelib.hang_analyzer.process_list import Pinfo +from buildscripts.resmokelib import config as resmoke_config Dumpers = namedtuple('Dumpers', ['dbg', 'jstack']) @@ -330,6 +332,20 @@ class LLDBDumper(Dumper): class GDBDumper(Dumper): """GDBDumper class.""" + def __init__(self, root_logger: logging.Logger, dbg_output: str, + timeout_seconds_for_gdb_process=720): + """Initialize GDBDumper.""" + if resmoke_config.EVERGREEN_TASK_ID is None: + # Set 24 hours time out for hang analyzer being run in locally + timeout_seconds_for_gdb_process = 86400 + #Timeout for hang analyzer, default timeout is 12mins(out of total 15mins) in Evergreen + self._timeout_seconds_for_gdb_process = timeout_seconds_for_gdb_process + super().__init__(root_logger, dbg_output) + + def _reduce_timeout_for_gdb_process(self, timeout_period: int): + """Reduce timeout for remaining gdb processes.""" + self._timeout_seconds_for_gdb_process -= timeout_period + def _find_debugger(self, debugger): """Find the installed debugger.""" return find_program(debugger, ['/opt/mongodbtoolchain/v3/bin', '/usr/bin']) @@ -445,12 +461,19 @@ class GDBDumper(Dumper): debugger = "gdb" dbg = self._find_debugger(debugger) logger = _get_process_logger(self._dbg_output, pinfo.name) + _start_time = datetime.now() if dbg is None: self._root_logger.warning("Debugger %s not found, skipping dumping of %s", debugger, str(pinfo.pidv)) return + if self._timeout_seconds_for_gdb_process <= 0: + self._root_logger.warning( + "Skipping dumping of %s processes with PIDs %s because the time limit expired", + pinfo.name, str(pinfo.pidv)) + return + self._root_logger.info("Debugger %s, analyzing %s processes with PIDs %s", dbg, pinfo.name, str(pinfo.pidv)) @@ -467,8 +490,11 @@ class GDBDumper(Dumper): skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( - itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) + itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger, + self._timeout_seconds_for_gdb_process, pinfo) + time_period = (datetime.now() - _start_time).total_seconds() + self._reduce_timeout_for_gdb_process(time_period) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, str(pinfo.pidv)) diff --git a/buildscripts/resmokelib/hang_analyzer/process.py b/buildscripts/resmokelib/hang_analyzer/process.py index 629f78a2594..ee528aa6de6 100644 --- a/buildscripts/resmokelib/hang_analyzer/process.py +++ b/buildscripts/resmokelib/hang_analyzer/process.py @@ -22,7 +22,7 @@ if _IS_WINDOWS: PROCS_TIMEOUT_SECS = 60 -def call(args, logger): +def call(args, logger, timeout_seconds=None, pinfo=None): """Call subprocess on args list.""" logger.info(str(args)) @@ -31,7 +31,16 @@ def call(args, logger): logger_pipe = core.pipe.LoggerPipe(logger, logging.INFO, process.stdout) logger_pipe.wait_until_started() - ret = process.wait() + try: + ret = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + logger.error("Killing %s processes with PIDs %s because time limit expired", pinfo.name, + str(pinfo.pidv)) + process.kill() + process.wait() + logger_pipe.wait_until_finished() + return + logger_pipe.wait_until_finished() if ret != 0: diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 692b4532c47..07bc214517a 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -161,6 +161,29 @@ class BufferedHandler(logging.Handler): logging.Handler.close(self) +class BufferedFileHandler(BufferedHandler): + """File handler with in-memory buffering.""" + + def __init__(self, filename, capacity=2000, interval_secs=600): + """Initialize the handler with the filename and buffer capacity and flush interval.""" + super().__init__(capacity, interval_secs) + self.file = open(filename, "a", encoding="utf-8") + + def process_record(self, record): + """Return the formatted record message appended with a newline.""" + return self.format(record) + "\n" + + def _flush_buffer_with_lock(self, buf, close_called): + """Write the buffered log lines to the destination file.""" + self.file.writelines(buf) + + def close(self): + """Close the handler and the file descriptor.""" + super().close() + + self.file.close() + + class HTTPHandler(object): """A class which sends data to a web server using POST requests.""" diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index f93cfbd4f4a..2aade997a42 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -13,6 +13,7 @@ from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import redirect as redirect_lib from buildscripts.resmokelib.logging import buildlogger from buildscripts.resmokelib.logging import formatters +from buildscripts.resmokelib.logging.handlers import BufferedFileHandler _DEFAULT_FORMAT = "[%(name)s] %(message)s" @@ -37,6 +38,11 @@ _BUILD_ID_REGISTRY: dict = {} # Maps job nums to fixture loggers. _FIXTURE_LOGGER_REGISTRY: dict = {} +# URL of parsley logs. +RAW_TEST_LOGS_URL = "https://evergreen.mongodb.com/rest/v2/tasks/{task_id}/build/TestLogs/job{job_num}%2F{test_id}.log?execution={execution}&print_time=true" +RAW_JOBS_LOGS_URL = "https://evergreen.mongodb.com/rest/v2/tasks/{task_id}/build/TestLogs/job{job_num}?execution={execution}&print_time=true" +PARSLEY_JOBS_LOGS_URL = "https://parsley.mongodb.com/test/{task_id}/{execution}/job{job_num}/all" + def _build_logger_server(): """Create and return a new BuildloggerServer. @@ -209,7 +215,7 @@ def new_test_logger(test_shortname, test_basename, command, parent, job_num, tes name = "%s:%s" % (parent.name, test_shortname) logger = logging.Logger(name) logger.parent = parent - _add_evergreen_handler(logger, job_num, test_id) + _add_evergreen_handler(logger, job_num, test_id, test_basename) def _get_test_endpoint(job_num, test_basename, command, meta_logger): """Get a new test endpoint for the buildlogger server.""" @@ -363,7 +369,7 @@ def _write_evergreen_log_spec(): yaml.dump(log_spec, fd) -def _add_evergreen_handler(logger, job_num, test_id=None): +def _add_evergreen_handler(logger, job_num, test_id=None, test_name=None): """Add a new evergreen handler to a logger.""" logger_info = config.LOGGING_CONFIG[TESTS_LOGGER_NAME] evergreen_handler_info = None @@ -376,15 +382,35 @@ def _add_evergreen_handler(logger, job_num, test_id=None): fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}" os.makedirs(os.path.dirname(fp), exist_ok=True) - handler = logging.FileHandler(filename=fp, mode="a") + handler = BufferedFileHandler(fp) handler.setFormatter( formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT))) logger.addHandler(handler) if test_id: + raw_url = RAW_TEST_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + test_id=test_id, + execution=config.EVERGREEN_EXECUTION, + ) ROOT_EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp) + ROOT_EXECUTOR_LOGGER.info("Raw logs for %s can be viewed at %s", test_name, raw_url) else: + parsley_url = PARSLEY_JOBS_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + execution=config.EVERGREEN_EXECUTION, + ) + raw_url = RAW_JOBS_LOGS_URL.format( + task_id=config.EVERGREEN_TASK_ID, + job_num=job_num, + execution=config.EVERGREEN_EXECUTION, + ) ROOT_EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp) + ROOT_EXECUTOR_LOGGER.info("Parsley logs for job #%s can be viewed at %s", job_num, + parsley_url) + ROOT_EXECUTOR_LOGGER.info("Raw logs for job #%s can be viewed at %s", job_num, raw_url) def _get_evergreen_log_dirname(): diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 71579bd0fea..7160d1ff026 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -96,7 +96,8 @@ class MultiversionPlugin(PluginInterface): :param kwargs: additional args. :return: None or a Subcommand. """ - configure_resmoke.validate_and_update_config(parser, parsed_args) if subcommand == MULTIVERSION_SUBCOMMAND: + configure_resmoke.detect_evergreen_config(parsed_args) + configure_resmoke.validate_and_update_config(parser, parsed_args) return MultiversionConfigSubcommand(parsed_args) return None diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 621d5ade9ae..f009e44fe45 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -78,6 +78,10 @@ class VersionConstantValues(NamedTuple): """Get a string version of the latest FCV.""" return version_str(self.latest) + def get_fcv_tags_less_than_latest(self) -> List[str]: + """Get the list of all fcv tags less than the latest.""" + return [tag_str(fcv) for fcv in self.fcvs_less_than_latest] + def build_last_lts_binary(self, base_name: str) -> str: """ Build the name of the binary that the LTS version of the given tool will have. diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 3646d139b1f..a8aacb3f60f 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,7 +2,9 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output +import http import requests +from retry import retry import structlog @@ -17,7 +19,7 @@ LAST_CONTINUOUS = "last_continuous" # We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date # with the latest EOL versions. If a "last-continuous" version is EOL, we don't include # it in the multiversion config and therefore don't test against it. -MASTER_RELEASES_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" +MASTER_RELEASES_REMOTE_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" LOGGER = structlog.getLogger(__name__) @@ -36,11 +38,17 @@ def generate_mongo_version_file(): mongo_version_fh.write("mongo_version: " + res) +@retry(tries=5, delay=3) def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. with open(RELEASES_YAML, "wb") as file: - file.write(requests.get(MASTER_RELEASES_FILE).content) + response = requests.get(MASTER_RELEASES_REMOTE_FILE) + if response.status_code != http.HTTPStatus.OK: + raise RuntimeError( + f"Fetching releases.yml file returned unsuccessful status: {response.status_code}, " + f"response body: {response.text}\n") + file.write(response.content) def in_git_root_dir(): @@ -102,6 +110,8 @@ REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() # All multiversion tests should be run with these tags excluded. REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() +REQUIRES_FCV_TAGS_LESS_THAN_LATEST = version_constants.get_fcv_tags_less_than_latest() + # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) diff --git a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py index d392c6ae0b8..c45f93192eb 100644 --- a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py +++ b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py @@ -4,70 +4,70 @@ This hook runs continuously, but the run_aggregate_metrics_background.js file it internally sleep for 1 second between runs. """ -import os.path +import random +import pymongo -from buildscripts.resmokelib import errors -from buildscripts.resmokelib.testing.hooks import jsfile -from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase +from buildscripts.resmokelib.testing.hooks.bghook import BGHook -class AggregateResourceConsumptionMetricsInBackground(jsfile.JSHook): - """A hook to run $operationMetrics stage in the background.""" - - IS_BACKGROUND = True - - def __init__(self, hook_logger, fixture, shell_options=None): - """Initialize AggregateResourceConsumptionMetricsInBackground.""" - description = "Run background $operationMetrics on all mongods while a test is running" - js_filename = os.path.join("jstests", "hooks", "run_aggregate_metrics_background.js") - jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description, - shell_options=shell_options) - self._background_job = None +def verify_metrics(doc): + """Checks whether the output from $operatiomMetrics has the schema we expect.""" - def before_suite(self, test_report): - """Start the background thread.""" - self._background_job = _BackgroundJob("AggregateResourceConsumptionMetricsInBackground") - self.logger.info("Starting the background aggregate metrics thread.") - self._background_job.start() + top_level_fields = [ + "docBytesWritten", "docUnitsWritten", "idxEntryBytesWritten", "idxEntryUnitsWritten", + "totalUnitsWritten", "cpuNanos", "db", "primaryMetrics", "secondaryMetrics" + ] + read_fields = [ + "docBytesRead", "docUnitsRead", "idxEntryBytesRead", "idxEntryUnitsRead", "keysSorted", + "docUnitsReturned" + ] - def after_suite(self, test_report, teardown_flag=None): - """Signal the background aggregate metrics thread to exit, and wait until it does.""" - if self._background_job is None: - return + for key in top_level_fields: + assert key in doc, ("The metrics output is missing the property: " + key) - self.logger.info("Stopping the background aggregate metrics thread.") - self._background_job.stop() + primary_metrics = doc["primaryMetrics"] + for key in read_fields: + assert key in primary_metrics, ( + "The metrics output is missing the property: primaryMetrics." + key) - def before_test(self, test, test_report): - """Instruct the background aggregate metrics thread to run while 'test' is also running.""" - if self._background_job is None: - return + secondary_metrics = doc["secondaryMetrics"] + for key in read_fields: + assert key in secondary_metrics, ( + "The metrics output is missing the property: secondaryMetrics." + key) - hook_test_case = _ContinuousDynamicJSTestCase.create_before_test( - test.logger, test, self, self._js_filename, self._shell_options) - hook_test_case.configure(self.fixture) - self.logger.info("Resuming the background aggregate metrics thread.") - self._background_job.resume(hook_test_case, test_report) - - def after_test(self, test, test_report): # noqa: D205,D400 - """Instruct the background aggregate metrics thread to stop running now that 'test' has - finished running. - """ - if self._background_job is None: - return +class AggregateResourceConsumptionMetricsInBackground(BGHook): + """A hook to run $operationMetrics stage in the background.""" - self.logger.info("Pausing the background aggregate metrics thread.") - self._background_job.pause() + def __init__(self, hook_logger, fixture): + """Initialize AggregateResourceConsumptionMetricsInBackground.""" - if self._background_job.exc_info is not None: - if isinstance(self._background_job.exc_info[1], errors.TestFailure): - # If the mongo shell process running the JavaScript file exited with a non-zero - # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's - # test execution to stop. - raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) - else: - self.logger.error( - "Encountered an error inside the background aggregate metrics thread.", - exc_info=self._background_job.exc_info) - raise self._background_job.exc_info[1] + description = "Run background $operationMetrics on all mongods while a test is running" + super().__init__(hook_logger, fixture, description, tests_per_cycle=None, + loop_delay_ms=1000) + + def run_action(self): + """Collects $operationMetrics on all non-arbiter nodes in the fixture.""" + for node_info in self.fixture.get_node_info(): + conn = pymongo.MongoClient(port=node_info.port) + # Filter out arbiters. + if "arbiterOnly" in conn.admin.command({"isMaster": 1}): + self.logger.info( + "Skipping background aggregation against test node: %s because it is an " + + "arbiter and has no data.", node_info.full_name) + return + + # Clear the metrics about 10% of the time. + clear_metrics = random.random() < 0.1 + self.logger.info("Running $operationMetrics with {clearMetrics: %s} on host: %s", + clear_metrics, node_info.full_name) + with conn.admin.aggregate( + [{"$operationMetrics": {"clearMetrics": clear_metrics}}]) as cursor: + for doc in cursor: + try: + verify_metrics(doc) + except: + self.logger.info( + "caught exception while verifying that all expected fields are in the" + + " metrics output: ", doc) + raise diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py index 92dbb5437a4..cd775aabd4a 100644 --- a/buildscripts/resmokelib/testing/hooks/background_job.py +++ b/buildscripts/resmokelib/testing/hooks/background_job.py @@ -3,6 +3,7 @@ import sys import threading +from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing.hooks import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/bghook.py b/buildscripts/resmokelib/testing/hooks/bghook.py index b61f37b4e2d..1b3fed3b104 100644 --- a/buildscripts/resmokelib/testing/hooks/bghook.py +++ b/buildscripts/resmokelib/testing/hooks/bghook.py @@ -13,11 +13,13 @@ class BGJob(threading.Thread): BGJob will call 'run_action' without any delay and expects the 'run_action' function to add some form of delay. """ - def __init__(self, hook): + def __init__(self, hook, loop_delay_ms=None): """Initialize the background job.""" threading.Thread.__init__(self, name=f"BGJob-{hook.__class__.__name__}") + self._loop_delay_ms = loop_delay_ms self.daemon = True self._hook = hook + self._interrupt_event = threading.Event() self.__is_alive = True self.err = None @@ -29,6 +31,14 @@ class BGJob(threading.Thread): try: self._hook.run_action() + if self._loop_delay_ms is not None: + # The configured loop delay asked us to wait before running the action again. Do + # that wait, but listen to see if we finish running the test or are killed in + # the meantime. + interrupted = self._interrupt_event.wait(self._loop_delay_ms / 1000.0) + if interrupted: + self._hook.logger.info("interrupted") + break except Exception as err: # pylint: disable=broad-except self._hook.logger.error("Background thread caught exception: %s.", err) self.err = err @@ -37,6 +47,7 @@ class BGJob(threading.Thread): def kill(self): """Kill the background job.""" self.__is_alive = False + self._interrupt_event.set() class BGHook(interface.Hook): @@ -46,8 +57,13 @@ class BGHook(interface.Hook): # By default, we continuously run the background hook for the duration of the suite. DEFAULT_TESTS_PER_CYCLE = math.inf - def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None): - """Initialize the background hook.""" + def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None, loop_delay_ms=None): + """ + Initialize the background hook. + + 'tests_per_cycle' or 'loop_delay_ms' can be used to configure how often the background job + is restarted, and how often run_action() is called, respectively. + """ interface.Hook.__init__(self, hook_logger, fixture, desc) self.logger = hook_logger @@ -57,15 +73,21 @@ class BGHook(interface.Hook): self._test_num = 0 # The number of tests we execute before restarting the background hook. self._tests_per_cycle = self.DEFAULT_TESTS_PER_CYCLE if tests_per_cycle is None else tests_per_cycle + self._loop_delay_ms = loop_delay_ms def run_action(self): - """Perform an action. This function will be called continuously in the BgJob.""" + """ + Perform an action. This function will be called continuously in the BgJob. + + If a sleep_delay_ms was given, that many milliseconds of sleep will happen between each + invocation. + """ raise NotImplementedError def before_suite(self, test_report): """Start the background thread.""" self.logger.info("Starting the background thread.") - self._background_job = BGJob(self) + self._background_job = BGJob(self, self._loop_delay_ms) self._background_job.start() def after_suite(self, test_report, teardown_flag=None): @@ -86,7 +108,7 @@ class BGHook(interface.Hook): return self.logger.info("Restarting the background thread.") - self._background_job = BGJob(self) + self._background_job = BGJob(self, self._loop_delay_ms) self._background_job.start() def after_test(self, test, test_report): diff --git a/buildscripts/resmokelib/testing/hooks/run_query_stats.py b/buildscripts/resmokelib/testing/hooks/run_query_stats.py new file mode 100644 index 00000000000..736c5e261c7 --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/run_query_stats.py @@ -0,0 +1,83 @@ +""" +Test hook for verifying $queryStats collects expected metrics and can redact query shapes. + +This runs in the background as other tests are ongoing. +""" + +from bson import binary +import pymongo.errors +from buildscripts.resmokelib.testing.hooks.interface import Hook + +QUERY_STATS_NOT_ENABLED_CODES = [224, 7373500, 6579000] + + +class RunQueryStats(Hook): + """Runs $queryStats after every test, and clears the query stats store before every test.""" + + IS_BACKGROUND = False + + def __init__(self, hook_logger, fixture, allow_feature_not_supported=False): + """Initialize the RunQueryStats hook. + + Args: + hook_logger: the logger instance for this hook. + fixture: the target fixture (replica sets or a sharded cluster). + allow_feature_not_supported: absorb 'QueryFeatureNotAllowed' errors when calling + $queryStats. This is to support fuzzer suites that may manipulate the FCV. + """ + description = "Read query stats data after each test." + super().__init__(hook_logger, fixture, description) + self.client = self.fixture.mongo_client() + self.hmac_key = binary.Binary(("0" * 32).encode('utf-8'), 8) + self.allow_feature_not_supported = allow_feature_not_supported + + def verify_query_stats(self, querystats_spec): + """Verify a $queryStats call has all the right properties.""" + query_stats_pipeline = [ + {"$queryStats": querystats_spec}, + # SERVER-90921: The pymongo version we use on this branch has trouble parsing invalid + # DBRefs, which can be produced by the 'key' field. We'll overwrite that field with a + # dummy one, since the contents aren't important for this test/check. + {"$set": {"key": "Redacted due to issues with DBRefs"}} + ] + try: + with self.client.admin.aggregate(query_stats_pipeline) as cursor: + nreturned = 0 + for operation in cursor: + assert "key" in operation + assert "metrics" in operation + assert "asOf" in operation + nreturned += 1 + self.logger.info("Found %d query stats entries.", nreturned) + except pymongo.errors.OperationFailure as err: + if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: + self.logger.info("Encountered an error while running $queryStats. " + "$queryStats will not be run for this test.") + else: + raise err + + def after_test(self, test, test_report): + """After the test, make sure we can ingest the query stats, with and without hmac.""" + self.verify_query_stats({}) + self.verify_query_stats( + {"transformIdentifiers": {"algorithm": "hmac-sha-256", "hmacKey": self.hmac_key}}) + + # Log the number of evictions we encountered. + server_status = self.client.admin.command({"serverStatus": 1}) + num_evicted_entries = server_status["metrics"]["queryStats"]["numEvicted"] + if num_evicted_entries > 0: + self.logger.info("Evicted %d query stats entries during test execution.", + num_evicted_entries) + + def before_test(self, test, test_report): + """Before the test, reset the contents of the query stats store.""" + try: + # Clear out all existing entries, then reset the size cap. + self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="0%") + self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="1%") + except pymongo.errors.OperationFailure as err: + if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: + self.logger.info("Encountered an error while configuring the query stats store. " + "Query stats will not be collected for this test.") + else: + raise err diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 7658a415fa6..81202daceb8 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -85,7 +85,7 @@ def get_task_name_without_suffix(task_name, variant_name): """Return evergreen task name without suffix added to the generated task. Remove evergreen variant name, numerical suffix and underscores between them from evergreen task name. - Example: "noPassthrough_0_enterprise-rhel-80-64-bit-dynamic-required" -> "noPassthrough" + Example: "noPassthrough_0_enterprise-rhel-8-64-bit-dynamic-required" -> "noPassthrough" """ task_name = task_name if task_name else "" return re.sub(fr"(_[0-9]+)?(_{variant_name})?$", "", task_name) |