diff options
Diffstat (limited to 'buildscripts')
202 files changed, 6213 insertions, 5197 deletions
diff --git a/buildscripts/auto_install_db_contrib_tool.sh b/buildscripts/auto_install_db_contrib_tool.sh index 23a86ad0bc4..1ba77c12460 100755 --- a/buildscripts/auto_install_db_contrib_tool.sh +++ b/buildscripts/auto_install_db_contrib_tool.sh @@ -20,7 +20,6 @@ if [[ -f "$HOME/.zshrc" ]]; then rc_file="$HOME/.zshrc" fi -export PIP_CACHE_DIR=${workdir}/pip_cache if ! command -v db-contrib-tool &> /dev/null; then if ! python3 -c "import sys; sys.exit(sys.version_info < (3, 7))" &> /dev/null; then actual_version=$(python3 -c 'import sys; print(sys.version)') diff --git a/buildscripts/benchmarks/analyze.py b/buildscripts/benchmarks/analyze.py index d8bbefcaaf4..924db911955 100644 --- a/buildscripts/benchmarks/analyze.py +++ b/buildscripts/benchmarks/analyze.py @@ -26,24 +26,15 @@ def get_baseline_task_id(evg_api_config: str, current_task_id: str) -> str: for version in prior_versions: baseline_task_candidate = None - - try: - build = version.build_by_variant(current_task.build_variant) - # If previous version doesn't contain the build variant of the current task - # assume that it's not possible to find the baseline task - except KeyError: - return "" - + build = version.build_by_variant(current_task.build_variant) for task in build.get_tasks(): if task.display_name == current_task.display_name: baseline_task_candidate = task break - # If previous build doesn't contain the task with the same 'display_name' # assume that this is a new task and there is no baseline task if baseline_task_candidate is None: return "" - if baseline_task_candidate.is_success(): return baseline_task_candidate.task_id diff --git a/buildscripts/blackduck_hub.py b/buildscripts/blackduck_hub.py new file mode 100644 index 00000000000..111f0d7b03c --- /dev/null +++ b/buildscripts/blackduck_hub.py @@ -0,0 +1,1334 @@ +#!/usr/bin/env python3 +"""Utility script to run Black Duck scans and query Black Duck database.""" +#pylint: disable=too-many-lines + +import argparse +import functools +import io +import json +import logging +import os +import re +import subprocess +import sys +import tempfile +import time +import warnings + +from abc import ABCMeta, abstractmethod +from typing import Dict, List, Optional + +import urllib3.util.retry as urllib3_retry +import requests +import yaml + +from blackduck.HubRestApi import HubInstance + +try: + import requests.packages.urllib3.exceptions as urllib3_exceptions #pylint: disable=ungrouped-imports +except ImportError: + # Versions of the requests package prior to 1.2.0 did not vendor the urllib3 package. + urllib3_exceptions = None + +LOGGER = logging.getLogger(__name__) + +############################################################################ + +# Name of project to upload to and query about +BLACKDUCK_PROJECT = "mongodb/mongo" + +# Version of project to query about +# Black Duck automatically determines the version based on branch +BLACKDUCK_PROJECT_VERSION = "master" + +# Timeout to wait for a Black Duck scan to complete +BLACKDUCK_TIMEOUT_SECS = 600 + +# Black Duck hub api uses this file to get settings +BLACKDUCK_RESTCONFIG = ".restconfig.json" + +# Wiki page where we document more information about Black Duck +BLACKDUCK_WIKI_PAGE = "https://wiki.corp.mongodb.com/display/KERNEL/Black+Duck" + +# Black Duck failed report prefix +BLACKDUCK_FAILED_PREFIX = "A Black Duck scan was run and failed" + +# Black Duck default teaam +BLACKDUCK_DEFAULT_TEAM = "Service Development Platform" + +############################################################################ + +# Globals +BLACKDUCK_PROJECT_URL = None + +############################################################################ + +# Build Logger constants + +BUILD_LOGGER_CREATE_BUILD_ENDPOINT = "/build" +BUILD_LOGGER_APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" +BUILD_LOGGER_CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" +BUILD_LOGGER_APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" + +BUILD_LOGGER_DEFAULT_URL = "https://logkeeper.mongodb.org" +BUILD_LOGGER_TIMEOUT_SECS = 65 + +LOCAL_REPORTS_DIR = "bd_reports" + +############################################################################ + +THIRD_PARTY_DIRECTORIES = [ + 'src/third_party/wiredtiger/test/3rdparty', + 'src/third_party', +] + +THIRD_PARTY_COMPONENTS_FILE = "etc/third_party_components.yml" + +############################################################################ + +RE_LETTERS = re.compile("[A-Za-z]{2,}") + + +def default_if_none(value, default): + """Set default if value is 'None'.""" + return value if value is not None else default + + +# Derived from buildscripts/resmokelib/logging/handlers.py +class HTTPHandler(object): + """A class which sends data to a web server using POST requests.""" + + def __init__(self, url_root, username, password, should_retry=False): + """Initialize the handler with the necessary authentication credentials.""" + + self.auth_handler = requests.auth.HTTPBasicAuth(username, password) + + self.session = requests.Session() + + if should_retry: + retry_status = [500, 502, 503, 504] # Retry for these statuses. + retry = urllib3_retry.Retry( + backoff_factor=0.1, # Enable backoff starting at 0.1s. + allowed_methods=False, # Support all HTTP verbs. + status_forcelist=retry_status) + + adapter = requests.adapters.HTTPAdapter(max_retries=retry) + self.session.mount('http://', adapter) + self.session.mount('https://', adapter) + + self.url_root = url_root + + def make_url(self, endpoint): + """Generate a url to post to.""" + return "%s/%s/" % (self.url_root.rstrip("/"), endpoint.strip("/")) + + def post(self, endpoint, data=None, headers=None, timeout_secs=BUILD_LOGGER_TIMEOUT_SECS): + """ + Send a POST request to the specified endpoint with the supplied data. + + Return the response, either as a string or a JSON object based + on the content type. + """ + + data = default_if_none(data, []) + data = json.dumps(data) + + headers = default_if_none(headers, {}) + headers["Content-Type"] = "application/json; charset=utf-8" + + url = self.make_url(endpoint) + + LOGGER.info("POSTING to %s", url) + + with warnings.catch_warnings(): + if urllib3_exceptions is not None: + try: + warnings.simplefilter("ignore", urllib3_exceptions.InsecurePlatformWarning) + except AttributeError: + # Versions of urllib3 prior to 1.10.3 didn't define InsecurePlatformWarning. + # Versions of requests prior to 2.6.0 didn't have a vendored copy of urllib3 + # that defined InsecurePlatformWarning. + pass + + try: + warnings.simplefilter("ignore", urllib3_exceptions.InsecureRequestWarning) + except AttributeError: + # Versions of urllib3 prior to 1.9 didn't define InsecureRequestWarning. + # Versions of requests prior to 2.4.0 didn't have a vendored copy of urllib3 + # that defined InsecureRequestWarning. + pass + + response = self.session.post(url, data=data, headers=headers, timeout=timeout_secs, + auth=self.auth_handler, verify=True) + + response.raise_for_status() + + if not response.encoding: + response.encoding = "utf-8" + + headers = response.headers + + if headers["Content-Type"].startswith("application/json"): + return response.json() + + return response.text + + +# Derived from buildscripts/resmokelib/logging/buildlogger.py +class BuildloggerServer(object): + # pylint: disable=too-many-instance-attributes + """ + A remote server to which build logs can be sent. + + It is used to retrieve handlers that can then be added to logger + instances to send the log to the servers. + """ + + def __init__(self, username, password, task_id, builder, build_num, build_phase, url): + # pylint: disable=too-many-arguments + """Initialize BuildloggerServer.""" + self.username = username + self.password = password + self.builder = builder + self.build_num = build_num + self.build_phase = build_phase + self.url = url + self.task_id = task_id + + self.handler = HTTPHandler(url_root=self.url, username=self.username, + password=self.password, should_retry=True) + + def new_build_id(self, suffix): + """Return a new build id for sending global logs to.""" + builder = "%s_%s" % (self.builder, suffix) + build_num = int(self.build_num) + + response = self.handler.post( + BUILD_LOGGER_CREATE_BUILD_ENDPOINT, data={ + "builder": builder, + "buildnum": build_num, + "task_id": self.task_id, + }) + + return response["id"] + + def new_test_id(self, build_id, test_filename, test_command): + """Return a new test id for sending test logs to.""" + endpoint = BUILD_LOGGER_CREATE_TEST_ENDPOINT % {"build_id": build_id} + + response = self.handler.post( + endpoint, data={ + "test_filename": test_filename, + "command": test_command, + "phase": self.build_phase, + "task_id": self.task_id, + }) + + return response["id"] + + def post_new_file(self, build_id, test_name, lines): + """Post a new file to the build logger server.""" + test_id = self.new_test_id(build_id, test_name, "foo") + endpoint = BUILD_LOGGER_APPEND_TEST_LOGS_ENDPOINT % { + "build_id": build_id, + "test_id": test_id, + } + + dt = time.time() + + dlines = [(dt, line) for line in lines] + + try: + self.handler.post(endpoint, data=dlines) + except requests.HTTPError as err: + # Handle the "Request Entity Too Large" error, set the max size and retry. + raise ValueError("Encountered an HTTP error: %s" % (err)) + except requests.RequestException as err: + raise ValueError("Encountered a network error: %s" % (err)) + except: # pylint: disable=bare-except + raise ValueError("Encountered an error.") + + return self.handler.make_url(endpoint) + + +def _to_dict(items, func): + dm = {} + + for i in items: + tuple1 = func(i) + dm[tuple1[0]] = tuple1[1] + + return dm + + +def _compute_security_risk(security_risk_profile): + counts = security_risk_profile["counts"] + + cm = _to_dict(counts, lambda i: (i["countType"], int(i["count"]))) + + priorities = ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'OK', 'UNKNOWN'] + + for priority in priorities: + if cm[priority] > 0: + return priority + + return "OK" + + +@functools.total_ordering +class VersionInfo: + """Parse and break apart version strings so they can be compared.""" + + def __init__(self, ver_str): + """Parse a version string input a tuple of ints or mark it as a beta release.""" + try: + + self.ver_str = ver_str + self.production_version = True + + # Abseil has an empty string for one version + # Abseil thinks "~" is character to use in versions + # Geary thinks "+" is character to use in versions + if self.ver_str == "" or "~" in self.ver_str or "+" in self.ver_str: + self.production_version = False + return + + # Special case Intel's Decimal library since it is just too weird + if ver_str == "v2.0 U1": + self.ver_array = [2, 0] + return + + # BlackDuck thinks boost 1.70.0 was released on 2007 which means we have to check hundreds of versions + bad_keywords = ["(", "+b", "-b", "b1", ".0a"] + if [bad for bad in bad_keywords if bad in self.ver_str]: + self.production_version = False + return + + # Clean the version information + # Some versions start with 'v'. Some components have a mix of 'v' and not 'v' prefixed versions so trim the 'v' + # MongoDB versions start with 'r' + if ver_str[0] == 'v' or ver_str[0] == 'r': + self.ver_str = ver_str[1:] + + # Git hashes are not valid versions + if len(self.ver_str) == 40 and bytes.fromhex(self.ver_str): + self.production_version = False + return + + # Clean out Mozilla's suffix + self.ver_str = self.ver_str.replace("esr", "") + + # Clean out GPerfTool's prefix + self.ver_str = self.ver_str.replace("gperftools-", "") + + # Clean out Yaml Cpp's prefix + self.ver_str = self.ver_str.replace("yaml-cpp-", "") + + # Clean out Boosts's prefix + self.ver_str = self.ver_str.replace("boost-", "") + self.ver_str = self.ver_str.replace("asio-", "") + + if self.ver_str.endswith('-'): + self.ver_str = self.ver_str[0:-1] + + # Boost keeps varying the version strings so filter for anything with 2 or more ascii charaters + if RE_LETTERS.search(self.ver_str): + self.production_version = False + return + + # Some versions end with "-\d", change the "-" since it just means a patch release from a debian/rpm package + # yaml-cpp has this problem where Black Duck sourced the wrong version information + self.ver_str = self.ver_str.replace("-", ".") + + # If we trimmed the string to nothing, treat it as a beta version + if self.ver_str == '': + self.production_version = False + return + + # Versions are generally a multi-part integer tuple + self.ver_array = [int(part) for part in self.ver_str.split(".")] + + except: + LOGGER.error("Failed to parse version '%s' as '%s', exception", ver_str, self.ver_str) + raise + + def __repr__(self): + return self.__str__() + + def __str__(self): + return ".".join([str(val) for val in self.ver_array]) + + def __eq__(self, other): + return (self.production_version, self.ver_array) == (other.production_version, + other.ver_array) + + def __gt__(self, other): + if self.production_version != other.production_version: + return self.production_version + + return self.ver_array > other.ver_array + + +def _test_version_info(): + VersionInfo("v2.0 U1") + VersionInfo("60.7.0-esr") + VersionInfo("v1.1") + VersionInfo("0.4.2-1") + VersionInfo("7.0.2") + VersionInfo("gperftools-2.8") + VersionInfo("v1.5-rc2") + VersionInfo("r4.7.0-alpha") + VersionInfo("r4.2.10") + VersionInfo("2.0.0.1") + VersionInfo("7.0.2-2") + VersionInfo("git") + VersionInfo("20200225.2") + VersionInfo('release-68-alpha') + VersionInfo('cldr/2020-09-22') + VersionInfo('release-67-rc') + VersionInfo('66.1~rc') + VersionInfo('release-66-rc') + VersionInfo('release-66-preview') + VersionInfo('65.1') + VersionInfo('release-65-rc') + VersionInfo('64.2-rc') + VersionInfo('release-64-rc2') + VersionInfo('release-63-rc') + VersionInfo('last-cvs-commit') + VersionInfo('last-svn-commit') + VersionInfo('release-62-rc') + VersionInfo('cldr-32-beta2') + VersionInfo('release-60-rc') + VersionInfo('milestone-60-0-1') + VersionInfo('release-59-rc') + VersionInfo('milestone-59-0-1') + VersionInfo('release-58-2-eclipse-20170118') + VersionInfo('tools-release-58') + VersionInfo('icu-latest') + VersionInfo('icu4j-latest') + VersionInfo('icu4j-release-58-1') + VersionInfo('icu4j-release-58-rc') + VersionInfo('icu-release-58-rc') + VersionInfo('icu-milestone-58-0-1') + VersionInfo('icu4j-milestone-58-0-1') + + VersionInfo('yaml-cpp-0.6.3') + + VersionInfo('gb-c8-task188949.100') + VersionInfo('1.2.8-alt1.M80C.1') + VersionInfo('1.2.8-alt2') + + assert VersionInfo('7.0.2.2') > VersionInfo('7.0.0.1') + assert VersionInfo('7.0.2.2') > VersionInfo('7.0.2') + assert VersionInfo('7.0.2.2') > VersionInfo('3.1') + assert VersionInfo('7.0.2.2') <= VersionInfo('8.0.2') + + +def _retry_on_except(count, func): + # Retry func() COUNT times until func() does not raise an exception + # pylint: disable=bare-except + retry = 0 + while retry < count: + + try: + return func() + except: + exception_info = sys.exc_info()[0] + LOGGER.error("Failed to execute retriable function (%s), retrying", exception_info) + + retry += 1 + + raise ValueError("Failed to run query after retries %s" % (count)) + + +class Component: + """ + Black Duck Component description. + + Contains a subset of information about a component extracted from Black Duck for a given project and version + """ + + def __init__(self, name, version, licenses, policy_status, security_risk, newest_release, + is_manually_added): + # pylint: disable=too-many-arguments + """Initialize Black Duck component.""" + self.name = name + self.version = version + self.licenses = licenses + self.policy_status = policy_status + self.security_risk = security_risk + self.newest_release = newest_release + self.is_manually_added = is_manually_added + + @staticmethod + def parse(hub, component): + # pylint: disable=too-many-locals + """Parse a Black Duck component from a dictionary.""" + name = component["componentName"] + cversion = component.get("componentVersionName", "unknown_version") + licenses = ",".join([a.get("spdxId", a["licenseDisplay"]) for a in component["licenses"]]) + + policy_status = component["policyStatus"] + security_risk = _compute_security_risk(component['securityRiskProfile']) + + newer_releases = component["activityData"].get("newerReleases", 0) + + is_manually_added = 'MANUAL_BOM_COMPONENT' in component['matchTypes'] + + LOGGER.info("Retrievinng version information for Comp %s - %s Releases %s", name, cversion, + newer_releases) + cver = VersionInfo(cversion) + newest_release = None + + # Blackduck's newerReleases is based on "releasedOn" date. This means that if a upstream component releases a beta or rc, + # it counts as newer but we do not consider those newer for our purposes + # Missing newerReleases means we do not have to upgrade + if newer_releases > 0: + limit = newer_releases + 1 + versions_url = component["component"] + f"/versions?sort=releasedon%20desc&limit={limit}" + + LOGGER.info("Retrieving version information via %s", versions_url) + + def get_version_info(): + vjson = hub.execute_get(versions_url).json() + + if "items" not in vjson: + LOGGER.warn("Missing items in response: %s", vjson) + raise ValueError("Missing items in response for " + versions_url) + + return vjson + + vjson = _retry_on_except(5, get_version_info) + + versions = [(ver["versionName"], ver["releasedOn"]) for ver in vjson["items"]] + + LOGGER.info("Known versions: %s ", versions) + + versions = [ver["versionName"] for ver in vjson["items"]] + + # For Firefox, only examine Extended Service Releases (i.e. esr), their long term support releases + if name == "Mozilla Firefox": + versions = [ver for ver in versions if "esr" in ver] + + # For yaml-cpp, we need to clean the list of versions a little + # yaml-cpp uses #.#.# but there are some entires with #.#.#.# so the later needs to + # be filtered out. + if name == "jbeder/yaml-cpp": + ver_regex = re.compile(r"\d+\.\d+\.\d+$") + versions = [ver for ver in versions if ver_regex.match(ver)] + + # For Boost C++ Libraries - boost, we need to clean the list of versions a little + # All boost versions for the last 10 years start with 1.x.x. Black Duck thinks some + # versions are 4.x.x which are bogus and throw off the sorting. + # Also, boost uses #.#.# but there are some entires with #.#.#.# so the later needs to + # be filtered out. + if name == "Boost C++ Libraries - boost": + ver_regex = re.compile(r"\d+\.\d+\.\d+$") + versions = [ver for ver in versions if ver.startswith("1") and ver_regex.match(ver)] + + ver_info = [VersionInfo(ver) for ver in versions] + ver_info = [ver for ver in ver_info if ver.production_version] + LOGGER.info("Filtered versions: %s ", ver_info) + + ver_info = sorted([ver for ver in ver_info if ver.production_version and ver > cver], + reverse=True) + + LOGGER.info("Sorted versions: %s ", ver_info) + + if ver_info: + newest_release = ver_info[0] + + return Component(name, cversion, licenses, policy_status, security_risk, newest_release, + is_manually_added) + + +class BlackDuckConfig: + """ + Black Duck configuration settings. + + Format is defined by Black Duck Python hub API. + """ + + def __init__(self): + """Init Black Duck config from disk.""" + if not os.path.exists(BLACKDUCK_RESTCONFIG): + raise ValueError("Cannot find %s for blackduck configuration" % (BLACKDUCK_RESTCONFIG)) + + with open(BLACKDUCK_RESTCONFIG, "r") as rfh: + rc = json.loads(rfh.read()) + + self.url = rc["baseurl"] + self.username = rc["username"] + self.password = rc["password"] + + +def _run_scan(): + # Get user name and password from .restconfig.json + bdc = BlackDuckConfig() + + with tempfile.NamedTemporaryFile() as fp: + fp.write(f"""#/!bin/sh +curl --retry 5 -s -L https://detect.synopsys.com/detect.sh | bash -s -- --blackduck.url={bdc.url} --blackduck.username={bdc.username} --blackduck.password={bdc.password} --detect.report.timeout={BLACKDUCK_TIMEOUT_SECS} --snippet-matching --upload-source --detect.wait.for.results=true +""".encode()) + fp.flush() + + subprocess.check_call(["/bin/sh", fp.name]) + + +def _scan_cmd_args(args): + # pylint: disable=unused-argument + LOGGER.info("Running Black Duck Scan") + + _run_scan() + + +def _query_blackduck(): + # pylint: disable=global-statement + global BLACKDUCK_PROJECT_URL + + hub = HubInstance() + + LOGGER.info("Getting version from blackduck") + version = hub.execute_get(hub.get_urlbase() + "/api/current-version").json() + LOGGER.info("Version: %s", version) + + # Get a list of all projects, this is a privileged call and will fail if we do not have a valid license + LOGGER.info("Get All Projects") + projects = hub.get_projects() + LOGGER.info("Projects: %s", projects) + + LOGGER.info("Fetching project %s from blackduck", BLACKDUCK_PROJECT) + project = hub.get_project_by_name(BLACKDUCK_PROJECT) + + LOGGER.info("Fetching project version %s from blackduck", BLACKDUCK_PROJECT_VERSION) + version = hub.get_version_by_name(project, BLACKDUCK_PROJECT_VERSION) + + LOGGER.info("Getting version components from blackduck") + bom_components = hub.get_version_components(version) + + components = [ + Component.parse(hub, comp) for comp in bom_components["items"] if comp['ignored'] is False + ] + + BLACKDUCK_PROJECT_URL = version["_meta"]["href"] + + return components + + +class TestResultEncoder(json.JSONEncoder): + """JSONEncoder for TestResults.""" + + def default(self, o): + """Serialize objects by default as a dictionary.""" + # pylint: disable=method-hidden + return o.__dict__ + + +class TestResult: + """A single test result in the Evergreen report.json format.""" + + def __init__(self, name, status, url): + """Init test result.""" + # This matches the report.json schema + # See https://github.com/evergreen-ci/evergreen/blob/789bee107d3ffb9f0f82ae344d72502945bdc914/model/task/task.go#L264-L284 + assert status in ["pass", "fail"] + + self.test_file = name + self.status = status + self.exit_code = 1 + + if url: + self.url = url + self.url_raw = url + "?raw=1" + + if status == "pass": + self.exit_code = 0 + + +class TestResults: + """Evergreen TestResult format for report.json.""" + + def __init__(self): + """Init test results.""" + self.results = [] + + def add_result(self, result: TestResult): + """Add a test result.""" + self.results.append(result) + + def write(self, filename: str): + """Write the test results to disk.""" + + with open(filename, "w") as wfh: + wfh.write(json.dumps(self, cls=TestResultEncoder)) + + +class ReportLogger(object, metaclass=ABCMeta): + """Base Class for all report loggers.""" + + @abstractmethod + def log_report(self, name: str, content: str) -> Optional[str]: + """Get the command to run a linter.""" + pass + + +class LocalReportLogger(ReportLogger): + """Write reports to local directory as a set of files.""" + + def __init__(self): + """Init logger and create directory.""" + if not os.path.exists(LOCAL_REPORTS_DIR): + os.mkdir(LOCAL_REPORTS_DIR) + + def log_report(self, name: str, content: str) -> Optional[str]: + """Log report to a local file.""" + file_name = os.path.join(LOCAL_REPORTS_DIR, name + ".log") + + with open(file_name, "w") as wfh: + wfh.write(content) + + +class BuildLoggerReportLogger(ReportLogger): + """Write reports to a build logger server.""" + + def __init__(self, build_logger): + """Init logger.""" + self.build_logger = build_logger + + self.build_id = self.build_logger.new_build_id("bdh") + + def log_report(self, name: str, content: str) -> Optional[str]: + """Log report to a build logger.""" + + content = content.split("\n") + + return self.build_logger.post_new_file(self.build_id, name, content) + + +def _get_default(list1, idx, default): + if (idx + 1) < len(list1): + return list1[idx] + + return default + + +class TableWriter: + """Generate an ASCII table that summarizes the results of all the reports generated.""" + + def __init__(self, headers: List[str]): + """Init writer.""" + self._headers = headers + self._rows = [] + + def add_row(self, row: List[str]): + """Add a row to the table.""" + self._rows.append(row) + + @staticmethod + def _write_row(col_sizes: List[int], row: List[str], writer: io.StringIO): + writer.write("|") + for idx, row_value in enumerate(row): + writer.write(" ") + writer.write(row_value) + writer.write(" " * (col_sizes[idx] - len(row_value))) + writer.write(" |") + writer.write("\n") + + def print(self, writer: io.StringIO): + """Print the final table to the string stream.""" + cols = max([len(r) for r in self._rows]) + + assert cols == len(self._headers) + + col_sizes = [] + for col in range(0, cols): + col_sizes.append( + max([len(_get_default(row, col, [])) + for row in self._rows] + [len(self._headers[col])])) + + TableWriter._write_row(col_sizes, self._headers, writer) + + TableWriter._write_row(col_sizes, ["-" * c for c in col_sizes], writer) + + for row in self._rows: + TableWriter._write_row(col_sizes, row, writer) + + +class TableData: + """Store scalar values in a two-dimensional matrix indexed by the first column's value.""" + + def __init__(self): + """Init table data.""" + self._rows = {} + + def add_value(self, col: str, value: str): + """Add a value for a given column. Order sensitive.""" + if col not in self._rows: + self._rows[col] = [] + + self._rows[col].append(value) + + def write(self, headers: List[str], writer: io.StringIO): + """Write table data as nice prettty table to writer.""" + tw = TableWriter(headers) + + for row in self._rows: + tw.add_row([row] + self._rows[row]) + + tw.print(writer) + + +class ReportManager: + """Manage logging reports to ReportLogger and generate summary report.""" + + def __init__(self, logger: ReportLogger): + """Init report manager.""" + self._logger = logger + self._results = TestResults() + self._data = TableData() + + @staticmethod + def _get_norm_comp_name(comp_name: str): + return comp_name.replace(" ", "_").replace("/", "_").lower() + + def write_report(self, comp_name: str, report_name: str, status: str, content: str): + """ + Write a report about a test to the build logger. + + status is a string of "pass" or "fail" + """ + comp_name = ReportManager._get_norm_comp_name(comp_name) + + name = comp_name + "_" + report_name + + LOGGER.info("Writing Report %s - %s", name, status) + + self._data.add_value(comp_name, status) + + url = self._logger.log_report(name, content) + + self._results.add_result(TestResult(name, status, url)) + + def add_report_metric(self, comp_name: str, metric: str): + """Add a column to be included in the pretty table.""" + comp_name = ReportManager._get_norm_comp_name(comp_name) + + self._data.add_value(comp_name, metric) + + def finish(self, reports_file: Optional[str], vulnerabilties_only: bool): + """Generate final summary of all reports run.""" + + if reports_file: + self._results.write(reports_file) + + stream = io.StringIO() + + if vulnerabilties_only: + self._data.write(["Component", "Vulnerability"], stream) + else: + self._data.write( + ["Component", "Vulnerability", "Upgrade", "Current Version", "Newest Version"], + stream) + + print(stream.getvalue()) + + +class ThirdPartyComponent: + """MongoDB Third Party component from third_party_components.yml.""" + + def __init__(self, name, homepage_url, local_path, team_owner): + """Init class.""" + # Required fields + self.name = name + self.homepage_url = homepage_url + self.local_path = local_path + self.team_owner = team_owner + + # optional fields + self.is_test_only = False + self.vulnerability_suppression = None + self.upgrade_suppression = None + + +def _get_field(name, ymap, field: str): + if field not in ymap: + raise ValueError("Missing field %s for component %s" % (field, name)) + + return ymap[field] + + +def _get_supression_field(ymap, field: str): + if field not in ymap: + return None + + value = ymap[field].lower() + + if not "todo" in value: + raise ValueError( + "Invalid suppression, a suppression must include the word 'TODO' so that the TODO scanner finds resolved tickets." + ) + + return value + + +def _read_third_party_components(): + with open(THIRD_PARTY_COMPONENTS_FILE) as rfh: + yaml_file = yaml.load(rfh.read()) + + third_party = [] + components = yaml_file["components"] + for comp in components: + cmap = components[comp] + + tp = ThirdPartyComponent(comp, _get_field(comp, cmap, 'homepage_url'), + _get_field(comp, cmap, 'local_directory_path'), + _get_field(comp, cmap, 'team_owner')) + + tp.is_test_only = cmap.get("is_test_only", False) + tp.vulnerability_suppression = _get_supression_field(cmap, "vulnerability_suppression") + tp.upgrade_suppression = _get_supression_field(cmap, "upgrade_suppression") + + third_party.append(tp) + + return third_party + + +def _generate_report_missing_blackduck_component(mgr: ReportManager, mcomp: ThirdPartyComponent): + mgr.write_report( + mcomp.name, "missing_blackduck_component", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The {mcomp.name} library was found in {THIRD_PARTY_COMPONENTS_FILE} but not detected by Black Duck. + +This is caused by one of two issues: +1. The {THIRD_PARTY_COMPONENTS_FILE} file is out of date and the entry needs to be removed for "{mcomp.name}". +or +2. A entry to the component needs to be manually added to Black Duck. + +Next Steps: + +Build Baron: +A BF ticket should be generated and assigned to "{BLACKDUCK_DEFAULT_TEAM}" with +this text. + +Developer: +To address this build failure, the next steps are as follows: +1. Verify that the component is correct in {THIRD_PARTY_COMPONENTS_FILE}. + +2. If the component is incorrect, add a comment to the component in Black Duck and mark it as "Ignored". +or +2. If the component is correct, add the correct information to {THIRD_PARTY_COMPONENTS_FILE}. + +If the "{BLACKDUCK_DEFAULT_TEAM}" cannot do this work for any reason, the BF should be assigned to +the component owner team "{mcomp.team_owner}". + +For more information, see {BLACKDUCK_WIKI_PAGE}. +""") + + +def _generate_report_blackduck_missing_directory(mgr: ReportManager, directory: str): + mgr.write_report( + directory, "missing_directory", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The directory "{directory}" was found in a known MongoDB third_party directory but is not known to +Black Duck or {THIRD_PARTY_COMPONENTS_FILE}. This directory likely needs to be added to Black Duck +manually. + +Next Steps: + +Build Baron: +A BF ticket should be generated and assigned to "{BLACKDUCK_DEFAULT_TEAM}" with +this text. + +Developer: +To address this build failure, the next steps are as follows: +1. Verify that the component is correct. +2. Add the component manually to the Black Duck project at + {BLACKDUCK_PROJECT_URL}. +3. Once the component has been accepted by Black Duck, add the correct information to + {THIRD_PARTY_COMPONENTS_FILE}. + +For more information, see {BLACKDUCK_WIKI_PAGE}. +""") + + +def _generate_report_missing_yaml_component(mgr: ReportManager, comp: Component): + mgr.write_report( + comp.name, "missing_yaml_component", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The {comp.name} library with version "{comp.version}" was detected by Black Duck but not found in +{THIRD_PARTY_COMPONENTS_FILE}. + +This is caused by one of two issues: +1. Black Duck has made an error and the software is not being vendored by MongoDB. +2. Black Duck is correct and {THIRD_PARTY_COMPONENTS_FILE} must be updated. + +Next Steps: + +Build Baron: +A BF ticket should be generated and assigned to "{BLACKDUCK_DEFAULT_TEAM}" with +this text. + +Developer: +To address this build failure, the next steps are as follows: +1. Verify that the component is correct at {BLACKDUCK_PROJECT_URL}. + +2. If the component is incorrect, add a comment to the component and mark it as "Ignored". +or +3. If the component is correct, add the correct information to {THIRD_PARTY_COMPONENTS_FILE}. + +For more information, see {BLACKDUCK_WIKI_PAGE}. +""") + + +def _generate_report_upgrade(mgr: ReportManager, comp: Component, mcomp: ThirdPartyComponent, + fail: bool): + if not fail: + mgr.write_report(comp.name, "upgrade_check", "pass", "Blackduck run passed") + else: + + if comp.is_manually_added: + component_explanation = f"""This component requires a manual update in the Black Duck Database because it was added to +Black Duck manually. After the update to the third-party library is committed, please update the +version information for this component at {BLACKDUCK_PROJECT_URL}. Click on the down arrow on the +far right of the component, choose edit and specify the new version.""" + else: + component_explanation = """This commponent was automatically detected by Black Duck. Black Duck should automatically detect +the new version after the library is updated and the daily scanner task runs again.""" + + mgr.write_report( + comp.name, "upgrade_check", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The {comp.name} library at {mcomp.local_path} is out of date. The current version is +"{comp.version}" and the newest version is "{comp.newest_release}" according to Black Duck. + +MongoDB policy requires all third-party software to be updated to the latest version on the master +branch. + +Next Steps: + +Build Baron: +A BF ticket should be generated and assigned to "{BLACKDUCK_DEFAULT_TEAM}" with +this text. + +Developer: +To address this build failure, the next steps are as follows: +1. File a SERVER ticket to update the software if one already does not exist. +2. Add a “upgrade_supression” to {THIRD_PARTY_COMPONENTS_FILE} with the SERVER ticket to acknowledge + this report. Note that you do not need to immediately update the library, just file a ticket. + +{component_explanation} + +If the "{BLACKDUCK_DEFAULT_TEAM}" cannot do this work for any reason, the BF should be assigned to +the component owner team "{mcomp.team_owner}". + +For more information, see {BLACKDUCK_WIKI_PAGE}. +""") + + mgr.add_report_metric(comp.name, str(comp.version)) + mgr.add_report_metric(comp.name, str(comp.newest_release)) + + +def _generate_report_vulnerability(mgr: ReportManager, comp: Component, mcomp: ThirdPartyComponent, + fail: bool): + if not fail: + mgr.write_report(comp.name, "vulnerability_check", "pass", "Blackduck run passed") + return + + mgr.write_report( + comp.name, "vulnerability_check", "fail", f"""{BLACKDUCK_FAILED_PREFIX} + +The {comp.name} library at {mcomp.local_path} had HIGH and/or CRITICAL security issues. The current +version in Black Duck is "{comp.version}". + +MongoDB policy requires all third-party software to be updated to a version clean of HIGH and +CRITICAL vulnerabilities on the master branch. + +Next Steps: + +Build Baron: +A BF ticket should be generated and assigned to "{BLACKDUCK_DEFAULT_TEAM}" with +this text. + +Developer: +To address this build failure, the next steps are as follows: +1. File a SERVER ticket to update the software if one already does not exist. Note that you do not + need to immediately update the library, just file a ticket. +2. Add a “vulnerability_supression” to {THIRD_PARTY_COMPONENTS_FILE} with the SERVER ticket to + acknowledge this report. + +If you believe the library is already up-to-date but Black Duck has the wrong version, please update +version information for this component at {BLACKDUCK_PROJECT_URL}. + +If the "{BLACKDUCK_DEFAULT_TEAM}" cannot do this work for any reason, the BF should be assigned to +the component owner team "{mcomp.team_owner}". + +For more information, see {BLACKDUCK_WIKI_PAGE}. +""") + + +def _get_third_party_directories(): + third_party = [] + for tp in THIRD_PARTY_DIRECTORIES: + for entry in os.scandir(tp): + if entry.name not in ["scripts"] and entry.is_dir(): + third_party.append(entry.path) + + return sorted(third_party) + + +class Analyzer: + """ + Analyze the MongoDB source code for software maintence issues. + + Queries Black Duck for out of date software + Consults a local yaml file for detailed information about third party components included in the MongoDB source code. + """ + + def __init__(self): + """Init analyzer.""" + self.third_party_components = None + self.third_party_directories = None + self.black_duck_components = None + self.mgr = None + + def _do_reports(self, vulnerabilties_only: bool): + for comp in self.black_duck_components: + # 1. Validate if this is in the YAML file + if self._verify_yaml_contains_component(comp, vulnerabilties_only): + + # 2. Validate there are no security issues + self._verify_vulnerability_status(comp) + + # 3. Check for upgrade issue + if not vulnerabilties_only: + self._verify_upgrade_status(comp) + + if vulnerabilties_only: + return + + # 4. Validate that each third_party directory is in the YAML file + self._verify_directories_in_yaml() + + # 5. Verify the YAML file has all the entries in Black Duck + self._verify_components_in_yaml() + + def _verify_yaml_contains_component(self, comp: Component, vulnerabilties_only: bool): + # It should be rare that Black Duck detects something that is not in the YAML file + # As a result, we do not generate a "pass" report for simply be consistent between Black Duck and the yaml file + if comp.name not in [c.name for c in self.third_party_components]: + if not vulnerabilties_only: + _generate_report_missing_yaml_component(self.mgr, comp) + return False + + return True + + def _verify_directories_in_yaml(self): + + comp_dirs = [c.local_path for c in self.third_party_components] + for cdir in self.third_party_directories: + # Ignore WiredTiger since it is not a third-party library but made by MongoDB, Inc. + if cdir in ["src/third_party/wiredtiger"]: + continue + + if cdir not in comp_dirs: + _generate_report_blackduck_missing_directory(self.mgr, cdir) + + def _verify_components_in_yaml(self): + + comp_names = [c.name for c in self.black_duck_components] + for mcomp in self.third_party_components: + # These components are known to be missing from Black Duck + # Aladdin MD5 is a pair of C files for MD5 computation + # timelib is simply missing + # Unicode is not code + if mcomp.name in ["Aladdin MD5", "timelib", "unicode"]: + continue + + if mcomp.name not in comp_names: + _generate_report_missing_blackduck_component(self.mgr, mcomp) + + def _verify_upgrade_status(self, comp: Component): + mcomp = self._get_mongo_component(comp) + + if comp.newest_release and not mcomp.upgrade_suppression and not mcomp.is_test_only: + _generate_report_upgrade(self.mgr, comp, mcomp, True) + else: + _generate_report_upgrade(self.mgr, comp, mcomp, False) + + def _verify_vulnerability_status(self, comp: Component): + mcomp = self._get_mongo_component(comp) + + if comp.security_risk in [ + "HIGH", "CRITICAL" + ] and not mcomp.vulnerability_suppression and not mcomp.is_test_only: + _generate_report_vulnerability(self.mgr, comp, mcomp, True) + else: + _generate_report_vulnerability(self.mgr, comp, mcomp, False) + + def _get_mongo_component(self, comp: Component): + mcomp = next((x for x in self.third_party_components if x.name == comp.name), None) + + if not mcomp: + raise ValueError( + "Cannot find third party component for Black Duck Component '%s'. Please update '%s'. " + % (comp.name, THIRD_PARTY_COMPONENTS_FILE)) + + return mcomp + + def run(self, logger: ReportLogger, report_file: Optional[str], vulnerabilties_only: bool): + """Run analysis of Black Duck scan and local files.""" + + self.third_party_directories = _get_third_party_directories() + + LOGGER.info("Found the following third party directories: %s", self.third_party_directories) + + self.third_party_components = _read_third_party_components() + + self.black_duck_components = _query_blackduck() + + # Black Duck detects ourself everytime we release a new version + # Rather then constantly have to supress this in Black Duck itself which will generate false positives + # We filter ourself our of the list of components. + self.black_duck_components = [ + comp for comp in self.black_duck_components + if not (comp.name == "MongoDB" or comp.name == "WiredTiger") + ] + + # Remove duplicate Black Duck components. We only care about the component with highest version number + # Black Duck can detect different versions of the same component for instance if an upgrade of a component happens + bd_names = {comp.name for comp in self.black_duck_components} + if len(bd_names) != len(self.black_duck_components): + LOGGER.warning("Found duplicate Black Duck components") + bd_unique = {} + for comp in self.black_duck_components: + if comp.name in bd_unique: + LOGGER.warning("Found duplicate Black Duck component: %s", comp.name) + first = bd_unique[comp.name] + if comp.version > first.version: + bd_unique[comp.name] = comp + else: + bd_unique[comp.name] = comp + + self.black_duck_components = list(bd_unique.values()) + + self.mgr = ReportManager(logger) + + self._do_reports(vulnerabilties_only) + + self.mgr.finish(report_file, vulnerabilties_only) + + +# Derived from buildscripts/resmokelib/logging/buildlogger.py +def _get_build_logger_from_file(filename, build_logger_url, task_id): + tmp_globals = {} + config = {} + + # The build logger config file is actually python + # It is a mix of quoted strings and ints + exec(compile(open(filename, "rb").read(), filename, 'exec'), tmp_globals, config) + + # Rename "slavename" to "username" if present. + if "slavename" in config and "username" not in config: + config["username"] = config["slavename"] + del config["slavename"] + + # Rename "passwd" to "password" if present. + if "passwd" in config and "password" not in config: + config["password"] = config["passwd"] + del config["passwd"] + + return BuildloggerServer(config["username"], config["password"], task_id, config["builder"], + config["build_num"], config["build_phase"], build_logger_url) + + +def _generate_reports_args(args): + LOGGER.info("Generating Reports") + + # Log to LOCAL_REPORTS_DIR directory unless build logger is explicitly chosen + logger = LocalReportLogger() + + if args.build_logger_local: + build_logger = BuildloggerServer("fake_user", "fake_pass", "fake_task", "fake_builder", 1, + "fake_build_phase", "http://localhost:8080") + logger = BuildLoggerReportLogger(build_logger) + elif args.build_logger: + if not args.build_logger_task_id: + raise ValueError("Must set build_logger_task_id if using build logger") + + build_logger = _get_build_logger_from_file(args.build_logger, args.build_logger_url, + args.build_logger_task_id) + logger = BuildLoggerReportLogger(build_logger) + + analyzer = Analyzer() + analyzer.run(logger, args.report_file, args.vulnerabilities_only) + + +def _scan_and_report_args(args): + LOGGER.info("Running Black Duck Scan And Generating Reports") + + _run_scan() + + _generate_reports_args(args) + + +def main() -> None: + """Execute Main entry point.""" + + parser = argparse.ArgumentParser(description='Black Duck hub controller.') + + parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging") + parser.add_argument('-d', "--debug", action='store_true', help="Enable debug logging") + + sub = parser.add_subparsers(title="Hub subcommands", help="sub-command help") + generate_reports_cmd = sub.add_parser('generate_reports', + help='Generate reports from Black Duck') + + generate_reports_cmd.add_argument("--report_file", type=str, + help="report json file to write to") + generate_reports_cmd.add_argument( + "--build_logger", type=str, help="Log to build logger with credentials from specified file") + generate_reports_cmd.add_argument("--build_logger_url", type=str, + default=BUILD_LOGGER_DEFAULT_URL, + help="build logger url to log to") + generate_reports_cmd.add_argument("--build_logger_task_id", type=str, + help="build logger task id") + generate_reports_cmd.add_argument("--build_logger_local", action='store_true', + help="Log to local build logger, logs to disk by default") + generate_reports_cmd.add_argument("--vulnerabilities_only", action='store_true', + help="Only check for security vulnerabilities") + generate_reports_cmd.set_defaults(func=_generate_reports_args) + + scan_cmd = sub.add_parser('scan', help='Do Black Duck Scan') + scan_cmd.set_defaults(func=_scan_cmd_args) + + scan_and_report_cmd = sub.add_parser('scan_and_report', + help='Run scan and then generate reports') + scan_and_report_cmd.add_argument("--report_file", type=str, help="report json file to write to") + + scan_and_report_cmd.add_argument( + "--build_logger", type=str, help="Log to build logger with credentials from specified file") + scan_and_report_cmd.add_argument("--build_logger_url", type=str, + default=BUILD_LOGGER_DEFAULT_URL, + help="build logger url to log to") + scan_and_report_cmd.add_argument("--build_logger_task_id", type=str, + help="build logger task id") + scan_and_report_cmd.add_argument("--build_logger_local", action='store_true', + help="Log to local build logger, logs to disk by default") + scan_and_report_cmd.add_argument("--vulnerabilities_only", action='store_true', + help="Only check for security vulnerabilities") + scan_and_report_cmd.set_defaults(func=_scan_and_report_args) + + args = parser.parse_args() + + _test_version_info() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + elif args.verbose: + logging.basicConfig(level=logging.INFO) + + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/buildscripts/build_system_options.py b/buildscripts/build_system_options.py index 140d1e4dffd..66a303bf320 100644 --- a/buildscripts/build_system_options.py +++ b/buildscripts/build_system_options.py @@ -9,7 +9,7 @@ class PathOptions: shared_library_folder_name = "lib" # extend the below list if there are new types of shared libraries - _shared_library_file_patterns = [r".+\.so(\.(\d{1,}|debug))?$", r".+\.dylib$"] + _shared_library_file_patterns = [r".+\.so(\.\d{1,})?$", r".+\.dylib$"] _compiled_shared_library_file_patterns = None @property diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py new file mode 100644 index 00000000000..5acb2e75861 --- /dev/null +++ b/buildscripts/burn_in_tags.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Generate burn in tests to run on certain build variants.""" +from collections import namedtuple +import logging +import os +import sys +from typing import Any, Dict, List + +import click + +from git import Repo +from shrub.v2 import ShrubProject, BuildVariant, ExistingTask +from evergreen.api import RetryingEvergreenApi, EvergreenApi + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +from buildscripts.patch_builds.task_generation import validate_task_generation_limit + +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# pylint: disable=wrong-import-position +from buildscripts.evergreen_burn_in_tests import GenerateBurnInExecutor, GenerateConfig, \ + EvergreenFileChangeDetector +from buildscripts.util.fileops import write_file_to_dir +import buildscripts.util.read_config as read_config +from buildscripts.ciconfig import evergreen +from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant +from buildscripts.burn_in_tests import create_tests_by_task, RepeatConfig, DEFAULT_REPO_LOCATIONS +# pylint: enable=wrong-import-position + +EXTERNAL_LOGGERS = { + "evergreen", + "git", + "urllib3", +} +CONFIG_DIRECTORY = "generated_burn_in_tags_config" +CONFIG_FILE = "burn_in_tags_gen.json" +EVERGREEN_FILE = "etc/evergreen.yml" +EVG_CONFIG_FILE = ".evergreen.yml" + +COMPILE_TASK = "compile_and_archive_dist_test_TG" +# Burn in tags requires running on RHEL80 currently. +COMPILE_TASK_DISTRO = "rhel80-large" + +TASK_ID_EXPANSION = "task_id" + +ConfigOptions = namedtuple("ConfigOptions", [ + "build_variant", + "run_build_variant", + "base_commit", + "max_revisions", + "branch", + "check_evergreen", + "distro", + "repeat_tests_secs", + "repeat_tests_min", + "repeat_tests_max", + "project", +]) + + +def _get_config_options(expansions_file_data, build_variant, run_build_variant): + """ + Get the configuration to use. + + :param expansions_file_data: Config data file to use. + :param build_variant: The buildvariant the current patch should be compared against to figure + out which tests have changed. + :param run_build_variant: The buildvariant the generated task should be run on. + :return: ConfigOptions for the generated task to use. + """ + base_commit = expansions_file_data["revision"] + max_revisions = int(expansions_file_data["max_revisions"]) + branch = expansions_file_data["branch_name"] + is_patch = expansions_file_data.get("is_patch", False) + check_evergreen = is_patch != "true" + distro = expansions_file_data["distro_id"] + repeat_tests_min = int(expansions_file_data["repeat_tests_min"]) + repeat_tests_max = int(expansions_file_data["repeat_tests_max"]) + repeat_tests_secs = int(expansions_file_data["repeat_tests_secs"]) + project = expansions_file_data["project"] + + return ConfigOptions(build_variant, run_build_variant, base_commit, max_revisions, branch, + check_evergreen, distro, repeat_tests_secs, repeat_tests_min, + repeat_tests_max, project) + + +def _create_evg_build_variant_map(expansions_file_data): + """ + Generate relationship of base buildvariant to generated buildvariant. + + :param expansions_file_data: Config data file to use. + :return: Map of base buildvariants to their generated buildvariants. + """ + burn_in_tag_build_variants = expansions_file_data["burn_in_tag_buildvariants"] + + if burn_in_tag_build_variants: + return { + base_variant: f"{base_variant}-required" + for base_variant in burn_in_tag_build_variants.split(" ") + } + + return {} + + +def _generate_evg_build_variant( + source_build_variant: Variant, + run_build_variant: str, + bypass_build_variant: str, +) -> BuildVariant: + """ + Generate a shrub build variant for the given run build variant. + + :param source_build_variant: The build variant to base configuration on. + :param run_build_variant: The build variant to generate. + :param bypass_build_variant: The build variant to get compile artifacts from. + :return: Shrub build variant configuration. + """ + display_name = f"! {source_build_variant.display_name}" + run_on = source_build_variant.run_on + modules = source_build_variant.modules + + expansions = source_build_variant.expansions + expansions["burn_in_bypass"] = bypass_build_variant + + build_variant = BuildVariant(run_build_variant, display_name, expansions=expansions, + modules=modules, run_on=run_on) + build_variant.add_existing_task(ExistingTask(COMPILE_TASK), distros=[COMPILE_TASK_DISTRO]) + return build_variant + + +# pylint: disable=too-many-arguments,too-many-locals +def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject, + task_expansions: Dict[str, Any], build_variant_map: Dict[str, str], + repos: List[Repo], evg_conf: EvergreenProjectConfig, + install_dir: str) -> None: + """ + Generate burn in tests tasks for a given shrub config and group of build variants. + + :param evergreen_api: Evergreen.py object. + :param shrub_project: Shrub config object that the build variants will be built upon. + :param task_expansions: Dictionary of expansions for the running task. + :param build_variant_map: Map of base buildvariants to their generated buildvariant. + :param repos: Git repositories. + """ + for build_variant, run_build_variant in build_variant_map.items(): + config_options = _get_config_options(task_expansions, build_variant, run_build_variant) + task_id = task_expansions[TASK_ID_EXPANSION] + change_detector = EvergreenFileChangeDetector(task_id, evergreen_api, os.environ) + changed_tests = change_detector.find_changed_tests(repos) + tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests, install_dir) + if tests_by_task: + shrub_build_variant = _generate_evg_build_variant( + evg_conf.get_variant(build_variant), run_build_variant, + task_expansions["build_variant"]) + gen_config = GenerateConfig(build_variant, config_options.project, run_build_variant, + config_options.distro, + include_gen_task=False).validate(evg_conf) + repeat_config = RepeatConfig(repeat_tests_min=config_options.repeat_tests_min, + repeat_tests_max=config_options.repeat_tests_max, + repeat_tests_secs=config_options.repeat_tests_secs) + + burn_in_generator = GenerateBurnInExecutor(gen_config, repeat_config, evergreen_api) + burn_in_generator.generate_tasks_for_variant(tests_by_task, shrub_build_variant) + shrub_project.add_build_variant(shrub_build_variant) + + +def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig, + evergreen_api: RetryingEvergreenApi, repos: List[Repo], install_dir: str): + """ + Execute main program. + + :param task_expansions: Dictionary of expansions for the running task. + :param evg_conf: Evergreen configuration. + :param evergreen_api: Evergreen.py object. + :param repos: Git repositories. + :param install_dir: path to bin directory of a testable installation + """ + shrub_project = ShrubProject.empty() + build_variant_map = _create_evg_build_variant_map(task_expansions) + _generate_evg_tasks(evergreen_api, shrub_project, task_expansions, build_variant_map, repos, + evg_conf, install_dir) + + if not validate_task_generation_limit(shrub_project): + sys.exit(1) + write_file_to_dir(CONFIG_DIRECTORY, CONFIG_FILE, shrub_project.json()) + + +def _configure_logging(verbose: bool): + """ + Configure logging for the application. + + :param verbose: If True set log level to DEBUG. + """ + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", + level=level, + stream=sys.stdout, + ) + for log_name in EXTERNAL_LOGGERS: + logging.getLogger(log_name).setLevel(logging.WARNING) + + +@click.command() +@click.option("--expansion-file", "expansion_file", required=True, + help="Location of expansions file generated by evergreen.") +@click.option("--verbose", is_flag=True) +@click.option("--install-dir", "install_dir", required=True, + help="Path to bin directory of a testable installation") +def main(expansion_file: str, verbose: bool, install_dir: str): + """ + Run new or changed tests in repeated mode to validate their stability. + + burn_in_tags detects jstests that are new or changed since the last git command and then + runs those tests in a loop to validate their reliability. + + \f + :param expansion_file: The expansion file containing the configuration params. + """ + _configure_logging(verbose) + evg_api = RetryingEvergreenApi.get_api(config_file=EVG_CONFIG_FILE) + repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] + expansions_file_data = read_config.read_config_file(expansion_file) + evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) + + burn_in(expansions_file_data, evg_conf, evg_api, repos, install_dir) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 7025a237219..27da0996b63 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Command line utility for determining what jstests have been added or modified.""" -import collections import copy -import json import logging import os.path import shlex @@ -15,7 +13,6 @@ from typing import Optional, Set, Tuple, List, Dict, NamedTuple import click import yaml from git import Repo -from pydantic import BaseModel import structlog from structlog.stdlib import LoggerFactory @@ -40,18 +37,16 @@ EXTERNAL_LOGGERS = { "urllib3", } -DEFAULT_VARIANT = "enterprise-rhel-8-64-bit-dynamic-required" +DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" ENTERPRISE_MODULE_PATH = "src/mongo/db/modules/enterprise" -DEFAULT_REPO_LOCATIONS = ["."] +DEFAULT_REPO_LOCATIONS = [".", f"./{ENTERPRISE_MODULE_PATH}"] REPEAT_SUITES = 2 -DEFAULT_EVG_PROJECT_FILE = "etc/evergreen.yml" +EVERGREEN_FILE = "etc/evergreen.yml" # The executor_file and suite_files defaults are required to make the suite resolver work # correctly. SELECTOR_FILE = "etc/burn_in_tests.yml" SUITE_FILES = ["with_server"] -BURN_IN_TEST_MEMBERSHIP_FILE = "burn_in_test_membership_map_file_for_ci.json" - SUPPORTED_TEST_KINDS = ("fsm_workload_test", "js_test", "json_schema_test", "multi_stmt_txn_passthrough", "parallel_fsm_workload_test", "all_versions_js_test") @@ -186,13 +181,7 @@ def create_executor_list(suites, exclude_suites): parameter. Returns a dict keyed by suite name / executor, value is tests to run under that executor. """ - try: - with open(BURN_IN_TEST_MEMBERSHIP_FILE) as file: - test_membership = collections.defaultdict(list, json.load(file)) - LOGGER.info(f"Using cached test membership file {BURN_IN_TEST_MEMBERSHIP_FILE}.") - except FileNotFoundError: - LOGGER.info("Getting test membership data.") - test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) + test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) memberships = defaultdict(list) for suite in suites: @@ -363,8 +352,7 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, - changed_tests: Set[str], - install_dir: Optional[str]) -> Dict[str, TaskInfo]: + changed_tests: Set[str], install_dir: str) -> Dict[str, TaskInfo]: """ Create a list of tests by task. @@ -379,11 +367,7 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*") changed_tests = filter_tests(changed_tests, exclude_tests) - run_options = "" - if install_dir is not None: - run_options = f"--installDir={shlex.quote(install_dir)}" - buildscripts.resmokelib.parser.set_run_options(run_options) - + buildscripts.resmokelib.parser.set_run_options(f"--installDir={shlex.quote(install_dir)}") if changed_tests: return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites, exclude_tasks) @@ -424,7 +408,7 @@ def _configure_logging(verbose: bool): logging.basicConfig( format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", level=level, - stream=sys.stderr, + stream=sys.stdout, ) for log_name in EXTERNAL_LOGGERS: logging.getLogger(log_name).setLevel(logging.WARNING) @@ -553,45 +537,11 @@ class LocalBurnInExecutor(BurnInExecutor): run_tests(tests_by_task, resmoke_cmd) -class DiscoveredTask(BaseModel): - """ - Model for a discovered task to run. - - * task_name: Name of discovered task. - * test_list: List of tests to run under discovered task. - """ - - task_name: str - test_list: List[str] - - -class DiscoveredTaskList(BaseModel): - """Model for a list of discovered tasks.""" - - discovered_tasks: List[DiscoveredTask] - - -class YamlBurnInExecutor(BurnInExecutor): - """A burn-in executor that outputs discovered tasks as YAML.""" - - def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: - """ - Report the given tasks and their tests to stdout. - - :param tests_by_task: Dictionary of tasks to run with tests to run in each. - """ - discovered_tasks = DiscoveredTaskList(discovered_tasks=[ - DiscoveredTask(task_name=task_name, test_list=task_info.tests) - for task_name, task_info in tests_by_task.items() - ]) - print(yaml.safe_dump(discovered_tasks.dict())) - - class BurnInOrchestrator: """Orchestrate the execution of burn_in_tests.""" def __init__(self, change_detector: FileChangeDetector, burn_in_executor: BurnInExecutor, - evg_conf: EvergreenProjectConfig, install_dir: Optional[str]) -> None: + evg_conf: EvergreenProjectConfig) -> None: """ Create a new orchestrator. @@ -602,9 +552,8 @@ class BurnInOrchestrator: self.change_detector = change_detector self.burn_in_executor = burn_in_executor self.evg_conf = evg_conf - self.install_dir = install_dir - def burn_in(self, repos: List[Repo], build_variant: str) -> None: + def burn_in(self, repos: List[Repo], build_variant: str, install_dir: str) -> None: """ Execute burn in tests for the given git repositories. @@ -615,19 +564,14 @@ class BurnInOrchestrator: LOGGER.info("Found changed tests", files=changed_tests) tests_by_task = create_tests_by_task(build_variant, self.evg_conf, changed_tests, - self.install_dir) + install_dir) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) self.burn_in_executor.execute(tests_by_task) -@click.group() -def cli(): - """Run the cli.""" - pass - - -@cli.command(context_settings=dict(ignore_unknown_options=True)) +# pylint: disable=too-many-function-args +@click.command(context_settings=dict(ignore_unknown_options=True)) @click.option("--no-exec", "no_exec", default=False, is_flag=True, help="Do not execute the found tests.") @click.option("--build-variant", "build_variant", default=DEFAULT_VARIANT, metavar='BUILD_VARIANT', @@ -640,22 +584,18 @@ def cli(): help="The maximum number of times to repeat tests if time option is specified.") @click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS", help="Repeat tests for the given time (in secs).") -@click.option("--yaml", "use_yaml", is_flag=True, default=False, - help="Output discovered tasks in YAML. Tests will not be run.") @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") @click.option( "--origin-rev", "origin_rev", default=None, help="The revision in the mongo repo that changes will be compared against if specified.") -@click.option("--install-dir", "install_dir", type=str, +@click.option("--install-dir", "install_dir", required=True, type=str, help="Path to bin directory of a testable installation") -@click.option("--evg-project-file", "evg_project_file", default=DEFAULT_EVG_PROJECT_FILE, - help="Evergreen project config file") @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED) -def run(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], - repeat_tests_min: Optional[int], repeat_tests_max: Optional[int], - repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool, - origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool, - evg_project_file: Optional[str]) -> None: +# pylint: disable=too-many-arguments,too-many-locals +def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], + repeat_tests_min: Optional[int], repeat_tests_max: Optional[int], + repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool, + origin_rev: Optional[str], install_dir: str) -> None: """ Run new or changed tests in repeated mode to validate their stability. @@ -686,9 +626,6 @@ def run(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], :param resmoke_args: Arguments to pass through to resmoke. :param verbose: Log extra debug information. :param origin_rev: The revision that local changes will be compared against. - :param install_dir: Path to bin directory of a testable installation. - :param use_yaml: Output discovered tasks in YAML. Tests will not be run. - :param evg_project_file: Evergreen project config file. """ _configure_logging(verbose) @@ -698,40 +635,16 @@ def run(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], repeat_tests_num=repeat_tests_num) # yapf: disable repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] - evg_conf = parse_evergreen_file(evg_project_file) + evg_conf = parse_evergreen_file(EVERGREEN_FILE) change_detector = LocalFileChangeDetector(origin_rev) executor = LocalBurnInExecutor(resmoke_args, repeat_config) - if use_yaml: - executor = YamlBurnInExecutor() - elif no_exec: + if no_exec: executor = NopBurnInExecutor() - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) - burn_in_orchestrator.burn_in(repos, build_variant) - - -@cli.command() -def generate_test_membership_map_file_for_ci(): - """ - Generate a file to cache test membership data for CI. - - This command should only be used in CI. The task generator runs many iterations of this script - for many build variants. The bottleneck is that creating the test membership file takes a long time. - Instead, we can cache this data & reuse it in CI for a significant speedup. - - Run this command in CI before running the burn in task generator. - """ - _configure_logging(False) - buildscripts.resmokelib.parser.set_run_options() - - LOGGER.info("Generating burn_in test membership mapping file.") - test_membership = create_test_membership_map(test_kind=SUPPORTED_TEST_KINDS) - with open(BURN_IN_TEST_MEMBERSHIP_FILE, "w") as file: - json.dump(test_membership, file) - LOGGER.info( - f"Finished writing burn_in test membership mapping to {BURN_IN_TEST_MEMBERSHIP_FILE}") + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) + burn_in_orchestrator.burn_in(repos, build_variant, install_dir) if __name__ == "__main__": - cli() + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 8f331b5e1ee..fb63b2201ec 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -320,7 +320,7 @@ class Variant(object): def is_required_variant(self) -> bool: """Return True if the variant is a required variant.""" - return self.display_name.startswith("!") + return self.display_name.startswith("! ") def get_task(self, task_name): """Return the task with the given name as an instance of VariantTask. diff --git a/buildscripts/combine_reports.py b/buildscripts/combine_reports.py index dd5564f4454..8952a564943 100755 --- a/buildscripts/combine_reports.py +++ b/buildscripts/combine_reports.py @@ -25,7 +25,7 @@ def report_exit(combined_test_report): """Return report exit code. The exit code of this script is based on the following: - 0: All tests have status "pass". + 0: All tests have status "pass", or only non-dynamic tests have status "silentfail". 31: At least one test has status "fail" or "timeout". Note: A test can be considered dynamic if its name contains a ":" character. """ diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py index 0f79b4349fe..977716a7c0f 100644 --- a/buildscripts/debugsymb_mapper.py +++ b/buildscripts/debugsymb_mapper.py @@ -4,13 +4,11 @@ import json import logging import os import pathlib -import re import shutil import subprocess import sys import time -from json import JSONDecoder -from typing import Optional, Tuple, Generator, Dict, List, NamedTuple +import typing import requests @@ -22,95 +20,30 @@ from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download from buildscripts.build_system_options import PathOptions -BUILD_INFO_RE = re.compile(r"Build Info: ({(\n.*)*})") -MONGOD = "mongod" +class LinuxBuildIDExtractor: + """Parse readlef command output & extract Build ID.""" -class CmdClient: - """Client to run commands.""" + default_executable_path = "readelf" - @staticmethod - def run(args: List[str]) -> str: - """ - Run command with args. - - :param args: Argument list. - :return: Command output. - """ - - out = subprocess.run(args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - check=False) - return out.stdout.strip().decode() - - -class BuildIdOutput(NamedTuple): - """ - Build ID and command output. - - * build_id: Build ID or None. - * cmd_output: Command output. - """ - - build_id: Optional[str] - cmd_output: str - - -class BinVersionOutput(NamedTuple): - """ - Mongodb bin version and command output. - - * mongodb_version: Bin version. - * cmd_output: Command output. - """ - - mongodb_version: Optional[str] - cmd_output: str - - -class CmdOutputExtractor: - """Data extractor from command output.""" - - def __init__(self, cmd_client: Optional[CmdClient] = None, - json_decoder: Optional[JSONDecoder] = None) -> None: - """ - Initialize. - - :param cmd_client: Client to run commands. - :param json_decoder: JSONDecoder object. - """ - self.cmd_client = cmd_client if cmd_client is not None else CmdClient() - self.json_decoder = json_decoder if json_decoder is not None else JSONDecoder() - - def get_build_id(self, bin_path: str) -> BuildIdOutput: - """ - Get build ID from readelf command. + def __init__(self, executable_path: str = None): + """Initialize instance.""" - :param bin_path: Path to binary of the build. - :return: Build ID or None and command output. - """ - out = self.cmd_client.run(["readelf", "-n", bin_path]) - build_id = self._extract_build_id(out) - return BuildIdOutput(build_id, out) + self.executable_path = executable_path or self.default_executable_path - def get_bin_version(self, bin_path: str) -> BinVersionOutput: - """ - Get mongodb bin version from `{bin} --version` command. + def callreadelf(self, binary_path: str) -> str: + """Call readelf command for given binary & return string output.""" - :param bin_path: Path to mongodb binary. - :return: Bin version or None and command output. - """ - out = self.cmd_client.run([os.path.abspath(bin_path), "--version"]) - mongodb_version = self._get_mongodb_version(out) - return BinVersionOutput(mongodb_version, out) + args = [self.executable_path, "-n", binary_path] + process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + process.wait() + return process.stdout.read().decode() @staticmethod - def _extract_build_id(out: str) -> Optional[str]: - """ - Parse readelf output and extract Build ID from it. + def extractbuildid(out: str) -> typing.Optional[str]: + """Parse readelf output and extract Build ID from it.""" - :param out: readelf command output. - :return: Build ID on None. - """ build_id = None for line in out.splitlines(): line = line.strip() @@ -120,21 +53,13 @@ class CmdOutputExtractor: build_id = line.split(': ')[1] return build_id - def _get_mongodb_version(self, out: str) -> Optional[str]: - """ - Parse version command output and extract mongodb version. - - :param out: Version command output. - :return: Version or None. - """ - mongodb_version = None + def run(self, binary_path: str) -> typing.Tuple[str, str]: + """Perform all necessary actions to get Build ID.""" - search = BUILD_INFO_RE.search(out) - if search: - build_info = self.json_decoder.decode(search.group(1)) - mongodb_version = build_info.get("version") + readelfout = self.callreadelf(binary_path) + buildid = self.extractbuildid(readelfout) - return mongodb_version + return buildid, readelfout class DownloadOptions(object): @@ -158,29 +83,24 @@ class Mapper: default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com" default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache') - selected_binaries = ('mongos', 'mongod', 'mongo') + selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug') default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') - def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str, - client_secret: str, cache_dir: str = None, web_service_base_url: str = None, + def __init__(self, version: str, client_id: str, client_secret: str, variant: str, + cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. - :param evg_version: Evergreen version ID. - :param evg_variant: Evergreen build variant name. - :param is_san_variant: Whether build variant is sanitizer build. - :param client_id: Client id for Okta Oauth. - :param client_secret: Secret key for Okta Oauth. - :param cache_dir: Full path to cache directory as a string. - :param web_service_base_url: URL of symbolizer web service. - :param logger: Debug symbols mapper logger. + :param version: version string + :param variant: build variant string + :param cache_dir: full path to cache directory as a string + :param web_service_base_url: URL of symbolizer web service """ - self.evg_version = evg_version - self.evg_variant = evg_variant - self.is_san_variant = is_san_variant + self.version = version + self.variant = variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url @@ -193,8 +113,8 @@ class Mapper: self.http_client = requests.Session() self.multiversion_setup = SetupMultiversion( - DownloadOptions(download_symbols=True, download_binaries=True), - variant=self.evg_variant, ignore_failed_push=True) + DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant, + ignore_failed_push=True) self.debug_symbols_url = None self.url = None self.configs = Configs( @@ -219,7 +139,7 @@ class Mapper: data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get("expire_time") if time.time() < expire_time: - # credentials haven't expired yet + # credentials hasn't expired yet self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) return @@ -227,7 +147,7 @@ class Mapper: configs=self.configs) self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) - # write credentials to local file for further usage + # write credentials to local file for further useage with open(self.default_creds_file_path, "w") as cfile: cfile.write( json.dumps({ @@ -264,19 +184,17 @@ class Mapper: def setup_urls(self): """Set up URLs using multiversion.""" - urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant) + urlinfo = self.multiversion_setup.get_urls(self.version, self.variant) + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) binaries_url = urlinfo.urls.get("Binaries", "") - if self.is_san_variant: - # Sanitizer builds are not stripped and contain debug symbols - download_symbols_url = binaries_url - else: - download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get( - "mongo-debugsymbols.zip") + + if not download_symbols_url: + download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None) if not download_symbols_url: self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", - self.evg_version, urlinfo.urls) + self.version, urlinfo.urls) raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}") self.debug_symbols_url = download_symbols_url @@ -315,64 +233,54 @@ class Mapper: tarball_full_path = download.download_from_s3(url) return tarball_full_path - def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]: + def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]: """ Extract build id from binaries and creates new dict using them. :return: mapped data as dict """ - extractor = CmdOutputExtractor() + readelf_extractor = LinuxBuildIDExtractor() + + debug_symbols_path = self.download(self.debug_symbols_url) + debug_symbols_unpacked_path = self.unpack(debug_symbols_path) binaries_path = self.download(self.url) binaries_unpacked_path = self.unpack(binaries_path) - # we need to analyze two directories: main binary folder and + # we need to analyze two directories: main binary folder inside debug-symbols and # shared libraries folder inside binaries. # main binary folder holds main binaries, like mongos, mongod, mongo ... # shared libraries folder holds shared libraries, tons of them. # some build variants do not contain shared libraries. + debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test') binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test') + self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s", + os.listdir(debug_symbols_unpacked_path)) self.logger.info("INSIDE unpacked binaries/dist-test: %s", os.listdir(binaries_unpacked_path)) - mongod_bin = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name, - MONGOD) - bin_version_output = extractor.get_bin_version(mongod_bin) - - if bin_version_output.mongodb_version is None: - self.logger.error("mongodb version could not be extracted. \n`%s --version` output: %s", - mongod_bin, bin_version_output.cmd_output) - return - else: - self.logger.info("Extracted mongodb version: %s", bin_version_output.mongodb_version) - # start with main binary folder for binary in self.selected_binaries: - full_bin_path = os.path.join(binaries_unpacked_path, + full_bin_path = os.path.join(debug_symbols_unpacked_path, self.path_options.main_binary_folder_name, binary) if not os.path.exists(full_bin_path): self.logger.error("Could not find binary at %s", full_bin_path) - continue + return - build_id_output = extractor.get_build_id(full_bin_path) + build_id, readelf_out = readelf_extractor.run(full_bin_path) - if not build_id_output.build_id: + if not build_id: self.logger.error("Build ID couldn't be extracted. \nReadELF output %s", - build_id_output.cmd_output) - continue - else: - self.logger.info("Extracted build ID: %s", build_id_output.build_id) + readelf_out) + return yield { - 'url': self.url, - 'debug_symbols_url': self.debug_symbols_url, - 'build_id': build_id_output.build_id, - 'file_name': binary, - 'version': bin_version_output.mongodb_version, + 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, + 'file_name': binary, 'version': self.version } # move to shared libraries folder. @@ -396,23 +304,20 @@ class Mapper: if not os.path.exists(sofile_path): self.logger.error("Could not find binary at %s", sofile_path) - continue + return - build_id_output = extractor.get_build_id(sofile_path) + build_id, readelf_out = readelf_extractor.run(sofile_path) - if not build_id_output.build_id: - self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", - build_id_output.cmd_output) - continue - else: - self.logger.info("Extracted build ID: %s", build_id_output.build_id) + if not build_id: + self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out) + return yield { 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, - 'build_id': build_id_output.build_id, + 'build_id': build_id, 'file_name': sofile, - 'version': bin_version_output.mongodb_version, + 'version': self.version, } def run(self): @@ -425,7 +330,6 @@ class Mapper: # mappings is a generator, we iterate over to generate mappings on the go for mapping in mappings: - self.logger.info("Creating mapping %s", mapping) response = self.http_client.post('/'.join((self.web_service_base_url, 'add')), json=mapping) if response.status_code != 200: @@ -440,20 +344,18 @@ def make_argument_parser(parser=None, **kwargs): if parser is None: parser = argparse.ArgumentParser(**kwargs) - parser.add_argument("--version") - parser.add_argument("--client-id") - parser.add_argument("--client-secret") - parser.add_argument("--variant") - parser.add_argument("--is-san-variant", action="store_true") - parser.add_argument("--web-service-base-url", default="") + parser.add_argument('--version') + parser.add_argument('--client-id') + parser.add_argument('--client-secret') + parser.add_argument('--variant') + parser.add_argument('--web-service-base-url', default="") return parser def main(options): """Execute mapper here. Main entry point.""" - mapper = Mapper(evg_version=options.version, evg_variant=options.variant, - is_san_variant=options.is_san_variant, client_id=options.client_id, + mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id, client_secret=options.client_secret, web_service_base_url=options.web_service_base_url) diff --git a/buildscripts/download_sys_perf_binaries.py b/buildscripts/download_sys_perf_binaries.py deleted file mode 100644 index f94a328fd7b..00000000000 --- a/buildscripts/download_sys_perf_binaries.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -"""Download the binaries from a previous sys-perf run.""" - -import argparse -import requests - -BASE_URI = 'https://evergreen.mongodb.com/rest/v2/' - - -def _get_auth_headers(evergreen_api_user, evergreen_api_key): - return { - 'Api-User': evergreen_api_user, - 'Api-Key': evergreen_api_key, - } - - -def _get_build_id(build_variant_name, version_id, auth_headers): - url = BASE_URI + 'versions/' + version_id - response = requests.get(url, headers=auth_headers) - if response.status_code != 200: - raise ValueError('Invalid version_id:', version_id) - - version_json = response.json() - build_variants = version_json['build_variants_status'] - for build_variant in build_variants: - if build_variant['build_variant'] == build_variant_name: - return build_variant['build_id'] - - raise RuntimeError('The compile-variant ' + build_variant_name + - ' does not exist for the build with version_id ' + version_id) - - -# All of our sys-perf compile variants have exactly one task, -# so we can safely select the first one from the build variants tasks. -def _get_task_id(build_id, auth_headers): - url = BASE_URI + 'builds/' + build_id - response = requests.get(url, headers=auth_headers) - if response.status_code != 200: - raise RuntimeError('Unexpected error when trying to reach' + url) - - build_json = response.json() - task_list = build_json['tasks'] - if len(task_list) != 1: - raise RuntimeError('Recieved unexpected tasklist:', task_list) - return task_list[0] - - -# The API used here always grabs the latest execution -def _get_binary_details(task_id, auth_headers): - url = BASE_URI + 'tasks/' + task_id - response = requests.get(url, headers=auth_headers) - if response.status_code != 200: - raise RuntimeError('Unexpected error when trying to reach' + url) - - task_json = response.json() - if task_json['status'] != 'success': - raise RuntimeError('The task ' + task_id + ' did not run sucessfully') - - # The binary will always be the first artifact, unless we make large changes to system_perf.yml - artifacts = task_json['artifacts'] - if len(artifacts) > 0 and artifacts[0]['name'].startswith('mongo'): - return artifacts[0] - raise RuntimeError('Unexpected list of artifacts:' + artifacts) - - -def _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key): - auth_headers = _get_auth_headers(evergreen_api_user, evergreen_api_key) - build_id = _get_build_id(build_variant, version_id, auth_headers) - task_id = _get_task_id(build_id, auth_headers) - binary_json = _get_binary_details(task_id, auth_headers) - return binary_json['url'] - - -def _download_binary_file(url, save_path): - response = requests.get(url, stream=True) - if response.status_code == 200: - with open(save_path, 'wb') as file: - for chunk in response.iter_content(chunk_size=1024): - file.write(chunk) - else: - raise RuntimeError('Failed to download the file ' + url) - - -def _download_sys_perf_binaries(version_id, build_variant, evergreen_api_user, evergreen_api_key): - url = _get_binary_url(version_id, build_variant, evergreen_api_user, evergreen_api_key) - _download_binary_file(url, 'binary.tar.gz') - - -if __name__ == '__main__': - argParser = argparse.ArgumentParser() - argParser.add_argument("-v", "--version_id", - help="Evergreen version_id from which binaries will be downloaded") - argParser.add_argument("-b", "--build_variant", - help="Build variant for which binaries will be downloaded") - argParser.add_argument( - "-u", "--evergreen_api_user", - help="Evergreen API user, see https://spruce.mongodb.com/preferences/cli") - argParser.add_argument("-k", "--evergreen_api_key", - help="Evergreen API key, see https://spruce.mongodb.com/preferences/cli") - args = argParser.parse_args() - - _download_sys_perf_binaries(args.version_id, args.build_variant, args.evergreen_api_user, - args.evergreen_api_key) diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py index bcabaa2e9cc..8d9330b4240 100755 --- a/buildscripts/errorcodes.py +++ b/buildscripts/errorcodes.py @@ -46,7 +46,6 @@ _CODE_PATTERNS = [ r"(?:StatusOK)?" r"(?:WithContext)?" r"\s*\(", - r"MONGO_UNREACHABLE_TASSERT\(", # DBException and AssertionException constructors r"(?:DB|Assertion)Exception\s*[({]", # Calls to all LOGV2* variants diff --git a/buildscripts/eslint.py b/buildscripts/eslint.py index b4d1d97df5c..07cbb0ddc5c 100755 --- a/buildscripts/eslint.py +++ b/buildscripts/eslint.py @@ -19,13 +19,12 @@ import sys import tarfile import tempfile import threading -import platform from typing import Optional import urllib.error import urllib.parse import urllib.request -from distutils import spawn +from distutils import spawn # pylint: disable=no-name-in-module from optparse import OptionParser import structlog @@ -51,17 +50,14 @@ ESLINT_VERSION = "7.22.0" # Name of ESLint as a binary. ESLINT_PROGNAME = "eslint" -# Arch of running system -ARCH = platform.machine() if platform.machine() != "aarch64" else "arm64" - # URL location of our provided ESLint binaries. ESLINT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ - ESLINT_VERSION + "-linux-" + ARCH + ".tar.gz" + ESLINT_VERSION + "-linux.tar.gz" ESLINT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/eslint-" + \ ESLINT_VERSION + "-darwin.tar.gz" # Path in the tarball to the ESLint binary. -ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$operating_system-$arch") +ESLINT_SOURCE_TAR_BASE = string.Template(ESLINT_PROGNAME + "-$platform-$arch") LOGGER = structlog.get_logger(__name__) @@ -80,15 +76,15 @@ def extract_eslint(tar_path, target_file): tarfp.close() -def get_eslint_from_cache(dest_file, operating_system, arch): +def get_eslint_from_cache(dest_file, platform, arch): """Get ESLint binary from mongodb's cache.""" # Get URL - if operating_system == "Linux": + if platform == "Linux": url = ESLINT_HTTP_LINUX_CACHE - elif operating_system == "Darwin": + elif platform == "Darwin": url = ESLINT_HTTP_DARWIN_CACHE else: - raise ValueError('ESLint is not available as a binary for ' + operating_system) + raise ValueError('ESLint is not available as a binary for ' + platform) dest_dir = tempfile.gettempdir() temp_tar_file = os.path.join(dest_dir, "temp.tar.gz") @@ -97,9 +93,9 @@ def get_eslint_from_cache(dest_file, operating_system, arch): print("Downloading ESLint %s from %s, saving to %s" % (ESLINT_VERSION, url, temp_tar_file)) urllib.request.urlretrieve(url, temp_tar_file) + # pylint: disable=too-many-function-args print("Extracting ESLint %s to %s" % (ESLINT_VERSION, dest_file)) - eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(operating_system=operating_system, - arch=arch) + eslint_distfile = ESLINT_SOURCE_TAR_BASE.substitute(platform=platform, arch=arch) extract_eslint(temp_tar_file, eslint_distfile) shutil.move(eslint_distfile, dest_file) @@ -113,7 +109,7 @@ class ESLint(object): # Initialize ESLint configuration information if sys.platform.startswith("linux"): - self.arch = ARCH + self.arch = "x86_64" self.tar_path = None elif sys.platform == "darwin": self.arch = "x86_64" diff --git a/buildscripts/eslint/README.md b/buildscripts/eslint/README.md index d4f0d7d8402..f8f6b692dfa 100644 --- a/buildscripts/eslint/README.md +++ b/buildscripts/eslint/README.md @@ -21,7 +21,6 @@ "pkg": { "scripts": [ "conf/**/*", "lib/**/*", "messages/**/*" ], "targets": [ "linux-x64", "macos-x64" ] - # "targets": [ "linux-arm64" ] }, ``` 6. Run pkg command to make ESLint executables. @@ -39,10 +38,6 @@ ``` eslint-macos --help ``` - or (if you are on arm) - ``` - eslint --help - ``` (*) If executable fails to find some .js files there are [extra steps](#extra-steps) required to be done before step 6. @@ -53,25 +48,19 @@ Rename produced files. ``` mv eslint-linux eslint-Linux-x86_64 mv eslint-macos eslint-Darwin-x86_64 -# arm -# mv eslint eslint-Linux-arm64 ``` -Archive files. (No leading v in version e.g. 8.28.0 NOT v8.28.0) +Archive files. ``` -tar -czvf eslint-${version}-linux-x86_64.tar.gz eslint-Linux-x86_64 +tar -czvf eslint-${version}-linux.tar.gz eslint-Linux-x86_64 tar -czvf eslint-${version}-darwin.tar.gz eslint-Darwin-x86_64 -# arm -# tar -czvf eslint-${version}-linux-arm64.tar.gz eslint-Linux-arm64 ``` ### Upload archives to `boxes.10gen.com` Archives should be available by the following links: ``` -https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux-x86_64.tar.gz +https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux.tar.gz https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-darwin.tar.gz -# arm -# https://s3.amazonaws.com/boxes.10gen.com/build/eslint-${version}-linux-arm64.tar.gz ``` Build team has an access to do that. You can create a build ticket in Jira for them to do it @@ -88,7 +77,7 @@ ESLINT_VERSION = "${version}" Unfortunately pkg doesn't work well with `require(variable)` statements and force include files using `assets` or `scripts` options might not help. -For the ESLint version 7.22.0 and 8.28.0 the following change was applied to the +For the ESLint version 7.22.0 the following change was applied to the source code to make everything work: ``` diff --git a/lib/cli-engine/cli-engine.js b/lib/cli-engine/cli-engine.js diff --git a/buildscripts/evergreen_activate_gen_tasks.py b/buildscripts/evergreen_activate_gen_tasks.py index 1ccd95642c5..13eed325701 100755 --- a/buildscripts/evergreen_activate_gen_tasks.py +++ b/buildscripts/evergreen_activate_gen_tasks.py @@ -21,9 +21,6 @@ from buildscripts.util.taskname import remove_gen_suffix LOGGER = structlog.getLogger(__name__) EVG_CONFIG_FILE = "./.evergreen.yml" -BURN_IN_TAGS = "burn_in_tags" -BURN_IN_TESTS = "burn_in_tests" -BURN_IN_VARIANT_SUFFIX = "generated-by-burn-in-tags" class EvgExpansions(BaseModel): @@ -31,12 +28,10 @@ class EvgExpansions(BaseModel): Evergreen expansions file contents. build_id: ID of build being run. - version_id: ID of version being run. task_name: Name of task creating the generated configuration. """ build_id: str - version_id: str task_name: str @classmethod @@ -50,7 +45,7 @@ class EvgExpansions(BaseModel): return remove_gen_suffix(self.task_name) -def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None: +def activate_task(build_id: str, task_name: str, evg_api: EvergreenApi) -> None: """ Activate the given task in the specified build. @@ -58,28 +53,27 @@ def activate_task(expansions: EvgExpansions, evg_api: EvergreenApi) -> None: :param task_name: Name of task to activate. :param evg_api: Evergreen API client. """ - if expansions.task == BURN_IN_TAGS: - version = evg_api.version_by_id(expansions.version_id) - burn_in_build_variants = [ - variant for variant in version.build_variants_map.keys() - if variant.endswith(BURN_IN_VARIANT_SUFFIX) - ] - for build_variant in burn_in_build_variants: - build_id = version.build_variants_map[build_variant] - task_list = evg_api.tasks_by_build(build_id) - - for task in task_list: - if task.display_name == BURN_IN_TESTS: - LOGGER.info("Activating task", task_id=task.task_id, - task_name=task.display_name) - evg_api.configure_task(task.task_id, activated=True) - - else: - task_list = evg_api.tasks_by_build(expansions.build_id) - for task in task_list: - if task.display_name == expansions.task: - LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) - evg_api.configure_task(task.task_id, activated=True) + build = evg_api.build_by_id(build_id) + task_list = build.get_tasks() + for task in task_list: + if task.display_name == task_name: + LOGGER.info("Activating task", task_id=task.task_id, task_name=task.display_name) + evg_api.configure_task(task.task_id, activated=True) + + # if any(ARCHIVE_DIST_TEST_TASK in dependency["id"] for dependency in task.depends_on): + # _activate_archive_debug_symbols(evg_api, task_list) + + +# def _activate_archive_debug_symbols(evg_api: EvergreenApi, task_list): +# debug_iter = filter(lambda tsk: tsk.display_name == ACTIVATE_ARCHIVE_DIST_TEST_DEBUG_TASK, +# task_list) +# activate_symbol_tasks = list(debug_iter) +# +# if len(activate_symbol_tasks) == 1: +# activated_symbol_task = activate_symbol_tasks[0] +# if not activated_symbol_task.activated: +# LOGGER.info("Activating debug symbols archival", task_id=activated_symbol_task.task_id) +# evg_api.configure_task(activated_symbol_task.task_id, activated=True) @click.command() @@ -102,7 +96,7 @@ def main(expansion_file: str, evergreen_config: str, verbose: bool) -> None: expansions = EvgExpansions.from_yaml_file(expansion_file) evg_api = RetryingEvergreenApi.get_api(config_file=evergreen_config) - activate_task(expansions, evg_api) + activate_task(expansions.build_id, expansions.task, evg_api) if __name__ == "__main__": diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py new file mode 100644 index 00000000000..37a0d2874ff --- /dev/null +++ b/buildscripts/evergreen_burn_in_tests.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +"""Wrapper around burn_in_tests for evergreen execution.""" +import logging +import os +import sys +from datetime import datetime, timedelta +from math import ceil +from typing import Optional, List, Dict, Set, NamedTuple + +import click +import requests +import structlog +from git import Repo +from shrub.v2 import ShrubProject, BuildVariant, Task, TaskDependency, ExistingTask +from evergreen import RetryingEvergreenApi, EvergreenApi + +from buildscripts.burn_in_tests import RepeatConfig, BurnInExecutor, TaskInfo, FileChangeDetector, \ + DEFAULT_REPO_LOCATIONS, BurnInOrchestrator +from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig +from buildscripts.patch_builds.change_data import RevisionMap +from buildscripts.patch_builds.evg_change_data import generate_revision_map_from_manifest +from buildscripts.patch_builds.task_generation import TimeoutInfo, resmoke_commands, \ + validate_task_generation_limit +from buildscripts.task_generation.constants import CONFIG_FILE, EVERGREEN_FILE, ARCHIVE_DIST_TEST_DEBUG_TASK, \ + BACKPORT_REQUIRED_TAG, RUN_TESTS, EXCLUDES_TAGS_FILE +from buildscripts.task_generation.suite_split import SubSuite, GeneratedSuite +from buildscripts.task_generation.task_types.resmoke_tasks import EXCLUDE_TAGS +from buildscripts.util.fileops import write_file +from buildscripts.util.taskname import name_generated_task +from buildscripts.util.teststats import TestRuntime, HistoricTaskData + +DEFAULT_PROJECT = "mongodb-mongo-master" +DEFAULT_VARIANT = "enterprise-rhel-80-64-bit-dynamic-required" +BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen" +BURN_IN_TESTS_TASK = "burn_in_tests" +BURN_IN_ENV_VAR = "BURN_IN_TESTS" +AVG_TEST_RUNTIME_ANALYSIS_DAYS = 14 +AVG_TEST_SETUP_SEC = 4 * 60 +AVG_TEST_TIME_MULTIPLIER = 3 +MIN_AVG_TEST_OVERFLOW_SEC = float(60) +MIN_AVG_TEST_TIME_SEC = 5 * 60 + +LOGGER = structlog.getLogger(__name__) +EXTERNAL_LOGGERS = { + "evergreen", + "git", + "urllib3", +} + + +def _configure_logging(verbose: bool): + """ + Configure logging for the application. + + :param verbose: If True set log level to DEBUG. + """ + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", + level=level, + stream=sys.stdout, + ) + for log_name in EXTERNAL_LOGGERS: + logging.getLogger(log_name).setLevel(logging.WARNING) + + +class GenerateConfig(object): + """Configuration for how to generate tasks.""" + + def __init__(self, build_variant: str, project: str, run_build_variant: Optional[str] = None, + distro: Optional[str] = None, task_id: Optional[str] = None, + task_prefix: str = "burn_in", include_gen_task: bool = True) -> None: + # pylint: disable=too-many-arguments,too-many-locals + """ + Create a GenerateConfig. + + :param build_variant: Build variant to get tasks from. + :param project: Project to run tasks on. + :param run_build_variant: Build variant to run new tasks on. + :param distro: Distro to run tasks on. + :param task_id: Evergreen task being run under. + :param task_prefix: Prefix to include in generated task names. + :param include_gen_task: Indicates the "_gen" task should be grouped in the display task. + """ + self.build_variant = build_variant + self._run_build_variant = run_build_variant + self.distro = distro + self.project = project + self.task_id = task_id + self.task_prefix = task_prefix + self.include_gen_task = include_gen_task + + @property + def run_build_variant(self): + """Build variant tasks should run against.""" + if self._run_build_variant: + return self._run_build_variant + return self.build_variant + + def validate(self, evg_conf: EvergreenProjectConfig): + """ + Raise an exception if this configuration is invalid. + + :param evg_conf: Evergreen configuration. + :return: self. + """ + self._check_variant(self.build_variant, evg_conf) + return self + + @staticmethod + def _check_variant(build_variant: str, evg_conf: EvergreenProjectConfig): + """ + Check if the build_variant is found in the evergreen file. + + :param build_variant: Build variant to check. + :param evg_conf: Evergreen configuration to check against. + """ + if not evg_conf.get_variant(build_variant): + raise ValueError(f"Build variant '{build_variant}' not found in Evergreen file") + + +def _parse_avg_test_runtime(test: str, + task_avg_test_runtime_stats: List[TestRuntime]) -> Optional[float]: + """ + Parse list of test runtimes to find runtime for particular test. + + :param task_avg_test_runtime_stats: List of average historic runtimes of tests. + :param test: Test name. + :return: Historical average runtime of the test. + """ + for test_stat in task_avg_test_runtime_stats: + if test_stat.test_name == test: + return test_stat.runtime + return None + + +def _calculate_timeout(avg_test_runtime: float) -> int: + """ + Calculate timeout_secs for the Evergreen task. + + :param avg_test_runtime: How long a test has historically taken to run. + :return: The test runtime times AVG_TEST_TIME_MULTIPLIER, or MIN_AVG_TEST_TIME_SEC (whichever + is higher). + """ + return max(MIN_AVG_TEST_TIME_SEC, ceil(avg_test_runtime * AVG_TEST_TIME_MULTIPLIER)) + + +def _calculate_exec_timeout(repeat_config: RepeatConfig, avg_test_runtime: float) -> int: + """ + Calculate exec_timeout_secs for the Evergreen task. + + :param repeat_config: Information about how the test will repeat. + :param avg_test_runtime: How long a test has historically taken to run. + :return: repeat_tests_secs + an amount of padding time so that the test has time to finish on + its final run. + """ + LOGGER.debug("Calculating exec timeout", repeat_config=repeat_config, + avg_test_runtime=avg_test_runtime) + repeat_tests_secs = repeat_config.repeat_tests_secs + if avg_test_runtime > repeat_tests_secs and repeat_config.repeat_tests_min: + # If a single execution of the test takes longer than the repeat time, then we don't + # have to worry about the repeat time at all and can just use the average test runtime + # and minimum number of executions to calculate the exec timeout value. + return ceil(avg_test_runtime * AVG_TEST_TIME_MULTIPLIER * repeat_config.repeat_tests_min) + + test_execution_time_over_limit = avg_test_runtime - (repeat_tests_secs % avg_test_runtime) + test_execution_time_over_limit = max(MIN_AVG_TEST_OVERFLOW_SEC, test_execution_time_over_limit) + return ceil(repeat_tests_secs + (test_execution_time_over_limit * AVG_TEST_TIME_MULTIPLIER) + + AVG_TEST_SETUP_SEC) + + +class BurnInGenTaskParams(NamedTuple): + """Parameters describing how a specific resmoke burn in suite should be generated.""" + + resmoke_args: str + require_multiversion_setup: bool + distro: str + + +class BurnInGenTaskService: + """Class to generate task configurations.""" + + def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig, + task_runtime_stats: List[TestRuntime]) -> None: + """ + Create a new task generator. + + :param generate_config: Generate configuration to use. + :param repeat_config: Repeat configuration to use. + :param task_runtime_stats: Historic runtime of tests associated with task. + """ + self.generate_config = generate_config + self.repeat_config = repeat_config + self.task_runtime_stats = task_runtime_stats + + def generate_timeouts(self, test: str) -> TimeoutInfo: + """ + Add timeout.update command to list of commands for a burn in execution task. + + :param test: Test name. + :return: TimeoutInfo to use. + """ + if self.task_runtime_stats: + avg_test_runtime = _parse_avg_test_runtime(test, self.task_runtime_stats) + if avg_test_runtime: + LOGGER.debug("Avg test runtime", test=test, runtime=avg_test_runtime) + + timeout = _calculate_timeout(avg_test_runtime) + exec_timeout = _calculate_exec_timeout(self.repeat_config, avg_test_runtime) + LOGGER.debug("Using timeout overrides", exec_timeout=exec_timeout, timeout=timeout) + timeout_info = TimeoutInfo.overridden(exec_timeout, timeout) + + LOGGER.debug("Override runtime for test", test=test, timeout=timeout_info) + return timeout_info + + return TimeoutInfo.default_timeout() + + def _generate_run_tests_vars(self, task_name: str, suite_name: str, params: BurnInGenTaskParams, + test_arg: str) -> Dict[str, str]: + run_test_vars = {"suite": suite_name} + + resmoke_args = f"{params.resmoke_args} {self.repeat_config.generate_resmoke_options()} {test_arg}" + + if params.require_multiversion_setup: + run_test_vars["require_multiversion_setup"] = params.require_multiversion_setup + + # TODO: inspect the suite for version instead of doing string parsing on the name. + if "last_continuous" in suite_name: + run_test_vars["multiversion_exclude_tags_version"] = "last_continuous" + resmoke_args += f" --tagFile={EXCLUDES_TAGS_FILE}" + elif "last_lts" in suite_name: + run_test_vars["multiversion_exclude_tags_version"] = "last_lts" + resmoke_args += f" --tagFile={EXCLUDES_TAGS_FILE}" + + resmoke_args += f" --excludeWithAnyTags={EXCLUDE_TAGS},{task_name}_{BACKPORT_REQUIRED_TAG} " + + run_test_vars["resmoke_args"] = resmoke_args + + return run_test_vars + + def _generate_task_name(self, gen_suite: GeneratedSuite, index: int) -> str: + """ + Generate a subtask name. + + :param gen_suite: GeneratedSuite object. + :param index: Index of subtask. + :return: Name to use for generated sub-task. + """ + prefix = self.generate_config.task_prefix + task_name = gen_suite.task_name + + return name_generated_task(f"{prefix}:{task_name}", index, len(gen_suite), + self.generate_config.run_build_variant) + + def generate_tasks(self, gen_suite: GeneratedSuite, params: BurnInGenTaskParams) -> Set[Task]: + """Create the task configuration for the given test using the given index.""" + + tasks = set() + for index, suite in enumerate(gen_suite.sub_suites): + if len(suite.test_list) != 1: + raise ValueError( + f"Can only run one test per suite in burn-in; got {suite.test_list}") + test_name = suite.test_list[0] + test_unix_style = test_name.replace('\\', '/') + run_tests_vars = self._generate_run_tests_vars( + gen_suite.task_name, gen_suite.suite_name, params, test_unix_style) + + timeout_cmd = self.generate_timeouts(test_name) + commands = resmoke_commands(RUN_TESTS, run_tests_vars, timeout_cmd, + params.require_multiversion_setup) + dependencies = {TaskDependency(ARCHIVE_DIST_TEST_DEBUG_TASK)} + + tasks.add(Task(self._generate_task_name(gen_suite, index), commands, dependencies)) + + return tasks + + +class EvergreenFileChangeDetector(FileChangeDetector): + """A file changes detector for detecting test change in evergreen.""" + + def __init__(self, task_id: str, evg_api: EvergreenApi, env_map: Dict[str, str]) -> None: + """ + Create a new evergreen file change detector. + + :param task_id: Id of task being run under. + :param evg_api: Evergreen API client. + :param env_map: Map of environment variables. + """ + self.task_id = task_id + self.evg_api = evg_api + self.env_map = env_map + + def create_revision_map(self, repos: List[Repo]) -> RevisionMap: + """ + Create a map of the repos and the given revisions to diff against. + + :param repos: List of repos being tracked. + :return: Map of repositories and revisions to diff against. + """ + return generate_revision_map_from_manifest(repos, self.task_id, self.evg_api) + + def find_changed_tests(self, repos: List[Repo]) -> Set[str]: + """ + Find the list of tests that have changed. + + :param repos: List of repos to check. + :return: Set of all test files that have changed. + """ + tests_set = super().find_changed_tests(repos) + if BURN_IN_ENV_VAR in self.env_map: + # The burn in env var can be set to a list of tests the user has manually specified + # should be included. Add those to the already discovered tests. + tests_set.update(self.env_map[BURN_IN_ENV_VAR].split(",")) + return tests_set + + +def _tests_dict_to_generated_suites(task_info: TaskInfo, tests_runtimes: List[TestRuntime]): + """Convert diction of tests to `GenerateSuite` objects to conform to the *GenTaskService interface.""" + sub_suites = [] + for _, test in enumerate(task_info.tests): + sub_suites.append(SubSuite([test], 0, tests_runtimes)) + return GeneratedSuite(sub_suites=sub_suites, build_variant=task_info.build_variant, + task_name=task_info.display_task_name, suite_name=task_info.suite) + + +class GenerateBurnInExecutor(BurnInExecutor): + """A burn-in executor that generates tasks.""" + + # pylint: disable=too-many-arguments + def __init__(self, generate_config: GenerateConfig, repeat_config: RepeatConfig, + evg_api: EvergreenApi, generate_tasks_file: Optional[str] = None, + history_end_date: Optional[datetime] = None) -> None: + """ + Create a new generate burn-in executor. + + :param generate_config: Configuration for how to generate tasks. + :param repeat_config: Configuration for how tests should be repeated. + :param evg_api: Evergreen API client. + :param generate_tasks_file: File to write generated task configuration to. + :param history_end_date: End date of range to query for historic test data. + """ + self.generate_config = generate_config + self.repeat_config = repeat_config + self.evg_api = evg_api + self.generate_tasks_file = generate_tasks_file + self.history_end_date = history_end_date if history_end_date else datetime.utcnow()\ + .replace(microsecond=0) + + def get_task_runtime_history(self, task: str) -> List[TestRuntime]: + """ + Query the runtime history of the specified task. + + :param task: Task to query. + :return: List of runtime histories for all tests in specified task. + """ + try: + project = self.generate_config.project + variant = self.generate_config.build_variant + end_date = self.history_end_date + start_date = end_date - timedelta(days=AVG_TEST_RUNTIME_ANALYSIS_DAYS) + test_stats = HistoricTaskData.from_evg(self.evg_api, project, start_date=start_date, + end_date=end_date, task=task, variant=variant) + return test_stats.get_tests_runtimes() + except requests.HTTPError as err: + if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE: + # Evergreen may return a 503 when the service is degraded. + # We fall back to returning no test history + return [] + else: + raise + + def _get_existing_tasks(self) -> Optional[Set[ExistingTask]]: + """Get any existing tasks that should be included in the generated display task.""" + if self.generate_config.include_gen_task: + return {ExistingTask(BURN_IN_TESTS_GEN_TASK)} + return None + + def generate_tasks_for_variant(self, tests_by_task: Dict[str, TaskInfo], variant: BuildVariant): + """Add tasks to the passed in variant.""" + tasks = set() + for task in sorted(tests_by_task): + task_info = tests_by_task[task] + task_history = self.get_task_runtime_history(task_info.display_task_name) + gen_suite = _tests_dict_to_generated_suites(task_info, task_history) + + task_generator = BurnInGenTaskService(self.generate_config, self.repeat_config, + task_history) + + params = BurnInGenTaskParams( + resmoke_args=task_info.resmoke_args, + require_multiversion_setup=task_info.require_multiversion_setup, + distro=task_info.distro, + ) + new_tasks = task_generator.generate_tasks(gen_suite, params) + tasks = tasks.union(new_tasks) + variant.display_task(BURN_IN_TESTS_TASK, tasks, + execution_existing_tasks=self._get_existing_tasks()) + + def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + """ + Execute the given tests in the given tasks. + + :param tests_by_task: Dictionary of tasks to run with tests to run in each. + """ + + build_variant = BuildVariant(self.generate_config.run_build_variant) + self.generate_tasks_for_variant(tests_by_task, build_variant) + + shrub_project = ShrubProject.empty() + shrub_project.add_build_variant(build_variant) + if not validate_task_generation_limit(shrub_project): + sys.exit(1) + + assert self.generate_tasks_file is not None + if self.generate_tasks_file: + write_file(self.generate_tasks_file, shrub_project.json()) + + +# pylint: disable=too-many-arguments +def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig, + repeat_config: RepeatConfig, evg_api: EvergreenApi, evg_conf: EvergreenProjectConfig, + repos: List[Repo], generate_tasks_file: str, install_dir: str) -> None: + """ + Run burn_in_tests. + + :param task_id: Id of task running. + :param build_variant: Build variant to run against. + :param generate_config: Configuration for how to generate tasks. + :param repeat_config: Configuration for how to repeat tests. + :param evg_api: Evergreen API client. + :param evg_conf: Evergreen project configuration. + :param repos: Git repos containing changes. + :param generate_tasks_file: File to write generate tasks configuration to. + :param install_dir: Path to bin directory of a testable installation + """ + change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ) + executor = GenerateBurnInExecutor(generate_config, repeat_config, evg_api, generate_tasks_file) + + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) + burn_in_orchestrator.burn_in(repos, build_variant, install_dir) + + +@click.command() +@click.option("--generate-tasks-file", "generate_tasks_file", default=None, metavar='FILE', + help="Run in 'generate.tasks' mode. Store task config to given file.") +@click.option("--build-variant", "build_variant", default=DEFAULT_VARIANT, metavar='BUILD_VARIANT', + help="Tasks to run will be selected from this build variant.") +@click.option("--run-build-variant", "run_build_variant", default=None, metavar='BUILD_VARIANT', + help="Burn in tasks will be generated on this build variant.") +@click.option("--distro", "distro", default=None, metavar='DISTRO', + help="The distro the tasks will execute on.") +@click.option("--project", "project", default=DEFAULT_PROJECT, metavar='PROJECT', + help="The evergreen project the tasks will execute on.") +@click.option("--repeat-tests", "repeat_tests_num", default=None, type=int, + help="Number of times to repeat tests.") +@click.option("--repeat-tests-min", "repeat_tests_min", default=None, type=int, + help="The minimum number of times to repeat tests if time option is specified.") +@click.option("--repeat-tests-max", "repeat_tests_max", default=None, type=int, + help="The maximum number of times to repeat tests if time option is specified.") +@click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS", + help="Repeat tests for the given time (in secs).") +@click.option("--evg-api-config", "evg_api_config", default=CONFIG_FILE, metavar="FILE", + help="Configuration file with connection info for Evergreen API.") +@click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") +@click.option("--task_id", "task_id", required=True, metavar='TASK_ID', + help="The evergreen task id.") +@click.option("--install-dir", "install_dir", required=True, + help="Path to bin directory of a testable installation.") +# pylint: disable=too-many-arguments,too-many-locals +def main(build_variant: str, run_build_variant: str, distro: str, project: str, + generate_tasks_file: str, repeat_tests_num: Optional[int], repeat_tests_min: Optional[int], + repeat_tests_max: Optional[int], repeat_tests_secs: Optional[int], evg_api_config: str, + verbose: bool, task_id: str, install_dir: str): + """ + Run new or changed tests in repeated mode to validate their stability. + + burn_in_tests detects jstests that are new or changed since the last git command and then + runs those tests in a loop to validate their reliability. + + The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can + either be repeated a specified number of times with the `--repeat-tests` option, or they can + be repeated for a certain time period with the `--repeat-tests-secs` option. + + Specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration + file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks + to do all the test executions. This is the mode used to run tests in patch builds. + + NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate + in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would + be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen + can handle. + \f + + :param build_variant: Build variant to query tasks from. + :param run_build_variant:Build variant to actually run against. + :param distro: Distro to run tests on. + :param project: Project to run tests on. + :param generate_tasks_file: Create a generate tasks configuration in this file. + :param repeat_tests_num: Repeat each test this number of times. + :param repeat_tests_min: Repeat each test at least this number of times. + :param repeat_tests_max: Once this number of repetitions has been reached, stop repeating. + :param repeat_tests_secs: Continue repeating tests for this number of seconds. + :param evg_api_config: Location of configuration file to connect to evergreen. + :param verbose: Log extra debug information. + :param task_id: Id of evergreen task being run in. + :param install_dir: path to bin directory of a testable installation + """ + _configure_logging(verbose) + + repeat_config = RepeatConfig(repeat_tests_secs=repeat_tests_secs, + repeat_tests_min=repeat_tests_min, + repeat_tests_max=repeat_tests_max, + repeat_tests_num=repeat_tests_num) # yapf: disable + + repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] + evg_conf = parse_evergreen_file(EVERGREEN_FILE) + evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) + + generate_config = GenerateConfig(build_variant=build_variant, + run_build_variant=run_build_variant, + distro=distro, + project=project, + task_id=task_id) # yapf: disable + generate_config.validate(evg_conf) + + burn_in(task_id, build_variant, generate_config, repeat_config, evg_api, evg_conf, repos, + generate_tasks_file, install_dir) + + +if __name__ == "__main__": + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py new file mode 100644 index 00000000000..86f8c046c5b --- /dev/null +++ b/buildscripts/evergreen_gen_build_variant.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +"""Generate configuration for a build variant.""" +import hashlib +import os +from concurrent.futures import ThreadPoolExecutor as Executor +from datetime import datetime, timedelta +from time import perf_counter +from typing import Any, Optional, Set, Tuple + +import click +import inject +import structlog +from pydantic import BaseModel +from evergreen import EvergreenApi, RetryingEvergreenApi +from evergreen import Task as EvgTask + +from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, Task, Variant, + parse_evergreen_file) +from buildscripts.task_generation.constants import (EXPANSION_RE, GEN_PARENT_TASK, + GENERATED_CONFIG_DIR, LOOKBACK_DURATION_DAYS, + MAX_TASK_PRIORITY, MAX_WORKERS) +from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder +from buildscripts.task_generation.gen_config import GenerationConfiguration +from buildscripts.task_generation.gen_task_validation import \ + GenTaskValidationService +from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService +from buildscripts.task_generation.suite_split import (SuiteSplitConfig, SuiteSplitParameters) +from buildscripts.task_generation.suite_split_strategies import ( + FallbackStrategy, SplitStrategy, greedy_division, round_robin_fallback) +from buildscripts.task_generation.task_types.fuzzer_tasks import \ + FuzzerGenTaskParams +from buildscripts.task_generation.task_types.gentask_options import \ + GenTaskOptions +from buildscripts.task_generation.task_types.resmoke_tasks import \ + ResmokeGenTaskParams +from buildscripts.util.cmdutils import enable_logging +from buildscripts.util.fileops import read_yaml_file +from buildscripts.util.taskname import remove_gen_suffix + +LOGGER = structlog.get_logger(__name__) + + +class EvgExpansions(BaseModel): + """ + Evergreen expansions needed to generate tasks. + + build_id: Build ID being run under. + build_variant: Build variant being generated. + exec_timeout_secs: Seconds to wait before considering a task timed out. + gen_task_gran: Granularity of how tasks are being generated. + is_patch: Whether generation is part of a patch build. + project: Evergreen project being run under. + max_test_per_suite: Maximum amount of tests to include in a suite. + max_sub_suites: Maximum number of sub-suites to generate per task. + mainline_max_sub_suites: Max number of sub-suites to generate per task on mainline builds. + resmoke_repeat_suites: Number of times suites should be repeated. + revision: Git revision being run against. + task_name: Name of task running. + target_resmoke_time: Target time of generated sub-suites. + task_id: ID of task being run under. + timeout_secs: Seconds to wait with no output before considering a task timed out. + """ + + build_id: str + build_variant: str + exec_timeout_secs: Optional[int] = None + is_patch: Optional[bool] + project: str + max_tests_per_suite: Optional[int] = 100 + max_sub_suites: Optional[int] = 5 + mainline_max_sub_suites: Optional[int] = 1 + resmoke_repeat_suites: Optional[int] = None + revision: str + task_name: str + target_resmoke_time: Optional[int] = None + task_id: str + timeout_secs: Optional[int] = None + + @classmethod + def from_yaml_file(cls, path: str) -> "EvgExpansions": + """ + Read the evergreen expansions from the given YAML file. + + :param path: Path to expansions YAML file. + :return: Expansions read from file. + """ + return cls(**read_yaml_file(path)) + + def get_max_sub_suites(self) -> int: + """Get the max_sub_suites to use.""" + if not self.is_patch: + return self.mainline_max_sub_suites + return self.max_sub_suites + + def build_suite_split_config(self, start_date: datetime, + end_date: datetime) -> SuiteSplitConfig: + """ + Get the configuration for splitting suites based on Evergreen expansions. + + :param start_date: Start date for historic stats lookup. + :param end_date: End date for historic stats lookup. + :return: Configuration to use for splitting suites. + """ + return SuiteSplitConfig( + evg_project=self.project, + target_resmoke_time=self.target_resmoke_time if self.target_resmoke_time else 60, + max_sub_suites=self.get_max_sub_suites(), + max_tests_per_suite=self.max_tests_per_suite, + start_date=start_date, + end_date=end_date, + ) + + def build_evg_config_gen_options(self) -> GenTaskOptions: + """ + Get the configuration for generating tasks from Evergreen expansions. + + :return: Configuration to use for splitting suites. + """ + return GenTaskOptions( + create_misc_suite=True, + is_patch=self.is_patch, + generated_config_dir=GENERATED_CONFIG_DIR, + use_default_timeouts=False, + timeout_secs=self.timeout_secs, + exec_timeout_secs=self.exec_timeout_secs, + ) + + def config_location(self) -> str: + """Location where generated configuration is stored.""" + generated_task_name = remove_gen_suffix(self.task_name) + file = f"{self.build_variant}/{self.revision}/generate_tasks/{generated_task_name}_gen-{self.build_id}" + # hash 'file' to shorten the file path dramatically + sha1 = hashlib.sha1() + sha1.update(file.encode('utf-8')) + return f"gtcl/{sha1.hexdigest()}.tgz" + + +def translate_run_var(run_var: str, build_variant: Variant) -> Any: + """ + Translate the given "run_var" into an actual value. + + Run_vars can contain evergreen expansions, in which case, the expansion (and possible default + value) need to be translated into a value we can use. + + :param run_var: Run var to translate. + :param build_variant: Build variant configuration. + :return: Value of run_var. + """ + match = EXPANSION_RE.search(run_var) + if match: + value = build_variant.expansion(match.group("id")) + if value is None: + value = match.group("default") + return value + return run_var + + +class GenerateBuildVariantOrchestrator: + """Orchestrator for generating tasks in a build variant.""" + + # pylint: disable=too-many-arguments + @inject.autoparams() + def __init__( + self, + gen_task_validation: GenTaskValidationService, + gen_task_options: GenTaskOptions, + evg_project_config: EvergreenProjectConfig, + evg_expansions: EvgExpansions, + evg_api: EvergreenApi, + ) -> None: + """ + Initialize the orchestrator. + + :param gen_task_validation: Service to validate task generation. + :param gen_task_options: Options for how tasks should be generated. + :param evg_project_config: Configuration for Evergreen Project. + :param evg_expansions: Evergreen expansions for running task. + :param evg_api: Evergreen API client. + """ + self.gen_task_validation = gen_task_validation + self.gen_task_options = gen_task_options + self.evg_project_config = evg_project_config + self.evg_expansions = evg_expansions + self.evg_api = evg_api + + def get_build_variant_expansion(self, build_variant_name: str, expansion: str) -> Any: + """ + Get the value of the given expansion for the specified build variant. + + :param build_variant_name: Build Variant to query. + :param expansion: Expansion to query. + :return: Value of given expansion. + """ + build_variant = self.evg_project_config.get_variant(build_variant_name) + return build_variant.expansion(expansion) + + def task_def_to_split_params(self, task_def: Task, + build_variant_gen: str) -> SuiteSplitParameters: + """ + Build parameters for how a task should be split based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant_gen: Name of Build Variant being generated. + :return: Parameters for how task should be split. + """ + build_variant = self.evg_project_config.get_variant(build_variant_gen) + task_name = remove_gen_suffix(task_def.name) + run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) + + suite_name = run_vars.get("suite", task_name) + return SuiteSplitParameters( + build_variant=build_variant_gen, + task_name=task_name, + suite_name=suite_name, + filename=suite_name, + is_asan=build_variant.is_asan_build(), + ) + + def task_def_to_gen_params(self, task_def: Task, build_variant: str) -> ResmokeGenTaskParams: + """ + Build parameters for how a task should be generated based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how task should be generated. + """ + run_func = task_def.generate_resmoke_tasks_command + run_vars = run_func.get("vars", {}) + + repeat_suites = 1 + if self.evg_expansions.resmoke_repeat_suites: + repeat_suites = self.evg_expansions.resmoke_repeat_suites + + return ResmokeGenTaskParams( + use_large_distro=run_vars.get("use_large_distro"), + require_multiversion_setup=task_def.require_multiversion_setup(), + require_multiversion_version_combo=task_def.require_multiversion_version_combo(), + repeat_suites=repeat_suites, + resmoke_args=run_vars.get("resmoke_args"), + resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), + large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), + config_location=self.evg_expansions.config_location(), + dependencies=self.determine_task_dependencies(task_def), + ) + + def determine_task_dependencies(self, task_def: Task) -> Set[str]: + """ + Determine the dependencies to use for tasks generated from the given task definition. + + This should include all tasks that the task definition depends on except for the currently + running task. + + :param task_def: Task definition to use. + :return: Set of dependencies to generate. + """ + dependency_set = { + task["name"] + for task in task_def.depends_on if task["name"] != self.evg_expansions.task_name + } + + return dependency_set + + def task_def_to_fuzzer_params(self, task_def: Task, build_variant: str) -> FuzzerGenTaskParams: + """ + Build parameters for how a fuzzer task should be generated based on its task definition. + + :param task_def: Task definition in evergreen project config. + :param build_variant: Name of Build Variant being generated. + :return: Parameters for how a fuzzer task should be generated. + """ + variant = self.evg_project_config.get_variant(build_variant) + run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) + run_vars = {k: translate_run_var(v, variant) for k, v in run_vars.items()} + + num_tasks = min(int(run_vars.get("num_tasks")), self.evg_expansions.get_max_sub_suites()) + + return FuzzerGenTaskParams( + task_name=remove_gen_suffix(task_def.name), + variant=build_variant, + suite=run_vars.get("suite"), + num_files=int(run_vars.get("num_files")), + num_tasks=num_tasks, + resmoke_args=run_vars.get("resmoke_args"), + npm_command=run_vars.get("npm_command", "jstestfuzz"), + jstestfuzz_vars=run_vars.get("jstestfuzz_vars", ""), + continue_on_failure=run_vars.get("continue_on_failure"), + resmoke_jobs_max=run_vars.get("resmoke_jobs_max"), + should_shuffle=run_vars.get("should_shuffle"), + timeout_secs=run_vars.get("timeout_secs"), + require_multiversion_setup=task_def.require_multiversion_setup(), + use_large_distro=run_vars.get("use_large_distro", False), + large_distro_name=self.get_build_variant_expansion(build_variant, "large_distro_name"), + config_location=self.evg_expansions.config_location(), + dependencies=self.determine_task_dependencies(task_def), + ) + + def generate(self, task_id: str, build_variant_name: str, output_file: str) -> None: + """ + Write task configuration for a build variant to disk. + + :param task_id: ID of running task. + :param build_variant_name: Name of build variant to generate. + :param output_file: Filename to write generated configuration to. + """ + if not self.gen_task_validation.should_task_be_generated(task_id): + LOGGER.info("Not generating configuration due to previous successful generation.") + return + + builder = EvgConfigBuilder() # pylint: disable=no-value-for-parameter + builder = self.generate_build_variant(builder, build_variant_name) + + generated_config = builder.build(output_file) + generated_config.write_all_to_dir(self.gen_task_options.generated_config_dir) + + with open('gtcl_update_expansions.yml', "w+") as fh: + fh.write(f"gtcl: {self.evg_expansions.config_location()}") + + # pylint: disable=too-many-locals + def generate_build_variant(self, builder: EvgConfigBuilder, + build_variant_name: str) -> EvgConfigBuilder: + """ + Generate task configuration for a build variant. + + :param builder: Evergreen configuration builder to use. + :param build_variant_name: Name of build variant to generate. + :return: Evergreen configuration builder with build variant configuration. + """ + LOGGER.info("Generating config", build_variant=build_variant_name) + start_time = perf_counter() + task_list = self.evg_project_config.get_variant(build_variant_name).task_names + tasks_to_hide = set() + with Executor(max_workers=MAX_WORKERS) as exe: + jobs = [] + for task_name in task_list: + task_def = self.evg_project_config.get_task(task_name) + if task_def.is_generate_resmoke_task: + tasks_to_hide.add(task_name) + + run_vars = task_def.generate_resmoke_tasks_command.get("vars", {}) + requires_npm = run_vars.get("is_jstestfuzz", False) + + if requires_npm: + fuzzer_params = self.task_def_to_fuzzer_params(task_def, build_variant_name) + jobs.append(exe.submit(builder.generate_fuzzer, fuzzer_params)) + else: + split_params = self.task_def_to_split_params(task_def, build_variant_name) + gen_params = self.task_def_to_gen_params(task_def, build_variant_name) + jobs.append(exe.submit(builder.generate_suite, split_params, gen_params)) + + [j.result() for j in jobs] # pylint: disable=expression-not-assigned + + end_time = perf_counter() + duration = end_time - start_time + + LOGGER.info("Finished BV", build_variant=build_variant_name, duration=duration, + task_count=len(tasks_to_hide)) + + builder.add_display_task(GEN_PARENT_TASK, tasks_to_hide, build_variant_name) + self.adjust_gen_tasks_priority(tasks_to_hide) + return builder + + def adjust_task_priority(self, task: EvgTask) -> None: + """ + Increase the priority of the given task by 1. + + :param task: Task to increase priority of. + """ + priority = min(task.priority + 1, MAX_TASK_PRIORITY) + LOGGER.info("Configure task", task_id=task.task_id, priority=priority) + self.evg_api.configure_task(task.task_id, priority=priority) + + @classmethod + def _should_adjust_task_priority(cls, task, gen_tasks): + if task.display_name in gen_tasks: + return True + # Test out the effect of Evergreen capacity constraints. + if task.build_variant.endswith("-query-patch-only"): + return True + return False + + def adjust_gen_tasks_priority(self, gen_tasks: Set[str]) -> int: + """ + Increase the priority of any "_gen" tasks. + + We want to minimize the time it tasks for the "_gen" tasks to activate the generated + sub-tasks. We will do that by increase the priority of the "_gen" tasks. + + :param gen_tasks: Set of "_gen" tasks that were found. + """ + build = self.evg_api.build_by_id(self.evg_expansions.build_id) + task_list = build.get_tasks() + + with Executor(max_workers=MAX_WORKERS) as exe: + jobs = [ + exe.submit(self.adjust_task_priority, task) for task in task_list + if self._should_adjust_task_priority(task, gen_tasks) + ] + + results = [j.result() for j in jobs] + return len(results) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +@click.option("--expansion-file", type=str, required=True, + help="Location of expansions file generated by evergreen.") +@click.option("--evg-api-config", type=str, required=True, + help="Location of evergreen api configuration.") +@click.option("--evg-project-config", type=str, default="etc/evergreen.yml", + help="Location of Evergreen project configuration.") +@click.option("--output-file", type=str, help="Name of output file to write.") +@click.option("--verbose", is_flag=True, default=False, help="Enable verbose logging.") +@click.argument('resmoke_run_args', nargs=-1, type=click.UNPROCESSED) +def main(expansion_file: str, evg_api_config: str, evg_project_config: str, output_file: str, + verbose: bool, resmoke_run_args: Tuple[str]) -> None: + """ + Generate task configuration for a build variant. + \f + :param expansion_file: Location of evergreen expansions for task. + :param evg_api_config: Location of file containing evergreen API authentication information. + :param evg_project_config: Location of file containing evergreen project configuration. + :param output_file: Location to write generated configuration to. + :param verbose: Should verbose logging be used. + :param resmoke_run_args: Args to forward to `resmoke.py run`. + """ + enable_logging(verbose) + + end_date = datetime.utcnow().replace(microsecond=0) + start_date = end_date - timedelta(days=LOOKBACK_DURATION_DAYS) + + evg_expansions = EvgExpansions.from_yaml_file(expansion_file) + + # pylint: disable=no-value-for-parameter + def dependencies(binder: inject.Binder) -> None: + binder.bind(EvgExpansions, evg_expansions) + binder.bind(SuiteSplitConfig, evg_expansions.build_suite_split_config(start_date, end_date)) + binder.bind(SplitStrategy, greedy_division) + binder.bind(FallbackStrategy, round_robin_fallback) + binder.bind(GenTaskOptions, evg_expansions.build_evg_config_gen_options()) + binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evg_api_config)) + binder.bind(EvergreenProjectConfig, parse_evergreen_file(evg_project_config)) + binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) + binder.bind(ResmokeProxyService, ResmokeProxyService(" ".join(resmoke_run_args))) + + inject.configure(dependencies) + + orchestrator = GenerateBuildVariantOrchestrator() # pylint: disable=no-value-for-parameter + start_time = perf_counter() + orchestrator.generate(evg_expansions.task_id, evg_expansions.build_variant, output_file) + end_time = perf_counter() + + LOGGER.info("Total runtime", duration=end_time - start_time) + + +if __name__ == '__main__': + main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/evergreen_gen_powercycle_tasks.py b/buildscripts/evergreen_gen_powercycle_tasks.py index 4be9115668e..9d949c7068a 100755 --- a/buildscripts/evergreen_gen_powercycle_tasks.py +++ b/buildscripts/evergreen_gen_powercycle_tasks.py @@ -27,7 +27,6 @@ Config = namedtuple("config", [ def make_config(expansions_file: Any) -> Config: """Group expansions into config.""" expansions = read_config_file(expansions_file) - private_key_file = "src/powercycle.pem" current_task_name = expansions.get("task_name", "powercycle") task_names = expansions.get("task_names", "powercycle_smoke_skip_compile") # Avoid duplicated task names @@ -38,12 +37,11 @@ def make_config(expansions_file: Any) -> Config: "timeout_secs": int(expansions.get("timeout_secs", 1800)), } remote_credentials_vars = { - "private_key_file": private_key_file, + "private_key_file": "src/powercycle.pem", "private_key_remote": "${__project_aws_ssh_key_value}", } set_up_ec2_instance_vars = { "set_up_retry_count": int(expansions.get("set_up_retry_count", 2)), - "private_key_file": private_key_file, } run_powercycle_vars = { "run_powercycle_args": expansions.get("run_powercycle_args"), diff --git a/buildscripts/evergreen_resmoke_job_count.py b/buildscripts/evergreen_resmoke_job_count.py index 4066c6f8f92..438648dea89 100644 --- a/buildscripts/evergreen_resmoke_job_count.py +++ b/buildscripts/evergreen_resmoke_job_count.py @@ -30,8 +30,8 @@ SYS_PLATFORM = sys.platform # Apply factor for a task based on the build variant it is running on. VARIANT_TASK_FACTOR_OVERRIDES = { - "enterprise-rhel-8-64-bit": [{"task": r"logical_session_cache_replication.*", "factor": 0.75}], - "enterprise-rhel-8-64-bit-inmem": [ + "enterprise-rhel-80-64-bit": [{"task": r"logical_session_cache_replication.*", "factor": 0.75}], + "enterprise-rhel-80-64-bit-inmem": [ {"task": "secondary_reads_passthrough", "factor": 0.3}, {"task": "multi_stmt_txn_jscore_passthrough_with_migration", "factor": 0.3}, ] diff --git a/buildscripts/evergreen_task_timeout.py b/buildscripts/evergreen_task_timeout.py index 7bee3a8e57b..5c0eabf7aef 100755 --- a/buildscripts/evergreen_task_timeout.py +++ b/buildscripts/evergreen_task_timeout.py @@ -7,7 +7,7 @@ import math import os import shlex import sys -from datetime import timedelta +from datetime import datetime, timedelta from pathlib import Path from typing import Dict, List, Optional @@ -19,7 +19,7 @@ from evergreen import EvergreenApi, RetryingEvergreenApi from buildscripts.ciconfig.evergreen import (EvergreenProjectConfig, parse_evergreen_file) from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService) +from buildscripts.timeouts.timeout_service import (TimeoutParams, TimeoutService, TimeoutSettings) from buildscripts.util.cmdutils import enable_logging from buildscripts.util.taskname import determine_task_base_name @@ -28,21 +28,20 @@ DEFAULT_TIMEOUT_OVERRIDES = "etc/evergreen_timeouts.yml" DEFAULT_EVERGREEN_CONFIG = "etc/evergreen.yml" DEFAULT_EVERGREEN_AUTH_CONFIG = "~/.evergreen.yml" COMMIT_QUEUE_ALIAS = "__commit_queue" +UNITTEST_TASK = "run_unittests" IGNORED_SUITES = { - "integration_tests_replset", - "integration_tests_replset_ssl_auth", - "integration_tests_sharded", - "integration_tests_standalone", - "integration_tests_standalone_audit", - "mongos_test", - "server_selection_json_test", - "sdam_json_test", + "integration_tests_replset", "integration_tests_replset_ssl_auth", "integration_tests_sharded", + "integration_tests_standalone", "integration_tests_standalone_audit", "mongos_test", + "server_selection_json_test" } HISTORY_LOOKBACK = timedelta(weeks=2) -COMMIT_QUEUE_TIMEOUT = timedelta(minutes=20) +COMMIT_QUEUE_TIMEOUT = timedelta(minutes=40) DEFAULT_REQUIRED_BUILD_TIMEOUT = timedelta(hours=1, minutes=20) DEFAULT_NON_REQUIRED_BUILD_TIMEOUT = timedelta(hours=2) +# 2x the longest "run tests" phase for unittests as of c9bf1dbc9cc46e497b2f12b2d6685ef7348b0726, +# which is 5 mins 47 secs, excluding outliers below +UNITTESTS_TIMEOUT = timedelta(minutes=12) class TimeoutOverride(BaseModel): @@ -144,6 +143,16 @@ class TimeoutOverrides(BaseModel): return None +def _is_required_build_variant(build_variant: str) -> bool: + """ + Determine if the given build variants is a required build variant. + + :param build_variant: Name of build variant to check. + :return: True if the given build variant is required. + """ + return build_variant.endswith("-required") + + def output_timeout(exec_timeout: timedelta, idle_timeout: Optional[timedelta], output_file: Optional[str]) -> None: """ @@ -213,7 +222,12 @@ class TaskTimeoutOrchestrator: LOGGER.info("Overriding configured timeout", exec_timeout_secs=override.total_seconds()) determined_timeout = override - elif self._is_required_build_variant( + elif task_name == UNITTEST_TASK and override is None: + LOGGER.info("Overriding unittest timeout", + exec_timeout_secs=UNITTESTS_TIMEOUT.total_seconds()) + determined_timeout = UNITTESTS_TIMEOUT + + elif _is_required_build_variant( variant) and determined_timeout > DEFAULT_REQUIRED_BUILD_TIMEOUT: LOGGER.info("Overriding required-builder timeout", exec_timeout_secs=DEFAULT_REQUIRED_BUILD_TIMEOUT.total_seconds()) @@ -259,12 +273,11 @@ class TaskTimeoutOrchestrator: return determined_timeout - def determine_historic_timeout(self, project: str, task: str, variant: str, suite_name: str, + def determine_historic_timeout(self, task: str, variant: str, suite_name: str, exec_timeout_factor: Optional[float]) -> TimeoutOverride: """ Calculate the timeout based on historic test results. - :param project: Name of project to query. :param task: Name of task to query. :param variant: Name of build variant to query. :param suite_name: Name of test suite being run. @@ -274,7 +287,7 @@ class TaskTimeoutOrchestrator: return TimeoutOverride(task=task, exec_timeout=None, idle_timeout=None) timeout_params = TimeoutParams( - evg_project=project, + evg_project="mongodb-mongo-master", build_variant=variant, task_name=task, suite_name=suite_name, @@ -301,20 +314,9 @@ class TaskTimeoutOrchestrator: bv = self.evg_project_config.get_variant(build_variant) return bv.is_asan_build() - def _is_required_build_variant(self, build_variant: str) -> bool: - """ - Determine if the given build variants is a required build variant. - - :param build_variant: Name of build variant to check. - :param evergreen_project_config: Evergreen config to query the variant name. - :return: True if the given build variant is required. - """ - bv = self.evg_project_config.get_variant(build_variant) - return "!" in bv.display_name - def determine_timeouts(self, cli_idle_timeout: Optional[timedelta], - cli_exec_timeout: Optional[timedelta], outfile: Optional[str], - project: str, task: str, variant: str, evg_alias: str, suite_name: str, + cli_exec_timeout: Optional[timedelta], outfile: Optional[str], task: str, + variant: str, evg_alias: str, suite_name: str, exec_timeout_factor: Optional[float]) -> None: """ Determine the timeouts to use for the given task and write timeouts to expansion file. @@ -322,14 +324,12 @@ class TaskTimeoutOrchestrator: :param cli_idle_timeout: Idle timeout specified by the CLI. :param cli_exec_timeout: Exec timeout specified by the CLI. :param outfile: File to write timeout expansions to. - :param project: Evergreen project task is being run on. - :param task: Name of task. :param variant: Build variant task is being run on. :param evg_alias: Evergreen alias that triggered task. :param suite_name: Name of evergreen suite being run. :param exec_timeout_factor: Scaling factor to use when determining timeout. """ - historic_timeout = self.determine_historic_timeout(project, task, variant, suite_name, + historic_timeout = self.determine_historic_timeout(task, variant, suite_name, exec_timeout_factor) idle_timeout = self.determine_idle_timeout(task, variant, cli_idle_timeout, @@ -351,8 +351,6 @@ def main(): help="Resmoke suite being run against.") parser.add_argument("--build-variant", dest="variant", required=True, help="Build variant task is being executed on.") - parser.add_argument("--project", dest="project", required=True, - help="Evergreen project task is being executed on.") parser.add_argument("--evg-alias", dest="evg_alias", required=True, help="Evergreen alias used to trigger build.") parser.add_argument("--timeout", dest="timeout", type=int, help="Timeout to use (in sec).") @@ -371,6 +369,9 @@ def main(): options = parser.parse_args() + end_date = datetime.now() + start_date = end_date - HISTORY_LOOKBACK + timeout_override = timedelta(seconds=options.timeout) if options.timeout else None exec_timeout_override = timedelta( seconds=options.exec_timeout) if options.exec_timeout else None @@ -380,12 +381,12 @@ def main(): os.path.expanduser(options.timeout_overrides_file)) enable_logging(verbose=False) - LOGGER.info("Determining timeouts", cli_args=options) def dependencies(binder: inject.Binder) -> None: binder.bind( EvergreenApi, RetryingEvergreenApi.get_api(config_file=os.path.expanduser(options.evg_api_config))) + binder.bind(TimeoutSettings, TimeoutSettings(start_date=start_date, end_date=end_date)) binder.bind(TimeoutOverrides, timeout_overrides) binder.bind(EvergreenProjectConfig, parse_evergreen_file(os.path.expanduser(options.evg_project_config))) @@ -397,8 +398,8 @@ def main(): task_timeout_orchestrator = inject.instance(TaskTimeoutOrchestrator) task_timeout_orchestrator.determine_timeouts( - timeout_override, exec_timeout_override, options.outfile, options.project, task_name, - options.variant, options.evg_alias, options.suite_name, options.exec_timeout_factor) + timeout_override, exec_timeout_override, options.outfile, task_name, options.variant, + options.evg_alias, options.suite_name, options.exec_timeout_factor) if __name__ == "__main__": diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py index ea3bf3c45d9..2e1106bf121 100644 --- a/buildscripts/gdb/mongo.py +++ b/buildscripts/gdb/mongo.py @@ -69,7 +69,7 @@ def get_current_thread_name(): fallback_name = '"%s"' % (gdb.selected_thread().name or '') try: # This goes through the pretty printer for StringData which adds "" around the name. - name = str(gdb.parse_and_eval("mongo::getThreadName()")) + name = str(gdb.parse_and_eval("mongo::ThreadName::getStaticString()")) if name == '""': return fallback_name return name diff --git a/buildscripts/gdb/mongo_printers.py b/buildscripts/gdb/mongo_printers.py index ae02a085b83..fe9727447ea 100644 --- a/buildscripts/gdb/mongo_printers.py +++ b/buildscripts/gdb/mongo_printers.py @@ -1,19 +1,12 @@ """GDB Pretty-printers for MongoDB.""" -import os import re import struct import sys import uuid -from pathlib import Path import gdb.printing -ROOT_PATH = str(Path(os.path.abspath(__file__)).parent.parent.parent) -if ROOT_PATH not in sys.path: - sys.path.insert(0, ROOT_PATH) -from src.third_party.immer.dist.tools.gdb_pretty_printers.printers import ListIter as ImmerListIter # pylint: disable=wrong-import-position - try: import bson import bson.json_util @@ -578,75 +571,6 @@ class AbslFlatHashMapPrinter(AbslHashMapPrinterBase): yield ('value', kvp['value']) -class ImmutableMapIter(ImmerListIter): - """Iterator for mongo::immutable::map.""" - - def __init__(self, val): - """Initialize iterator for mongo::immutable::map.""" - super().__init__(val) - self.max = (1 << 64) - 1 - self.pair = None - self.curr = (None, self.max, self.max) - - def __next__(self): - """Advance iterator for mongo::immutable::map and return current element.""" - if self.pair: - result = ('value', self.pair['second']) - self.pair = None - self.i += 1 - return result - if self.i == self.size: - raise StopIteration - if self.i < self.curr[1] or self.i >= self.curr[2]: - self.curr = self.region() - self.pair = self.curr[0][self.i - self.curr[1]].cast( - gdb.lookup_type(self.v.type.template_argument(0).name)) - result = ('key', self.pair['first']) - return result - - -class ImmutableMapPrinter: - """Pretty-printer for mongo::immutable::map.""" - - def __init__(self, val): - """Initialize pretty-printer for mongo::immutable::map.""" - self.val = val - - def to_string(self): - """Print top-level information about map.""" - return '%s of size %d' % (self.val.type, int(self.val['_storage']['impl_']['size'])) - - def children(self): - """Initialize iterator to actual elements of map.""" - return ImmutableMapIter(self.val['_storage']) - - @staticmethod - def display_hint(): - """Ensure map is displayed as map in gdb.""" - return 'map' - - -class ImmutableSetPrinter: - """Pretty-printer for mongo::immutable::set.""" - - def __init__(self, val): - """Initialize pretty-printer for mongo::immutable::set.""" - self.val = val - - def to_string(self): - """Print top-level information about set.""" - return '%s of size %d' % (self.val.type, int(self.val['_storage']['impl_']['size'])) - - def children(self): - """Initialize iterator to actual elements of set.""" - return ImmerListIter(self.val['_storage']) - - @staticmethod - def display_hint(): - """Ensure set is displayed as array in gdb.""" - return 'array' - - def find_match_brackets(search, opening='<', closing='>'): """Return the index of the closing bracket that matches the first opening bracket. @@ -957,8 +881,6 @@ def build_pretty_printer(): pp.add('__wt_txn', '__wt_txn', False, WtTxnPrinter) pp.add('__wt_update', '__wt_update', False, WtUpdateToBsonPrinter) pp.add('CodeFragment', 'mongo::sbe::vm::CodeFragment', False, SbeCodeFragmentPrinter) - pp.add('immutable::map', 'mongo::immutable::map', True, ImmutableMapPrinter) - pp.add('immutable::set', 'mongo::immutable::set', True, ImmutableSetPrinter) # TODO: enable with SERVER-62044. # Optimizer/ABT related pretty printers that can be used only with a running process. diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index c4ca320641f..2bf7e67727e 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -7,7 +7,9 @@ $ python generate_compile_expansions.py --out compile_expansions.yml """ import argparse +import json import os +import re import sys import shlex import yaml @@ -23,6 +25,7 @@ def generate_expansions(): """ args = parse_args() expansions = {} + expansions.update(generate_version_expansions()) expansions.update(generate_scons_cache_expansions()) with open(args.out, "w") as out: @@ -37,6 +40,39 @@ def parse_args(): return parser.parse_args() +def generate_version_expansions(): + """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" + expansions = {} + + if os.path.exists(VERSION_JSON): + with open(VERSION_JSON, "r") as fh: + data = fh.read() + version_data = json.loads(data) + version_line = version_data['version'] + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version.json") + else: + if not os.getenv("MONGO_VERSION"): + raise Exception("$MONGO_VERSION not set and no version.json provided") + version_line = os.getenv("MONGO_VERSION").lstrip("r") + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version from stdin and no version.json provided") + + if version_parts[0]: + expansions["suffix"] = "v6.0-latest" + expansions["src_suffix"] = "v6.0-latest" + expansions["is_release"] = "false" + else: + expansions["suffix"] = version_line + expansions["src_suffix"] = "r{0}".format(version_line) + expansions["is_release"] = "true" + expansions["version"] = version_line + + return expansions + + def generate_scons_cache_expansions(): """Generate scons cache expansions from some files and environment variables.""" expansions = {} @@ -65,5 +101,21 @@ def generate_scons_cache_expansions(): return expansions +def match_verstr(verstr): + """Match a version string and capture the "extra" part. + + If the version is a release like "2.3.4" or "2.3.4-rc0", this will return + None. If the version is a pre-release like "2.3.4-325-githash" or + "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version + begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, + r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. + doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. + """ + res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) + if not res: + return False + return res.groups() + + if __name__ == "__main__": generate_expansions() diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py index 5b6183fa6d9..1f9fea582dd 100755 --- a/buildscripts/generate_compile_expansions_shared_cache.py +++ b/buildscripts/generate_compile_expansions_shared_cache.py @@ -7,7 +7,9 @@ $ python generate_compile_expansions.py --out compile_expansions.yml """ import argparse +import json import os +import re import sys import shlex import yaml @@ -23,6 +25,7 @@ def generate_expansions(): """ args = parse_args() expansions = {} + expansions.update(generate_version_expansions()) expansions.update(generate_scons_cache_expansions()) with open(args.out, "w") as out: @@ -37,6 +40,39 @@ def parse_args(): return parser.parse_args() +def generate_version_expansions(): + """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" + expansions = {} + + if os.path.exists(VERSION_JSON): + with open(VERSION_JSON, "r") as fh: + data = fh.read() + version_data = json.loads(data) + version_line = version_data['version'] + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version.json") + else: + if not os.getenv("MONGO_VERSION"): + raise Exception("$MONGO_VERSION not set and no version.json provided") + version_line = os.getenv("MONGO_VERSION").lstrip("r") + version_parts = match_verstr(version_line) + if not version_parts: + raise ValueError("Unable to parse version from stdin and no version.json provided") + + if version_parts[0]: + expansions["suffix"] = "v6.0-latest" + expansions["src_suffix"] = "v6.0-latest" + expansions["is_release"] = "false" + else: + expansions["suffix"] = version_line + expansions["src_suffix"] = "r{0}".format(version_line) + expansions["is_release"] = "true" + expansions["version"] = version_line + + return expansions + + def generate_scons_cache_expansions(): """Generate scons cache expansions from some files and environment variables.""" expansions = {} @@ -61,7 +97,7 @@ def generate_scons_cache_expansions(): if sys.platform.startswith("win"): shared_mount_root = 'X:\\' else: - shared_mount_root = '/efs/scons' + shared_mount_root = '/efs' default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache") expansions["scons_cache_path"] = default_cache_path expansions[ @@ -89,5 +125,21 @@ def generate_scons_cache_expansions(): return expansions +def match_verstr(verstr): + """Match a version string and capture the "extra" part. + + If the version is a release like "2.3.4" or "2.3.4-rc0", this will return + None. If the version is a pre-release like "2.3.4-325-githash" or + "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version + begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, + r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. + doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. + """ + res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) + if not res: + return False + return res.groups() + + if __name__ == "__main__": generate_expansions() diff --git a/buildscripts/generate_version_expansions.py b/buildscripts/generate_version_expansions.py deleted file mode 100755 index 4b0f9ccc0a8..00000000000 --- a/buildscripts/generate_version_expansions.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate the version expansions file used by Evergreen as part of the push/release process. - -Invoke by specifying an output file. -$ python generate_build_expansions.py --out version_expansions.yml -""" - -import argparse -import json -import os -import re -import sys -import yaml - -VERSION_JSON = "version.json" - - -def generate_expansions(): - """Entry point for the script. - - This calls functions to generate version and scons cache expansions and - writes them to a file. - """ - args = parse_args() - expansions = {} - expansions.update(generate_version_expansions()) - - with open(args.out, "w") as out: - print("saving compile expansions to {0}: ({1})".format(args.out, expansions)) - yaml.safe_dump(expansions, out, default_flow_style=False) - - -def parse_args(): - """Parse program arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument("--out", required=True) - return parser.parse_args() - - -def generate_version_expansions(): - """Generate expansions from a version.json file if given, or $MONGO_VERSION.""" - expansions = {} - - if os.path.exists(VERSION_JSON): - with open(VERSION_JSON, "r") as fh: - data = fh.read() - version_data = json.loads(data) - version_line = version_data['version'] - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version.json") - else: - version_line = os.getenv("MONGO_VERSION") - if not version_line: - raise Exception("$MONGO_VERSION not set and no version.json provided") - - version_line = version_line.lstrip("r") - version_parts = match_verstr(version_line) - if not version_parts: - raise ValueError("Unable to parse version from stdin and no version.json provided") - - if version_parts[0]: - expansions["suffix"] = "v6.0-latest" - expansions["src_suffix"] = "v6.0-latest" - expansions["is_release"] = "false" - else: - expansions["suffix"] = version_line - expansions["src_suffix"] = "r{0}".format(version_line) - expansions["is_release"] = "true" - expansions["version"] = version_line - - return expansions - - -def match_verstr(verstr): - """Match a version string and capture the "extra" part. - - If the version is a release like "2.3.4" or "2.3.4-rc0", this will return - None. If the version is a pre-release like "2.3.4-325-githash" or - "2.3.4-pre-", this will return "-pre-" or "-325-githash" If the version - begins with the letter 'r', it will also match, e.g. r2.3.4, r2.3.4-rc0, - r2.3.4-git234, r2.3.4-rc0-234-githash If the version is invalid (i.e. - doesn't start with "2.3.4" or "2.3.4-rc0", this will return False. - """ - res = re.match(r'^r?(?:\d+\.\d+\.\d+(?:-rc\d+|-alpha\d+)?)(-.*)?', verstr) - if not res: - return False - return res.groups() - - -if __name__ == "__main__": - generate_expansions() diff --git a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py index 2a9d3c72420..c18077ace76 100644 --- a/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py +++ b/buildscripts/idl/check_stable_api_commands_have_idl_definitions.py @@ -96,16 +96,16 @@ def list_commands_for_api(api_version: str, mongod_or_mongos: str, install_dir: if mongod_or_mongos == "mongod": logger = loggers.new_fixture_logger("MongoDFixture", 0) logger.parent = LOGGER - fixture: interface.Fixture = fixturelib.make_fixture( - "MongoDFixture", logger, 0, dbpath_prefix=dbpath.name, - mongod_executable=mongod_executable, mongod_options={"set_parameters": {}}) + fixture: interface.Fixture = fixturelib.make_fixture("MongoDFixture", logger, 0, + dbpath_prefix=dbpath.name, + mongod_executable=mongod_executable) else: logger = loggers.new_fixture_logger("ShardedClusterFixture", 0) logger.parent = LOGGER - fixture = fixturelib.make_fixture( - "ShardedClusterFixture", logger, 0, dbpath_prefix=dbpath.name, - mongos_executable=mongos_executable, mongod_executable=mongod_executable, - mongod_options={"set_parameters": {}}) + fixture = fixturelib.make_fixture("ShardedClusterFixture", logger, 0, + dbpath_prefix=dbpath.name, + mongos_executable=mongos_executable, + mongod_executable=mongod_executable, mongod_options={}) fixture.setup() fixture.await_ready() diff --git a/buildscripts/idl/gen_all_feature_flag_list.py b/buildscripts/idl/gen_all_feature_flag_list.py index 88fee6b367f..518583898cb 100644 --- a/buildscripts/idl/gen_all_feature_flag_list.py +++ b/buildscripts/idl/gen_all_feature_flag_list.py @@ -30,6 +30,7 @@ Generate a file containing a list of disabled feature flags. Used by resmoke.py to run only feature flag tests. """ +import argparse import os import sys @@ -42,49 +43,47 @@ sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../ # pylint: disable=wrong-import-position import buildscripts.idl.lib as lib -from buildscripts.idl.idl import parser -def gen_all_feature_flags(idl_dirs: List[str] = None): - """Generate a list of all feature flags.""" - default_idl_dirs = ["src", "buildscripts"] +def is_third_party_idl(idl_path: str) -> bool: + """Check if an IDL file is under a third party directory.""" + third_party_idl_subpaths = [os.path.join("third_party", "mozjs"), "win32com"] - if not idl_dirs: - idl_dirs = default_idl_dirs + for file_name in third_party_idl_subpaths: + if file_name in idl_path: + return True - all_flags = [] - for idl_dir in idl_dirs: - for idl_path in sorted(lib.list_idls(idl_dir)): - if lib.is_third_party_idl(idl_path): - continue - # Most IDL files do not contain feature flags. - # We can discard these quickly without expensive YAML parsing. - with open(idl_path) as idl_file: - if 'feature_flags' not in idl_file.read(): - continue - with open(idl_path) as idl_file: - doc = parser.parse_file(idl_file, idl_path) - for feature_flag in doc.spec.feature_flags: - if feature_flag.default.literal != "true": - all_flags.append(feature_flag.name) - - with open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml") as fully_disabled_ffs: - force_disabled_flags = yaml.safe_load(fully_disabled_ffs) + return False - return list(set(all_flags) - set(force_disabled_flags)) +def gen_all_feature_flags(idl_dir: str, import_dirs: List[str]): + """Generate a list of all feature flags.""" + all_flags = [] + for idl_path in sorted(lib.list_idls(idl_dir)): + if is_third_party_idl(idl_path): + continue + for feature_flag in lib.parse_idl(idl_path, import_dirs).spec.feature_flags: + if feature_flag.default.literal != "true": + all_flags.append(feature_flag.name) -def gen_all_feature_flags_file(filename: str = lib.ALL_FEATURE_FLAG_FILE): - """Output generated list of feature flags to specified file.""" - flags = gen_all_feature_flags() - with open(filename, "w") as output_file: - output_file.write("\n".join(flags)) - print("Generated: ", os.path.realpath(output_file.name)) + force_disabled_flags = yaml.safe_load( + open("buildscripts/resmokeconfig/fully_disabled_feature_flags.yml")) + + return list(set(all_flags) - set(force_disabled_flags)) def main(): """Run the main function.""" - gen_all_feature_flags_file() + arg_parser = argparse.ArgumentParser(description=__doc__) + arg_parser.add_argument("--import-dir", dest="import_dirs", type=str, action="append", + help="Directory to search for IDL import files") + + args = arg_parser.parse_args() + + flags = gen_all_feature_flags(os.getcwd(), args.import_dirs) + with open(lib.ALL_FEATURE_FLAG_FILE, "w") as output_file: + for flag in flags: + output_file.write("%s\n" % flag) if __name__ == '__main__': diff --git a/buildscripts/idl/gen_all_server_params_list.py b/buildscripts/idl/gen_all_server_params_list.py deleted file mode 100644 index 75dd5db45ff..00000000000 --- a/buildscripts/idl/gen_all_server_params_list.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (C) 2023-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# <http://www.mongodb.com/licensing/server-side-public-license>. -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. -""" -Generate a file containing a list of all available server parameters. - -Used by DSI to conditionally allow configuration of internalQueryStatsRateLimit parameter. -""" - -import os -import sys - -from typing import List - -# Permit imports from "buildscripts". -sys.path.append(os.path.normpath(os.path.join(os.path.abspath(__file__), '../../..'))) - -# pylint: disable=wrong-import-position -from buildscripts.idl import lib -from buildscripts.idl.idl import parser - - -def gen_all_server_params(idl_dirs: List[str] = None): - """Generate a list of all server parameters.""" - default_idl_dirs = ["src"] - - if not idl_dirs: - idl_dirs = default_idl_dirs - - all_params = [] - for idl_dir in idl_dirs: - for idl_path in sorted(lib.list_idls(idl_dir)): - if lib.is_third_party_idl(idl_path): - continue - # Most IDL files do not contain server parameters. - # We can discard these quickly without expensive YAML parsing. - with open(idl_path) as idl_file: - if 'server_parameters' not in idl_file.read(): - continue - with open(idl_path) as idl_file: - doc = parser.parse_file(idl_file, idl_path) - for server_param in doc.spec.server_parameters: - all_params.append(server_param.name) - - return all_params - - -def gen_all_server_params_file(filename: str = "all_server_params.txt"): - """Generate a file containing a list of all server parameters.""" - flags = gen_all_server_params() - with open(filename, "w") as output_file: - output_file.write("\n".join(flags)) - print("Generated: ", os.path.realpath(output_file.name)) - - -def main(): - """Run the main function.""" - gen_all_server_params_file() - - -if __name__ == '__main__': - main() diff --git a/buildscripts/idl/idl/ast.py b/buildscripts/idl/idl/ast.py index c81a8d26c0a..d86ccbbb257 100644 --- a/buildscripts/idl/idl/ast.py +++ b/buildscripts/idl/idl/ast.py @@ -34,7 +34,6 @@ This is a lossy translation from the IDL Syntax tree as the IDL AST only contain the enums and structs that need code generated for them, and just enough information to do that. """ from abc import ABCMeta, abstractmethod -import enum from typing import Any, Dict, List, Optional from . import common, errors @@ -114,9 +113,6 @@ class Type(common.SourceLocation): # A variant can have at most one alternative type which is a struct. Otherwise, if we saw # a sub-object while parsing BSON, we wouldn't know which struct to interpret it as. self.variant_struct_type = None # type: Type - # Marks whether this type is a query shape component. - # Can only be true if is_struct is true. - self.is_query_shape_component = False # type: bool super(Type, self).__init__(file_name, line, column) @@ -148,9 +144,6 @@ class Struct(common.SourceLocation): self.allow_global_collection_name = False # type: bool self.non_const_getter = False # type: bool self.cpp_validator_func = None # type: str - - # Determines whether or not this IDL struct can be a component of a query shape. See WRITING-13831. - self.query_shape_component = False # type: bool super(Struct, self).__init__(file_name, line, column) @@ -191,34 +184,6 @@ class Validator(common.SourceLocation): super(Validator, self).__init__(file_name, line, column) -@enum.unique -class QueryShapeFieldType(enum.Enum): - """Enum describing how to treat a field in the context of query shape computation.""" - - # Abstract literal from shape. - LITERAL = enum.auto() - # Leave value as-is in shape. - PARAMETER = enum.auto() - # Anonymize string value. - ANONYMIZE = enum.auto() - # IDL type uses custom serializer -- defer to that serializer. - CUSTOM = enum.auto() - - @classmethod - def bind(cls, string_value): - # type: (Optional[str]) -> Optional[QueryShapeFieldType] - """Parses the string to the enum type.""" - if string_value is None: - return None - bindings = { - "literal": cls.LITERAL, - "parameter": cls.PARAMETER, - "anonymize": cls.ANONYMIZE, - "custom": cls.CUSTOM, - } - return bindings.get(string_value, None) - - class Field(common.SourceLocation): """ An instance of a field in a struct. @@ -258,26 +223,8 @@ class Field(common.SourceLocation): # Validation rules. self.validator = None # type: Optional[Validator] - # Determines whether or not this field represents a literal value that should be abstracted when serializing a query shape. - # See WRITING-13831 for details on query shape. - self.query_shape = None # type: Optional[QueryShapeFieldType] - super(Field, self).__init__(file_name, line, column) - @property - def should_serialize_with_options(self): - # type: () -> bool - """Returns true if the IDL compiler should add a call to serialization options for this field.""" - return self.query_shape is not None and self.query_shape in [ - QueryShapeFieldType.LITERAL, QueryShapeFieldType.ANONYMIZE - ] - - @property - def should_shapify(self): - # type: () -> bool - """Returns true if the IDL compiler should treat this field as a query literal.""" - return self.query_shape is not None and self.query_shape != QueryShapeFieldType.PARAMETER - class Privilege(common.SourceLocation): """IDL privilege information.""" diff --git a/buildscripts/idl/idl/binder.py b/buildscripts/idl/idl/binder.py index 853088cede6..1511b6c1b07 100644 --- a/buildscripts/idl/idl/binder.py +++ b/buildscripts/idl/idl/binder.py @@ -271,7 +271,6 @@ def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): ast_struct.qualified_cpp_name = _get_struct_qualified_cpp_name(struct) ast_struct.allow_global_collection_name = struct.allow_global_collection_name ast_struct.non_const_getter = struct.non_const_getter - ast_struct.query_shape_component = struct.query_shape_component # Validate naming restrictions if ast_struct.name.startswith("array<"): @@ -308,20 +307,6 @@ def _bind_struct_common(ctxt, parsed_spec, struct, ast_struct): if not _is_duplicate_field(ctxt, ast_struct.name, ast_struct.fields, ast_field): ast_struct.fields.append(ast_field) - # Verify that each field on the struct defines a query shape type on the field if and only if - # query_shape_component is defined on the struct. - if not field.hidden and struct.query_shape_component and ast_field.query_shape is None: - ctxt.add_must_declare_shape_type(ast_field, ast_struct.name, ast_field.name) - - if not struct.query_shape_component and ast_field.query_shape is not None: - ctxt.add_must_be_query_shape_component(ast_field, ast_struct.name, ast_field.name) - - if ast_field.query_shape == ast.QueryShapeFieldType.ANONYMIZE and not ( - ast_field.type.cpp_type in ["std::string", "std::vector<std::string>"] - or 'string' in ast_field.type.bson_serialization_type): - ctxt.add_query_shape_anonymize_must_be_string(ast_field, ast_field.name, - ast_field.type.cpp_type) - # Fill out the field comparison_order property as needed if ast_struct.generate_comparison_operators and ast_struct.fields: # If the user did not specify an ordering of fields, then number all fields in @@ -435,7 +420,6 @@ def _bind_struct_type(struct): ast_type.name = struct.name ast_type.cpp_type = _get_struct_qualified_cpp_name(struct) ast_type.bson_serialization_type = ["object"] - ast_type.is_query_shape_component = struct.query_shape_component return ast_type @@ -985,7 +969,6 @@ def _bind_type(idltype): ast_type.bindata_subtype = idltype.bindata_subtype ast_type.serializer = _normalize_method_name(idltype.cpp_type, idltype.serializer) ast_type.deserializer = _normalize_method_name(idltype.cpp_type, idltype.deserializer) - ast_type.is_query_shape_component = True return ast_type @@ -1010,11 +993,6 @@ def _bind_field(ctxt, parsed_spec, field): ast_field.unstable = field.unstable ast_field.always_serialize = field.always_serialize - if field.query_shape is not None: - ast_field.query_shape = ast.QueryShapeFieldType.bind(field.query_shape) - if ast_field.query_shape is None: - ctxt.add_invalid_query_shape_value(ast_field, field.query_shape) - ast_field.cpp_name = field.name if field.cpp_name: ast_field.cpp_name = field.cpp_name @@ -1095,8 +1073,6 @@ def _bind_field(ctxt, parsed_spec, field): if ast_field.validator is None: return None - if ast_field.should_shapify and not ast_field.type.is_query_shape_component: - ctxt.add_must_be_query_shape_component(ast_field, ast_field.type.name, ast_field.name) return ast_field diff --git a/buildscripts/idl/idl/bson.py b/buildscripts/idl/idl/bson.py index c7ead4cd66e..8216b5d743d 100644 --- a/buildscripts/idl/idl/bson.py +++ b/buildscripts/idl/idl/bson.py @@ -73,7 +73,6 @@ _BINDATA_SUBTYPE = { "uuid": {'scalar': True, 'bindata_enum': 'newUUID'}, "md5": {'scalar': True, 'bindata_enum': 'MD5Type'}, "encrypt": {'scalar': True, 'bindata_enum': 'Encrypt'}, - "sensitive": {'scalar': True, 'bindata_enum': 'Sensitive'}, } diff --git a/buildscripts/idl/idl/cpp_types.py b/buildscripts/idl/idl/cpp_types.py index ba3a0b6bce6..d8e37dfcc56 100644 --- a/buildscripts/idl/idl/cpp_types.py +++ b/buildscripts/idl/idl/cpp_types.py @@ -577,14 +577,14 @@ class BsonCppTypeBase(object, metaclass=ABCMeta): pass @abstractmethod - def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): - # type: (writer.IndentedTextWriter, str, bool) -> str + def gen_serializer_expression(self, indented_writer, expression): + # type: (writer.IndentedTextWriter, str) -> str """Generate code with the text writer and return an expression to serialize the type.""" pass -def _call_method_or_global_function(expression, method_name, should_shapify=False): - # type: (str, str, bool) -> str +def _call_method_or_global_function(expression, method_name): + # type: (str, str) -> str """ Given a fully-qualified method name, call it correctly. @@ -592,19 +592,13 @@ def _call_method_or_global_function(expression, method_name, should_shapify=Fals not treated as a global C++ function though. This notion of functions is designed to support enum deserializers/serializers which are not methods. """ - shape_options = '' - if should_shapify: - shape_options = 'options' - short_method_name = writer.get_method_name(method_name) if writer.is_function(method_name): - return common.template_args('${method_name}(${expression}${shape_options})', - expression=expression, method_name=method_name, - shape_options=shape_options) + return common.template_args('${method_name}(${expression})', expression=expression, + method_name=method_name) - return common.template_args('${expression}.${method_name}(${shape_options})', - expression=expression, method_name=short_method_name, - shape_options=shape_options) + return common.template_args('${expression}.${method_name}()', expression=expression, + method_name=short_method_name) class _CommonBsonCppTypeBase(BsonCppTypeBase): @@ -625,10 +619,9 @@ class _CommonBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): - # type: (writer.IndentedTextWriter, str, bool) -> str - return _call_method_or_global_function(expression, self._ast_type.serializer, - should_shapify) + def gen_serializer_expression(self, indented_writer, expression): + # type: (writer.IndentedTextWriter, str) -> str + return _call_method_or_global_function(expression, self._ast_type.serializer) class _ObjectBsonCppTypeBase(BsonCppTypeBase): @@ -650,19 +643,12 @@ class _ObjectBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): - # type: (writer.IndentedTextWriter, str, bool) -> str + def gen_serializer_expression(self, indented_writer, expression): + # type: (writer.IndentedTextWriter, str) -> str method_name = writer.get_method_name(self._ast_type.serializer) - function_arguments = [] - # Provide options if custom shapification required. - if should_shapify: - function_arguments.append('options') - indented_writer.write_line( - common.template_args( - 'const BSONObj localObject = ${expression}.${method_name}(${function_arguments});', - expression=expression, method_name=method_name, - function_arguments=', '.join(function_arguments))) + common.template_args('const BSONObj localObject = ${expression}.${method_name}();', + expression=expression, method_name=method_name)) return "localObject" @@ -685,8 +671,8 @@ class _ArrayBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return self._ast_type.serializer is not None - def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): - # type: (writer.IndentedTextWriter, str, bool) -> str + def gen_serializer_expression(self, indented_writer, expression): + # type: (writer.IndentedTextWriter, str) -> str method_name = writer.get_method_name(self._ast_type.serializer) indented_writer.write_line( common.template_args('BSONArray localArray(${expression}.${method_name}());', @@ -709,8 +695,8 @@ class _BinDataBsonCppTypeBase(BsonCppTypeBase): # type: () -> bool return True - def gen_serializer_expression(self, indented_writer, expression, should_shapify=False): - # type: (writer.IndentedTextWriter, str, bool) -> str + def gen_serializer_expression(self, indented_writer, expression): + # type: (writer.IndentedTextWriter, str) -> str if self._ast_type.serializer: method_name = writer.get_method_name(self._ast_type.serializer) indented_writer.write_line( diff --git a/buildscripts/idl/idl/errors.py b/buildscripts/idl/idl/errors.py index 0f030cf6573..47cd8ecf3f4 100644 --- a/buildscripts/idl/idl/errors.py +++ b/buildscripts/idl/idl/errors.py @@ -128,12 +128,6 @@ ERROR_ID_DUPLICATE_ACCESS_CHECK = "ID0087" ERROR_ID_DUPLICATE_PRIVILEGE = "ID0088" ERROR_ID_EMPTY_ACCESS_CHECK = "ID0089" ERROR_ID_MISSING_ACCESS_CHECK = "ID0090" -ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL = "ID0094" -ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL = "ID0095" -ERROR_ID_INVALID_TYPE_FOR_SHAPIFY = "ID0096" -ERROR_ID_QUERY_SHAPE_PROPERTIES_MUTUALLY_EXCLUSIVE = "ID0097" -ERROR_ID_QUERY_SHAPE_PROPERTY_CANNOT_BE_FALSE = "ID0098" -ERROR_ID_QUERY_SHAPE_INVALID_VALUE = "ID0102" class IDLError(Exception): @@ -375,15 +369,6 @@ class ParserContext(object): return True return False - def get_required_bool(self, node): - # type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> bool - """Get a YAML bool and enforce it is specified.""" - boolean_value = yaml.safe_load(node.value) - if not isinstance(boolean_value, bool): - self._add_node_error(node, ERROR_ID_IS_NODE_VALID_BOOL, - "Illegal bool value, expected either 'true' or 'false'.") - return boolean_value - def get_list(self, node): # type: (Union[yaml.nodes.MappingNode, yaml.nodes.ScalarNode, yaml.nodes.SequenceNode]) -> List[str] """Get a YAML scalar or sequence node as a list of strings.""" @@ -982,34 +967,6 @@ class ParserContext(object): self._add_error(location, ERROR_ID_MISSING_ACCESS_CHECK, 'Command "%s" has api_version != "" but is missing access_check.' % (name)) - def add_must_declare_shape_type(self, location, struct_name, field_name): - # type: (common.SourceLocation, str, str) -> None - """Add an error about a field not specifying either query_shape_literal or query_shape_anonymize if the struct is query_shape_component.""" - self._add_error( - location, ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL, - f"Field '{field_name}' must specify either 'query_shape_literal' or 'query_shape_anonymize' since struct '{struct_name}' is a query shape component." - ) - - def add_must_be_query_shape_component(self, location, struct_name, field_name): - # type: (common.SourceLocation, str, str) -> None - """Add an error about specifying 'query_shape_literal' without being a shape component.""" - self._add_error( - location, ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL, - f"Field '{field_name}' cannot specify 'query_shape_literal' property since struct '{struct_name}' is not a query shape component." - ) - - def add_query_shape_anonymize_must_be_string(self, location, field_name, field_type): - """Add an error about field paths needing to be strings.""" - self._add_error( - location, ERROR_ID_INVALID_TYPE_FOR_SHAPIFY, - f"In order for {field_name} to be marked as a query shape fieldpath, it must have a string type, not {field_type}." - ) - - def add_invalid_query_shape_value(self, location, query_shape_value): - """Assert that the query shape value is one of the accepted values.""" - self._add_error(location, ERROR_ID_QUERY_SHAPE_INVALID_VALUE, - f"'{query_shape_value}' is not a valid value for 'query_shape'.") - def _assert_unique_error_messages(): # type: () -> None diff --git a/buildscripts/idl/idl/generator.py b/buildscripts/idl/idl/generator.py index d3c62204623..d8b38d51a0c 100644 --- a/buildscripts/idl/idl/generator.py +++ b/buildscripts/idl/idl/generator.py @@ -1032,10 +1032,6 @@ class _CppHeaderFileWriter(_CppFileWriterBase): if any(command.api_version for command in spec.commands): header_list.append('mongo/db/commands.h') - # Include serialization options only if there is a struct which is part of a query shape. - if any(struct.query_shape_component for struct in spec.structs): - header_list.append('mongo/db/query/query_shape/serialization_options.h') - header_list.sort() for include in header_list: @@ -1879,23 +1875,6 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._gen_command_deserializer(struct, "request.body") - def _gen_single_field_serialize_expression(self, template_params, field, bson_cpp_type): - """Helper to append the code to serialize a single field, as part of a custom type.""" - expression = bson_cpp_type.gen_serializer_expression( - self._writer, _access_member(field), - field.query_shape == ast.QueryShapeFieldType.CUSTOM) - template_params['expression'] = expression - if not field.should_serialize_with_options: - self._writer.write_template('builder->append(${field_name}, ${expression});') - elif field.query_shape == ast.QueryShapeFieldType.LITERAL: - self._writer.write_template( - 'options.serializeLiteral(${expression}).serializeForIDL(${field_name}, builder);') - else: - assert field.query_shape == ast.QueryShapeFieldType.ANONYMIZE - self._writer.write_template( - 'builder->append(${field_name}, options.serializeFieldPathFromString(${expression}));' - ) - def _gen_serializer_method_custom(self, field): # type: (ast.Field) -> None """Generate the serialize method definition for a custom type.""" @@ -1916,14 +1895,14 @@ class _CppSourceFileWriter(_CppFileWriterBase): self._writer.write_template( 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') with self._block('for (const auto& item : ${access_member}) {', '}'): - expression = bson_cpp_type.gen_serializer_expression( - self._writer, 'item', - field.query_shape == ast.QueryShapeFieldType.CUSTOM) + expression = bson_cpp_type.gen_serializer_expression(self._writer, 'item') template_params['expression'] = expression self._writer.write_template('arrayBuilder.append(${expression});') else: - self._gen_single_field_serialize_expression(template_params, field, - bson_cpp_type) + expression = bson_cpp_type.gen_serializer_expression( + self._writer, _access_member(field)) + template_params['expression'] = expression + self._writer.write_template('builder->append(${field_name}, ${expression});') elif field.type.bson_serialization_type[0] == 'any': # Any types are special @@ -1939,17 +1918,14 @@ class _CppSourceFileWriter(_CppFileWriterBase): # Call a method like class::method(BSONArrayBuilder*) self._writer.write_template('item.${method_name}(&arrayBuilder);') else: - template_params[ - 'query_shape_options'] = ', options' if field.query_shape == ast.QueryShapeFieldType.CUSTOM else '' if writer.is_function(field.type.serializer): + # Call a method like method(value, StringData, BSONObjBuilder*) self._writer.write_template( - '${method_name}(${access_member}, ${field_name}, builder${query_shape_options});' - ) + '${method_name}(${access_member}, ${field_name}, builder);') else: - # Call a method like class::method(StringData, BSONObjBuilder*, SerializationOptions) + # Call a method like class::method(StringData, BSONObjBuilder*) self._writer.write_template( - '${access_member}.${method_name}(${field_name}, builder${query_shape_options});' - ) + '${access_member}.${method_name}(${field_name}, builder);') else: method_name = writer.get_method_name(field.type.serializer) @@ -1984,29 +1960,18 @@ class _CppSourceFileWriter(_CppFileWriterBase): if field.chained: # Just directly call the serializer for chained structs without opening up a nested # document. - if not field.should_serialize_with_options: - self._writer.write_template('${access_member}.serialize(builder);') - else: - self._writer.write_template('${access_member}.serialize(builder, options);') - + self._writer.write_template('${access_member}.serialize(builder);') elif field.type.is_array: self._writer.write_template( 'BSONArrayBuilder arrayBuilder(builder->subarrayStart(${field_name}));') with self._block('for (const auto& item : ${access_member}) {', '}'): self._writer.write_line( 'BSONObjBuilder subObjBuilder(arrayBuilder.subobjStart());') - if not field.should_serialize_with_options: - self._writer.write_line('item.serialize(&subObjBuilder);') - else: - self._writer.write_line('item.serialize(&subObjBuilder, options);') + self._writer.write_line('item.serialize(&subObjBuilder);') else: self._writer.write_template( 'BSONObjBuilder subObjBuilder(builder->subobjStart(${field_name}));') - if not field.should_serialize_with_options: - self._writer.write_template('${access_member}.serialize(&subObjBuilder);') - else: - self._writer.write_template( - '${access_member}.serialize(&subObjBuilder, options);') + self._writer.write_template('${access_member}.serialize(&subObjBuilder);') def _gen_serializer_method_variant(self, field): # type: (ast.Field) -> None @@ -2025,44 +1990,18 @@ class _CppSourceFileWriter(_CppFileWriterBase): template_params[ 'cpp_type'] = 'std::vector<' + variant_type.cpp_type + '>' if variant_type.is_array else variant_type.cpp_type - template_params['param_opt'] = "builder" - if field.should_serialize_with_options: - template_params['param_opt'] += ', options' - with self._block('[${param_opt}](const ${cpp_type}& value) {', '},'): + with self._block('[builder](const ${cpp_type}& value) {', '},'): bson_cpp_type = cpp_types.get_bson_cpp_type(variant_type) if bson_cpp_type and bson_cpp_type.has_serializer(): assert not field.type.is_array expression = bson_cpp_type.gen_serializer_expression( - self._writer, 'value', - field.query_shape == ast.QueryShapeFieldType.CUSTOM) + self._writer, 'value') template_params['expression'] = expression - if not field.should_serialize_with_options: - self._writer.write_template( - 'builder->append(${field_name}, ${expression});') - elif field.query_shape == ast.QueryShapeFieldType.LITERAL: - self._writer.write_template( - 'options.serializeLiteral(${expression}).serializeForIDL(${field_name}, builder);' - ) - elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: - self._writer.write_template( - 'builder->append(${field_name}, options.serializeFieldPathFromString(${expression}));' - ) - else: - assert False + self._writer.write_template( + 'builder->append(${field_name}, ${expression});') else: - if not field.should_serialize_with_options: - self._writer.write_template( - 'idl::idlSerialize(builder, ${field_name}, value);') - elif field.query_shape == ast.QueryShapeFieldType.LITERAL: - self._writer.write_template( - 'options.serializeLiteral(value).serializeForIDL(${field_name}, builder);' - ) - elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: - self._writer.write_template( - 'idl::idlSerialize(builder, ${field_name}, options.serializeFieldPathFromString(value));' - ) - else: - assert False + self._writer.write_template( + 'idl::idlSerialize(builder, ${field_name}, value);') def _gen_serializer_method_common(self, field): # type: (ast.Field) -> None @@ -2091,27 +2030,11 @@ class _CppSourceFileWriter(_CppFileWriterBase): elif field.type.is_variant: self._gen_serializer_method_variant(field) else: - # Generate default serialization - # Note: BSONObjBuilder::append, which all three branches use, has overrides for std::vector also - if not field.should_serialize_with_options: - self._writer.write_line( - 'builder->append(%s, %s);' % (_get_field_constant_name(field), - _access_member(field))) - elif field.query_shape == ast.QueryShapeFieldType.LITERAL: - # serializeLiteral expects an ImplicitValue, which can't be constructed with an int64_t - expression_cast = "" - if field.type.cpp_type == "std::int64_t": - expression_cast = "(long long)" - self._writer.write_line( - 'options.serializeLiteral(%s%s).serializeForIDL(%s, builder);' - % (expression_cast, _access_member(field), - _get_field_constant_name(field))) - elif field.query_shape == ast.QueryShapeFieldType.ANONYMIZE: - self._writer.write_line( - 'builder->append(%s, options.serializeFieldPathFromString(%s));' % - (_get_field_constant_name(field), _access_member(field))) - else: - assert False + # Generate default serialization using BSONObjBuilder::append + # Note: BSONObjBuilder::append has overrides for std::vector also + self._writer.write_line( + 'builder->append(%s, %s);' % (_get_field_constant_name(field), + _access_member(field))) else: self._gen_serializer_method_struct(field) diff --git a/buildscripts/idl/idl/parser.py b/buildscripts/idl/idl/parser.py index 2582a6626b9..2d9925db500 100644 --- a/buildscripts/idl/idl/parser.py +++ b/buildscripts/idl/idl/parser.py @@ -134,8 +134,6 @@ def _generic_parser( if ctxt.is_mapping_node(second_node, first_name): syntax_node.__dict__[first_name] = rule_desc.mapping_parser_func( ctxt, second_node) - elif rule_desc.node_type == "required_bool_scalar": - syntax_node.__dict__[first_name] = ctxt.get_required_bool(second_node) else: raise errors.IDLError( "Unknown node_type '%s' for parser rule" % (rule_desc.node_type)) @@ -151,7 +149,7 @@ def _generic_parser( # A bool is never "None" like other types, it simply defaults to "false". # It means "if bool is None" will always return false and there is no support for required - # 'bool' at this time. Use the node type 'required_bool_scalar' if this behavior is not desired. + # 'bool' at this time. if not rule_desc.node_type == 'bool_scalar': if syntax_node.__dict__[name] is None: ctxt.add_missing_required_field_error(node, syntax_node_name, name) @@ -376,8 +374,6 @@ def _parse_field(ctxt, name, node): _RuleDesc("bool_scalar"), "always_serialize": _RuleDesc("bool_scalar"), - "query_shape": - _RuleDesc('scalar'), }) return field @@ -531,7 +527,6 @@ def _parse_struct(ctxt, spec, name, node): "generate_comparison_operators": _RuleDesc("bool_scalar"), "non_const_getter": _RuleDesc('bool_scalar'), "cpp_validator_func": _RuleDesc('scalar'), - "query_shape_component": _RuleDesc('bool_scalar'), }) # PyLint has difficulty with some iterables: https://github.com/PyCQA/pylint/issues/3105 @@ -1006,7 +1001,7 @@ def _propagate_globals(spec): idltype.cpp_type = _prefix_with_namespace(cpp_namespace, idltype.cpp_type) -def parse_file(stream, error_file_name): +def _parse(stream, error_file_name): # type: (Any, str) -> syntax.IDLParsedSpec """ Parse a YAML document into an idl.syntax tree. @@ -1110,7 +1105,7 @@ def parse(stream, input_file_name, resolver): """ # pylint: disable=too-many-locals - root_doc = parse_file(stream, input_file_name) + root_doc = _parse(stream, input_file_name) if root_doc.errors: return root_doc @@ -1147,7 +1142,7 @@ def parse(stream, input_file_name, resolver): # Parse imported file with resolver.open(resolved_file_name) as file_stream: - parsed_doc = parse_file(file_stream, resolved_file_name) + parsed_doc = _parse(file_stream, resolved_file_name) # Check for errors if parsed_doc.errors: diff --git a/buildscripts/idl/idl/struct_types.py b/buildscripts/idl/idl/struct_types.py index 7a8d0867a6d..b3b0df60532 100644 --- a/buildscripts/idl/idl/struct_types.py +++ b/buildscripts/idl/idl/struct_types.py @@ -67,27 +67,15 @@ class ArgumentInfo(object): def __init__(self, arg): # type: (str) -> None """Create a instance of the ArgumentInfo class by parsing the argument string.""" - self.defaults = None - equal_tokens = arg.split('=') - if len(equal_tokens) > 1: - self.defaults = equal_tokens[-1].strip() - - space_tokens = equal_tokens[0].strip().split(' ') - self.type = ' '.join(space_tokens[0:-1]) - self.name = space_tokens[-1] + parts = arg.split(' ') + self.type = ' '.join(parts[0:-1]) + self.name = parts[-1] def __str__(self): # type: () -> str """Return a formatted argument string.""" return "%s %s" % (self.type, self.name) # type: ignore - def get_string(self, get_defaults): - # type: (bool) -> str - """Return a formatted argument string.""" - if self.defaults and get_defaults: - return "%s %s = %s" % (self.type, self.name, self.defaults) # type: ignore - return "%s %s" % (self.type, self.name) # type: ignore - class MethodInfo(object): """Class that encapslates information about a method and how to declare, define, and call it.""" @@ -127,8 +115,7 @@ class MethodInfo(object): return common.template_args( "${pre_modifiers}${return_type}${method_name}(${args})${post_modifiers};", pre_modifiers=pre_modifiers, return_type=return_type_str, method_name=self.method_name, - args=', '.join( - [arg.get_string(True) for arg in self.args]), post_modifiers=post_modifiers) + args=', '.join([str(arg) for arg in self.args]), post_modifiers=post_modifiers) def get_definition(self): # type: () -> str @@ -147,7 +134,7 @@ class MethodInfo(object): "${pre_modifiers}${return_type}${class_name}::${method_name}(${args})${post_modifiers}", pre_modifiers=pre_modifiers, return_type=return_type_str, class_name=self.class_name, method_name=self.method_name, args=', '.join( - [arg.get_string(False) for arg in self.args]), post_modifiers=post_modifiers) + [str(arg) for arg in self.args]), post_modifiers=post_modifiers) def get_call(self, obj): # type: (Optional[str]) -> str @@ -281,19 +268,14 @@ class _StructTypeInfo(StructTypeInfoBase): def get_serializer_method(self): # type: () -> MethodInfo - args = ['BSONObjBuilder* builder'] - if self._struct.query_shape_component: - args.append("const SerializationOptions& options = {}") return MethodInfo( - common.title_case(self._struct.cpp_name), 'serialize', args, 'void', const=True) + common.title_case(self._struct.cpp_name), 'serialize', ['BSONObjBuilder* builder'], + 'void', const=True) def get_to_bson_method(self): # type: () -> MethodInfo - args = [] - if self._struct.query_shape_component: - args.append("const SerializationOptions& options = {}") return MethodInfo( - common.title_case(self._struct.cpp_name), 'toBSON', args, 'BSONObj', const=True) + common.title_case(self._struct.cpp_name), 'toBSON', [], 'BSONObj', const=True) def get_op_msg_request_serializer_method(self): # type: () -> Optional[MethodInfo] diff --git a/buildscripts/idl/idl/syntax.py b/buildscripts/idl/idl/syntax.py index 27edf420c43..0cfcf88e73d 100644 --- a/buildscripts/idl/idl/syntax.py +++ b/buildscripts/idl/idl/syntax.py @@ -471,10 +471,6 @@ class Field(common.SourceLocation): self.serialize_op_msg_request_only = False # type: bool self.constructed = False # type: bool - self.query_shape = None # type: Optional[str] - - self.hidden = False # type: bool - super(Field, self).__init__(file_name, line, column) @@ -545,8 +541,6 @@ class Struct(common.SourceLocation): # Internal property: cpp_namespace from globals section self.cpp_namespace = None # type: str - self.query_shape_component = False # type: bool - super(Struct, self).__init__(file_name, line, column) diff --git a/buildscripts/idl/idl_check_compatibility.py b/buildscripts/idl/idl_check_compatibility.py index 7b3ad4bee94..1a30f29428a 100644 --- a/buildscripts/idl/idl_check_compatibility.py +++ b/buildscripts/idl/idl_check_compatibility.py @@ -184,12 +184,6 @@ IGNORE_UNSTABLE_LIST: List[str] = [ # The 'runtimeConstants' field is a legacy field for internal use only and is not documented to # users. 'delete-param-runtimeConstants', - # The 'bypassEmptyTsReplacement' field is used by mongorestore and mongosync and is not - # documented to users. - 'insert-param-bypassEmptyTsReplacement', - 'update-param-bypassEmptyTsReplacement', - 'delete-param-bypassEmptyTsReplacement', - 'findAndModify-param-bypassEmptyTsReplacement', ] SKIPPED_FILES = [ @@ -830,13 +824,8 @@ def check_param_or_type_validator(ctxt: IDLCompatibilityContext, old_field: synt ctxt.add_command_or_param_type_validators_not_equal_error( cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) else: - new_field_name: str = cmd_name + "-param-" + new_field.name - # In SERVER-77382 we fixed the error handling of creating time-series collections by - # adding a new validator to two 'stable' fields, but it didn't break any stable API - # guarantees. - if new_field_name not in ["create-param-timeField", "create-param-metaField"]: - ctxt.add_command_or_param_type_contains_validator_error( - cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) + ctxt.add_command_or_param_type_contains_validator_error( + cmd_name, new_field.name, new_idl_file_path, type_name, is_command_parameter) def get_all_struct_fields(struct: syntax.Struct, idl_file: syntax.IDLParsedSpec, diff --git a/buildscripts/idl/lib.py b/buildscripts/idl/lib.py index 5484f98e9d3..c4ae1c08fea 100644 --- a/buildscripts/idl/lib.py +++ b/buildscripts/idl/lib.py @@ -56,14 +56,3 @@ def parse_idl(idl_path: str, import_directories: List[str]) -> syntax.IDLParsedS raise ValueError(f"Cannot parse {idl_path}") return parsed_doc - - -def is_third_party_idl(idl_path: str) -> bool: - """Check if an IDL file is under a third party directory.""" - third_party_idl_subpaths = [os.path.join("third_party", "mozjs"), "win32com"] - - for file_name in third_party_idl_subpaths: - if file_name in idl_path: - return True - - return False diff --git a/buildscripts/idl/tests/test_binder.py b/buildscripts/idl/tests/test_binder.py index 32f42772e8d..b52c755e34b 100644 --- a/buildscripts/idl/tests/test_binder.py +++ b/buildscripts/idl/tests/test_binder.py @@ -71,39 +71,6 @@ class TestBinder(testcase.IDLTestcase): # pylint: disable=too-many-public-methods - # Create a text wrap for common types. - common_types = textwrap.dedent(""" - types: - object: - description: foo - cpp_type: foo - bson_serialization_type: object - serializer: foo - deserializer: foo - - bool: - description: foo - cpp_type: foo - bson_serialization_type: any - serializer: foo - deserializer: foo - - string: - description: foo - cpp_type: foo - bson_serialization_type: string - serializer: foo - deserializer: foo - - any_type: - description: foo - cpp_type: foo - bson_serialization_type: any - serializer: foo - deserializer: foo - - """) - def test_empty(self): # type: () -> None """Test an empty document works.""" @@ -2775,172 +2742,6 @@ class TestBinder(testcase.IDLTestcase): reply_type: reply """), idl.errors.ERROR_ID_MISSING_ACCESS_CHECK) - def test_query_shape_component_validation(self): - """Tests for the query shape component fields.""" - self.assert_bind(self.common_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - query_shape: literal - type: string - field2: - type: bool - query_shape: parameter - """)) - - self.assert_bind_fail( - self.common_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - type: string - field2: - type: bool - query_shape: parameter - """), idl.errors.ERROR_ID_FIELD_MUST_DECLARE_SHAPE_LITERAL) - - self.assert_bind_fail( - self.common_types + textwrap.dedent(""" - structs: - struct1: - strict: true - description: "" - fields: - field1: - type: string - field2: - type: bool - query_shape: parameter - """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) - - # Validating query_shape_anonymize relies on std::string - basic_types = textwrap.dedent(""" - types: - string: - bson_serialization_type: string - description: "A BSON UTF-8 string" - cpp_type: "std::string" - deserializer: "mongo::BSONElement::str" - bool: - bson_serialization_type: bool - description: "A BSON bool" - cpp_type: "bool" - deserializer: "mongo::BSONElement::boolean" - """) - self.assert_bind(basic_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - query_shape: anonymize - type: string - field2: - query_shape: parameter - type: bool - """)) - - self.assert_bind(basic_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - query_shape: anonymize - type: array<string> - field2: - query_shape: parameter - type: bool - """)) - - self.assert_bind_fail( - basic_types + textwrap.dedent(""" - structs: - struct1: - strict: true - description: "" - fields: - field1: - query_shape: blah - type: string - """), idl.errors.ERROR_ID_QUERY_SHAPE_INVALID_VALUE) - - self.assert_bind_fail( - basic_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - query_shape: anonymize - type: bool - field2: - query_shape: parameter - type: bool - """), idl.errors.ERROR_ID_INVALID_TYPE_FOR_SHAPIFY) - - self.assert_bind_fail( - basic_types + textwrap.dedent(""" - structs: - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field1: - query_shape: anonymize - type: array<bool> - field2: - query_shape: parameter - type: bool - """), idl.errors.ERROR_ID_INVALID_TYPE_FOR_SHAPIFY) - - self.assert_bind_fail( - basic_types + textwrap.dedent(""" - structs: - StructZero: - strict: true - description: "" - fields: - field1: - query_shape: literal - type: string - """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) - - self.assert_bind_fail( - basic_types + textwrap.dedent(""" - structs: - StructZero: - strict: true - description: "" - fields: - field1: - type: string - struct1: - query_shape_component: true - strict: true - description: "" - fields: - field2: - type: StructZero - description: "" - query_shape: literal - """), idl.errors.ERROR_ID_CANNOT_DECLARE_SHAPE_LITERAL) - if __name__ == '__main__': diff --git a/buildscripts/idl/tests/test_generator.py b/buildscripts/idl/tests/test_generator.py index 97020f1eb8e..ee8a251aa53 100644 --- a/buildscripts/idl/tests/test_generator.py +++ b/buildscripts/idl/tests/test_generator.py @@ -38,7 +38,6 @@ $ coverage run run_tests.py && coverage html import os import unittest -from textwrap import dedent # import package so that it works regardless of whether we run as a module or file if __package__ is None: @@ -133,79 +132,6 @@ class TestGenerator(testcase.IDLTestcase): self.assertTrue(found, "Bad Header: " + header) - def test_object_with_custom_serializer_and_query_shape(self) -> None: - """Serialization with custom query_shape.""" - _, source = self.assert_generate(""" - types: - object_type_with_custom_serializer: - bson_serialization_type: object - description: ObjWithCustomSerializer - cpp_type: ObjWithCustomSerializer - serializer: ObjWithCustomSerializer::toBSON - deserializer: ObjWithCustomSerializer::parse - - structs: - QueryShapeSpec: - description: QueryShape - query_shape_component: true - fields: - internalObject: - type: object_type_with_custom_serializer - optional: false - description: internalObject - query_shape: custom - """) - - expected = dedent(""" - void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const { - invariant(_hasInternalObject); - - { - const BSONObj localObject = _internalObject.toBSON(options); - builder->append(kInternalObjectFieldName, localObject); - } - - }""") - self.assertIn(expected, source) - - def test_array_with_custom_serializer_and_query_shape(self) -> None: - """Serialization with custom query_shape used, array use case.""" - _, source = self.assert_generate(""" - types: - object_type_with_custom_serializer: - bson_serialization_type: object - description: ObjWithCustomSerializer - cpp_type: ObjWithCustomSerializer - serializer: ObjWithCustomSerializer::toBSON - deserializer: ObjWithCustomSerializer::parse - - structs: - QueryShapeSpec: - description: QueryShape - query_shape_component: true - fields: - internalObjectArray: - type: array<object_type_with_custom_serializer> - optional: false - description: internalObjectArray - query_shape: custom - """) - - expected = dedent(""" - void QueryShapeSpec::serialize(BSONObjBuilder* builder, const SerializationOptions& options) const { - invariant(_hasInternalObjectArray); - - { - BSONArrayBuilder arrayBuilder(builder->subarrayStart(kInternalObjectArrayFieldName)); - for (const auto& item : _internalObjectArray) { - const BSONObj localObject = item.toBSON(options); - arrayBuilder.append(localObject); - } - } - - }""") - self.assertIn(expected, source) - if __name__ == '__main__': diff --git a/buildscripts/jepsen_report.py b/buildscripts/jepsen_report.py deleted file mode 100644 index 00747f42f09..00000000000 --- a/buildscripts/jepsen_report.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Generate Evergreen reports from the Jepsen list-append workload.""" -import json -import re -import sys -import os -from datetime import datetime, timezone -from typing import List, Optional, Tuple - -from typing_extensions import TypedDict -import click - -from buildscripts.simple_report import Result, Report - - -class ParserOutput(TypedDict): - """Result of parsing jepsen log file. Each List[str] is a list of test names.""" - - success: List[str] - unknown: List[str] - crashed: List[str] - failed: List[str] - start: int - end: int - elapsed: int - - -_JEPSEN_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" -_JEPSEN_MILLI_RE = re.compile("([0-9]+){(.*)}") -_JEPSEN_TIME_RE = re.compile("[0-9]{4}-[0-8]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]+{.*}") - - -def _time_parse(time: str): - split = time.split(",") - date = datetime.strptime(split[0], _JEPSEN_TIME_FORMAT) - match = _JEPSEN_MILLI_RE.match(split[1]) - microseconds = 0 - if match: - microseconds = int(match[1]) * 1000 - - return date.replace(microsecond=microseconds, tzinfo=timezone.utc) - - -def _calc_time_from_log(log: str) -> Tuple[int, int, int]: - if not log: - return (0, 0, 0) - start_time = None - end_time = None - for line in log.splitlines(): - if _JEPSEN_TIME_RE.match(line): - if start_time is None: - start_time = _time_parse(line) - else: - end_time = _time_parse(line) - - if start_time is None or end_time is None: - return (0, 0, 0) - - elapsed_time = int(end_time.timestamp() - start_time.timestamp()) - - return (int(start_time.timestamp()), int(end_time.timestamp()), elapsed_time) - - -SUCCESS_RE = re.compile("([0-9]+) successes") -CRASH_RE = re.compile("([0-9]+) crashed") -UNKNOWN_RE = re.compile("([0-9]+) unknown") -FAIL_RE = re.compile("([0-9]+) failures") - - -def parse(text: List[str]) -> ParserOutput: - """Given a List of strings representing jepsen log file split by newlines, return the ParserOutput struct.""" - - successful_tests: List[str] = [] - indeterminate_tests: List[str] = [] - crashed_tests: List[str] = [] - failed_tests: List[str] = [] - target = None - table_matches = 0 - for line in text: - if "# Successful tests" in line: - target = successful_tests - continue - elif "# Indeterminate tests" in line: - target = indeterminate_tests - continue - elif "# Crashed tests" in line: - target = crashed_tests - continue - elif "# Failed tests" in line: - target = failed_tests - continue - - # at this point we're parsing this table: - # 29 successes - # 0 unknown - # 1 crashed - # 0 failures - s_match = SUCCESS_RE.match(line) - if s_match: - target = None - assert int(s_match[1]) == len(successful_tests) - table_matches += 1 - - u_match = UNKNOWN_RE.match(line) - if u_match: - target = None - assert int(u_match[1]) == len(indeterminate_tests) - table_matches += 1 - c_match = CRASH_RE.match(line) - if c_match: - target = None - assert int(c_match[1]) == len(crashed_tests) - table_matches += 1 - f_match = FAIL_RE.match(line) - if f_match: - target = None - assert int(f_match[1]) == len(failed_tests) - table_matches += 1 - - if target is not None and line.strip(): - target.append(line) - - assert table_matches == 4, f"Failed to parse summary table. Expected 4, found {table_matches}" - return ParserOutput({ - 'success': successful_tests, - 'unknown': indeterminate_tests, - 'crashed': crashed_tests, - 'failed': failed_tests, - }) - - -def _try_find_log_file(store: Optional[str], test_name) -> str: - if store is None: - return "" - - try: - with open(os.path.join(store, test_name, "jepsen.log")) as fh: - return fh.read() - - except Exception: # pylint: disable=broad-except - return "" - - -def report(out: ParserOutput, start_time: int, end_time: int, elapsed: int, - store: Optional[str]) -> Report: - """Given ParserOutput, return report.json as a dict.""" - - results = [] - failures = 0 - for test_name in out['success']: - log_raw = _try_find_log_file(store, test_name) - start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) - results.append( - Result(status='pass', exit_code=0, test_file=test_name, start=start_time, end=end_time, - elapsed=elapsed_time, log_raw=log_raw)) - - for test_name in out['failed']: - log_raw = _try_find_log_file(store, test_name) - start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) - failures += 1 - results.append( - Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, - elapsed=elapsed_time, log_raw=log_raw)) - - for test_name in out['crashed']: - log_raw = "Log files are unavailable for crashed tests because Jepsen does not save them separately. You may be able to find the exception and stack trace in the task log" - failures += 1 - results.append( - Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, - elapsed=elapsed, log_raw=log_raw)) - - for test_name in out['unknown']: - log_raw = _try_find_log_file(store, test_name) - start_time, end_time, elapsed_time = _calc_time_from_log(log_raw) - failures += 1 - results.append( - Result(status='fail', exit_code=1, test_file=test_name, start=start_time, end=end_time, - elapsed=elapsed_time, log_raw=log_raw)) - return Report({ - "failures": failures, - "results": results, - }) - - -def _get_log_lines(filename: str) -> List[str]: - with open(filename) as fh: - return fh.read().splitlines() - - -def _put_report(report_: Report) -> None: - with open("report.json", "w") as fh: - json.dump(report_, fh) - - -@click.command() -@click.option("--start_time", type=int, required=True) -@click.option("--end_time", type=int, required=True) -@click.option("--elapsed", type=int, required=True) -@click.option("--emit_status_files", type=bool, is_flag=True, default=False, - help="If true, emit status files for marking Evergreen tasks as system fails") -@click.option("--store", type=str, default=None, - help="Path to folder containing jepsen 'store' directory") -@click.argument("filename", type=str) -def main(filename: str, start_time: str, end_time: str, elapsed: str, emit_status_files: bool, - store: Optional[str]): - """Generate Evergreen reports from the Jepsen list-append workload.""" - - out = parse(_get_log_lines(filename)) - _put_report(report(out, start_time, end_time, elapsed, store)) - - exit_code = 255 - if out['crashed']: - exit_code = 2 - if emit_status_files: - with open("jepsen_system_fail.txt", "w") as fh: - fh.write(str(exit_code)) - else: - if out['unknown'] or out['failed']: - exit_code = 1 - else: - exit_code = 0 - - sys.exit(exit_code) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/linter/filediff.py b/buildscripts/linter/filediff.py index 64521d9ed47..7dbad6d9d58 100644 --- a/buildscripts/linter/filediff.py +++ b/buildscripts/linter/filediff.py @@ -19,15 +19,18 @@ from buildscripts.patch_builds.change_data import generate_revision_map, \ LOGGER = structlog.get_logger(__name__) MONGO_REVISION_ENV_VAR = "REVISION" +ENTERPRISE_REVISION_ENV_VAR = "ENTERPRISE_REV" def _get_repos_and_revisions() -> Tuple[List[Repo], RevisionMap]: """Get the repo object and a map of revisions to compare against.""" modules = git.get_module_paths() repos = [Repo(path) for path in modules] - revision_map = generate_revision_map(repos, { - "mongo": os.environ.get(MONGO_REVISION_ENV_VAR), - }) + revision_map = generate_revision_map( + repos, { + "mongo": os.environ.get(MONGO_REVISION_ENV_VAR), + "enterprise": os.environ.get(ENTERPRISE_REVISION_ENV_VAR) + }) return repos, revision_map diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py index b4d0bba0490..b31a9dbf8db 100644 --- a/buildscripts/moduleconfig.py +++ b/buildscripts/moduleconfig.py @@ -33,26 +33,16 @@ import os def discover_modules(module_root, allowed_modules): - # pylint: disable=too-many-branches """Scan module_root for subdirectories that look like MongoDB modules. Return a list of imported build.py module objects. """ found_modules = [] - found_module_names = [] if allowed_modules is not None: allowed_modules = allowed_modules.split(',') - # When `--modules=` is passed, the split on empty string is represented - # in memory as [''] - if allowed_modules == ['']: - allowed_modules = [] if not os.path.isdir(module_root): - if allowed_modules: - raise RuntimeError( - f"Requested the following modules: {allowed_modules}, but the module root '{module_root}' could not be found. Check the module root, or remove the module from the scons invocation." - ) return found_modules for name in os.listdir(module_root): @@ -76,17 +66,11 @@ def discover_modules(module_root, allowed_modules): if getattr(module, "name", None) is None: module.name = name found_modules.append(module) - found_module_names.append(name) finally: fp.close() except (FileNotFoundError, IOError): pass - if allowed_modules is not None: - missing_modules = set(allowed_modules) - set(found_module_names) - if missing_modules: - raise RuntimeError(f"Failed to locate all modules. Could not find: {missing_modules}") - return found_modules diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py index 035b00f3a07..7dc529e39d0 100755 --- a/buildscripts/mongosymb.py +++ b/buildscripts/mongosymb.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """Script and library for symbolizing MongoDB stack traces. To use as a script, paste the JSON object on the line after ----- BEGIN BACKTRACE ----- into the @@ -23,42 +24,20 @@ import signal import subprocess import sys import time -from abc import ABC, abstractmethod from collections import OrderedDict -from datetime import timedelta from pathlib import Path -from typing import Dict, List, Any, Union, Optional +from typing import Dict import requests # pylint: disable=wrong-import-position # pylint: disable=too-many-branches -from tenacity import wait_fixed, stop_after_delay, retry_if_result, Retrying - sys.path.append(str(Path(os.getcwd(), __file__).parent.parent)) -from buildscripts.util.oauth import Configs, get_oauth_credentials, get_client_cred_oauth_credentials +from buildscripts.util.oauth import Configs, get_oauth_credentials from buildscripts.build_system_options import PathOptions -SYMBOLIZER_PATH_ENV = "MONGOSYMB_SYMBOLIZER_PATH" -# since older versions may have issues with symbolizing, we are setting the toolchain version to v4 -DEFAULT_SYMBOLIZER_PATH = "/opt/mongodbtoolchain/v4/bin/llvm-symbolizer" - - -class DbgFileResolver(ABC): - """Base gdb path resolver class.""" - - @abstractmethod - def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]: - """ - To get path for given build info. - - :param soinfo: soinfo as dict - :return: path as string or None (if path not found) - """ - raise NotImplementedError - -class PathDbgFileResolver(DbgFileResolver): +class PathDbgFileResolver(object): """PathDbgFileResolver class.""" def __init__(self, bin_path_guess): @@ -75,7 +54,7 @@ class PathDbgFileResolver(DbgFileResolver): return path if path else self._bin_path_guess -class S3BuildidDbgFileResolver(DbgFileResolver): +class S3BuildidDbgFileResolver(object): """S3BuildidDbgFileResolver class.""" def __init__(self, cache_dir, s3_bucket): @@ -94,7 +73,7 @@ class S3BuildidDbgFileResolver(DbgFileResolver): if not os.path.exists(build_id_path): try: self._get_from_s3(build_id) - except Exception: # noqa pylint: disable=broad-except + except Exception: # pylint: disable=broad-except ex = sys.exc_info()[0] sys.stderr.write("Failed to find debug symbols for {} in s3: {}\n".format( build_id, ex)) @@ -168,7 +147,7 @@ class CachedResults(object): return self._cached_results.get(key) -class PathResolver(DbgFileResolver): +class PathResolver(object): """ Class to find path for given buildId. @@ -184,17 +163,16 @@ class PathResolver(DbgFileResolver): # This amount of attributes are necessary. # the main (API) sever that we'll be sending requests to - default_host = "https://symbolizer-service.server-tig.prod.corp.mongodb.com" - default_cache_dir = os.path.join(os.getcwd(), "build", "symbolizer_downloads_cache") - default_creds_file_path = os.path.join(os.getcwd(), ".symbolizer_credentials.json") + default_host = 'https://symbolizer-service.server-tig.prod.corp.mongodb.com' + default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbolizer_downloads_cache') + default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" - download_timeout_secs = timedelta(minutes=4).total_seconds() def __init__(self, host: str = None, cache_size: int = 0, cache_dir: str = None, client_credentials_scope: str = None, client_credentials_user_name: str = None, - client_id: str = None, client_secret: str = None, redirect_port: int = None, - scope: str = None, auth_domain: str = None): + client_id: str = None, redirect_port: int = None, scope: str = None, + auth_domain: str = None): """ Initialize instance. @@ -209,7 +187,6 @@ class PathResolver(DbgFileResolver): self.client_credentials_scope = client_credentials_scope or self.default_client_credentials_scope self.client_credentials_user_name = client_credentials_user_name or self.default_client_credentials_user_name self.client_id = client_id - self.client_secret = client_secret self.redirect_port = redirect_port self.scope = scope self.auth_domain = auth_domain @@ -235,17 +212,11 @@ class PathResolver(DbgFileResolver): data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get("expire_time") if time.time() < expire_time: - # credentials not expired yet + # credentials hasn't expired yet self.http_client.headers.update({"Authorization": f"Bearer {access_token}"}) return - if self.client_id and self.client_secret: - # auth using secrets - credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret, - self.configs) - else: - # since we don't have access to secrets, ask user to auth manually - credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True) + credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True) self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"}) # write credentials to local file for further useage @@ -294,7 +265,7 @@ class PathResolver(DbgFileResolver): :param url: download URL :return: full name for local file """ - return url.split("/")[-1] + return url.split('/')[-1] @staticmethod def unpack(path: str) -> str: @@ -302,9 +273,9 @@ class PathResolver(DbgFileResolver): Use to utar/unzip files. :param path: full path of file - :return: full path to directory of unpacked file + :return: full path of directory of unpacked file """ - out_dir = path.replace(".tgz", "", 1) + out_dir = path.replace('.tgz', '', 1) if not os.path.exists(out_dir): os.mkdir(out_dir) @@ -325,25 +296,12 @@ class PathResolver(DbgFileResolver): filename = self.url_to_filename(url) path = os.path.join(self.cache_dir, filename) if not os.path.exists(path): - print("Downloading the file...") - self.get_file_from_service(url, path) + subprocess.check_call(['wget', url], cwd=self.cache_dir) else: - print("File already exists in cache") + print('File aready exists in cache') exists_locally = True return path, exists_locally - def get_file_from_service(self, url: str, local_path: str) -> None: - """ - Get file from URL and write to a local file. - - :param url: URL string - :param local_path: full name for local file - """ - with requests.get(url, stream=True, timeout=self.download_timeout_secs) as response: - with open(local_path, "wb") as file: - for chunk in response.iter_content(chunk_size=2 * 1024 * 1024): - file.write(chunk) - def get_dbg_file(self, soinfo: dict) -> str or None: """ To get path for given buildId. @@ -352,26 +310,32 @@ class PathResolver(DbgFileResolver): :return: path as string or None (if path not found) """ build_id = soinfo.get("buildId", "").lower() - version = soinfo.get("version") - binary_name = "mongo" + binary_name = 'mongo' # search from cached results path = self.get_from_cache(build_id) if not path: # path does not exist in cache, so we send request to server try: - search_parameters = {"build_id": build_id} - if version: - search_parameters["version"] = version - print(f"Getting data from service... Search parameters: {search_parameters}") - response = self.http_client.get(f"{self.host}/find_by_id", params=search_parameters) + response = self.http_client.get(f'{self.host}/find_by_id', + params={'build_id': build_id}) if response.status_code != 200: + # if we could not find the path of binary, that might be system library. + # we can try using frame's own `path` data. + # symbolization can succeed only if that binary exists on local + # machine (more specifically: in the given path). + system_path = soinfo.get('path') + if system_path: + sys.stdout.write( + f"Could not find path of binary from symbolizer web service. Trying to use the " + f"provided path: {system_path}\n") + return system_path sys.stderr.write( f"Server returned unsuccessful status: {response.status_code}, " f"response body: {response.text}\n") return None else: - data = response.json().get("data", {}) - path, binary_name = data.get("debug_symbols_url"), data.get("file_name") + data = response.json().get('data', {}) + path, binary_name = data.get('debug_symbols_url'), data.get('file_name') except Exception as err: # noqa pylint: disable=broad-except sys.stderr.write(f"Error occurred while trying to get response from server " f"for buildId({build_id}): {err}\n") @@ -388,7 +352,7 @@ class PathResolver(DbgFileResolver): try: dl_path, exists_locally = self.download(path) if exists_locally: - path = dl_path.replace(".tgz", "", 1) + path = dl_path.replace('.tgz', '', 1) else: print("Downloaded, now unpacking...") path = self.unpack(dl_path) @@ -398,8 +362,8 @@ class PathResolver(DbgFileResolver): # if file has extension, it is good. if not, we should append .debug, because those without extension are # from release builds, and their debug symbol files contain .debug extension. # we need to map those 2 different file names ('<name>' becomes '<name>.debug'). - if not binary_name.endswith(".debug"): - binary_name = f"{binary_name}.debug" + if not binary_name.endswith('.debug') and not binary_name.endswith('.so'): + binary_name = f'{binary_name}.debug' inner_folder_name = self.path_options.get_binary_folder_name(binary_name) @@ -417,7 +381,6 @@ def parse_input(trace_doc, dbg_path_resolver): return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry} base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"]) - version = get_version(trace_doc) frames = [] for frame in trace_doc["backtrace"]: @@ -427,8 +390,6 @@ def parse_input(trace_doc, dbg_path_resolver): ) continue soinfo = base_addr_map.get(frame["b"], {}) - if version: - soinfo["version"] = version elf_type = soinfo.get("elfType", 0) if elf_type == 3: addr_base = "0" @@ -438,7 +399,7 @@ def parse_input(trace_doc, dbg_path_resolver): addr_base = soinfo.get("vmaddr", "0") addr = int(addr_base, 16) + int(frame["o"], 16) # addr currently points to the return address which is the one *after* the call. x86 is - # variable length so going backwards is difficult. However, llvm-symbolizer seems to do the + # variable length so going backwards is difficult. However llvm-symbolizer seems to do the # right thing if we just subtract 1 byte here. This has the downside of also adjusting the # address of instructions that cause signals (such as segfaults and divide-by-zero) which # are already correct, but there doesn't seem to be a reliable way to detect that case. @@ -450,16 +411,6 @@ def parse_input(trace_doc, dbg_path_resolver): return frames -def get_version(trace_doc: Dict[str, Any]) -> Optional[str]: - """ - Get version from trace doc. - - :param trace_doc: Traceback dict. - :return: Version string or None. - """ - return trace_doc.get("processInfo", {}).get("mongodbVersion") - - def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format, **kwargs): """Return a list of symbolized stack frames from a trace_doc in MongoDB stack dump format.""" @@ -467,23 +418,24 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i # Keep frames in kwargs to avoid changing the function signature. frames = kwargs.get("frames") if frames is None: - total_seconds_for_retries = kwargs.get("total_seconds_for_retries", 0) - frames = preprocess_frames_with_retries(dbg_path_resolver, trace_doc, input_format, - total_seconds_for_retries) + frames = preprocess_frames(dbg_path_resolver, trace_doc, input_format) if not symbolizer_path: - symbolizer_path = os.environ.get(SYMBOLIZER_PATH_ENV) + symbolizer_path_env = "MONGOSYMB_SYMBOLIZER_PATH" + default_symbolizer_path = "llvm-symbolizer" + symbolizer_path = os.environ.get(symbolizer_path_env) if not symbolizer_path: - print(f"Env value for '{SYMBOLIZER_PATH_ENV}' not found, using" - f" '{DEFAULT_SYMBOLIZER_PATH}' as a default executable path.") - symbolizer_path = DEFAULT_SYMBOLIZER_PATH + print( + f"Env value for '{symbolizer_path_env}' not found, using '{default_symbolizer_path}' " + f"as a defualt executable path.") + symbolizer_path = default_symbolizer_path symbolizer_args = [symbolizer_path] for dh in dsym_hint: symbolizer_args.append("-dsym-hint={}".format(dh)) symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=sys.stdout) + stderr=open("/dev/null")) def extract_symbols(stdin): """Extract symbol information from the output of llvm-symbolizer. @@ -515,7 +467,7 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i for frame in frames: if frame["path"] is None: - print(f"Path not found in frame: {frame}") + print("Path not found in frame:", frame) continue symbol_line = "CODE {path:} {addr:}\n".format(**frame) symbolizer_process.stdin.write(symbol_line.encode()) @@ -526,17 +478,8 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i return frames -def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], - input_format: str) -> List[Dict[str, Any]]: - """ - Process the paths in frame objects. - - :param dbg_path_resolver: debug symbols file path resolver - :param trace_doc: traceback object - :param input_format: format of input - :return: the list of traceback frames - """ - +def preprocess_frames(dbg_path_resolver, trace_doc, input_format): + """Process the paths in frame objects.""" if input_format == "classic": frames = parse_input(trace_doc, dbg_path_resolver) elif input_format == "thin": @@ -545,43 +488,9 @@ def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, A frame["path"] = dbg_path_resolver.get_dbg_file(frame) else: raise ValueError('Unknown input format "{}"'.format(input_format)) - return frames -def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool: - """ - Check whether not found paths in frames ratio is higher than 0.5. - - :param frames: the list of traceback frames - :return: True if ratio is higher than 0.5 - """ - not_found = [1 for f in frames if f.get("path") is None] - not_found_ratio = len(not_found) / (len(frames) or 1) - return not_found_ratio >= 0.5 - - -def preprocess_frames_with_retries(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any], - input_format: str, - total_seconds_for_retries: int = 0) -> List[Dict[str, Any]]: - """ - Process the paths in frame objects. - - :param dbg_path_resolver: debug symbols file path resolver - :param trace_doc: traceback object - :param input_format: format of input - :param total_seconds_for_retries: max wait time for retries in seconds - :return: the list of traceback frames - """ - - retrying = Retrying( - retry=retry_if_result(has_high_not_found_paths_ratio), wait=wait_fixed(60), - stop=stop_after_delay(total_seconds_for_retries), - retry_error_callback=lambda retry_state: retry_state.outcome.result()) - - return retrying(preprocess_frames, dbg_path_resolver, trace_doc, input_format) - - def classic_output(frames, outfile, **kwargs): # pylint: disable=unused-argument """Provide classic output.""" for frame in frames: @@ -607,12 +516,6 @@ def make_argument_parser(parser=None, **kwargs): parser.add_argument('--debug-file-resolver', choices=['path', 's3', 'pr'], default='pr') parser.add_argument('--src-dir-to-move', action="store", type=str, default=None, help="Specify a src dir to move to /data/mci/{original_buildid}/src") - parser.add_argument( - '--total-seconds-for-retries', default=0, type=int, - help="If web service fails to find path for given build id, it could be because mapping " - "process was not finished yet. We can wait for it to finish and retry again. Each retry" - " adds 2 minutes to previous wait time. It is guaranteed that total wait time does not exceed this " - "specified amount.") parser.add_argument('--live', action='store_true') s3_group = parser.add_argument_group( @@ -628,8 +531,6 @@ def make_argument_parser(parser=None, **kwargs): help='URL of web service running the API to get debug symbol URL') pr_group.add_argument('--pr-cache-dir', default='', help='Full path to a directory to store cache/files') - pr_group.add_argument('--client-secret', default='', help='Secret key for Okta Oauth') - pr_group.add_argument('--client-id', default='', help='Client id for Okta Oauth') # caching mechanism is currently not fully developed and needs more advanced cleaning techniques, we add an option # to enable it after completing the implementation @@ -664,17 +565,11 @@ def substitute_stdin(options, resolver): if not trace_doc["backtrace"]: print("Trace is empty, skipping...") continue - frames = symbolize_frames( - trace_doc, - resolver, - options.symbolizer_path, - [], - options.output_format, - ) + frames = symbolize_frames(trace_doc, resolver, options.symbolizer_path, [], + options.output_format) print(prefix) print("Symbolizing...") classic_output(frames, sys.stdout, indent=2) - print("Completed, waiting for input...") else: print(line) @@ -688,8 +583,7 @@ def main(options): elif options.debug_file_resolver == 's3': resolver = S3BuildidDbgFileResolver(options.s3_cache_dir, options.s3_bucket) elif options.debug_file_resolver == 'pr': - resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir, - client_secret=options.client_secret, client_id=options.client_id) + resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir) if options.live: print("Entering live mode") @@ -703,7 +597,7 @@ def main(options): if not trace_doc or not trace_doc.strip(): print("Please provide the backtrace through stdin for symbolization;" - " e.g. `your/symbolization/command < /file/with/stacktrace`") + "e.g. `your/symbolization/command < /file/with/stacktrace`") # Search the trace_doc for an object having "backtrace" and "processInfo" keys. def bt_search(obj): @@ -730,8 +624,8 @@ def main(options): except json.JSONDecodeError: pass else: - sys.stderr.write("could not find json backtrace object in input\n") - sys.exit(1) + print("could not find json backtrace object in input", file=sys.stderr) + exit(1) output_fn = None if options.output_format == 'json': @@ -739,8 +633,7 @@ def main(options): if options.output_format == 'classic': output_fn = classic_output - frames = preprocess_frames_with_retries(resolver, trace_doc, options.input_format, - options.total_seconds_for_retries) + frames = preprocess_frames(resolver, trace_doc, options.input_format) if options.src_dir_to_move and resolver.mci_build_dir is not None: try: diff --git a/buildscripts/package_test/files/sources.list.debian8 b/buildscripts/package_test/files/sources.list.debian8 new file mode 100644 index 00000000000..0ad1174270a --- /dev/null +++ b/buildscripts/package_test/files/sources.list.debian8 @@ -0,0 +1,4 @@ +deb http://archive.debian.org/debian jessie main +deb-src http://archive.debian.org/debian jessie main +deb http://security.debian.org/ jessie/updates main +deb-src http://security.debian.org/ jessie/updates main diff --git a/buildscripts/package_test/kitchen.legacy.yml b/buildscripts/package_test/kitchen.legacy.yml new file mode 100644 index 00000000000..c44c9a92d5e --- /dev/null +++ b/buildscripts/package_test/kitchen.legacy.yml @@ -0,0 +1,122 @@ +--- +driver: + name: ec2 + region: us-east-1 + vpc_mode: true + vpc_id: <%= ENV['KITCHEN_VPC'] %> + subnet_id: <%= ENV['KITCHEN_SUBNET'] %> + security_group_ids: + - <%= ENV['KITCHEN_SECURITY_GROUP'] %> + aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> + interface: private + associate_public_ip: true + tags: + name: "server package test" + owner: "build" + expire-on: "<%= ENV['KITCHEN_EXPIRE'] %>" + +verifier: + name: inspec + sudo: true + +provisioner: + name: chef_zero + log_level: info + require_chef_omnibus: 12 + +platforms: + - name: amazon-x86-64 + driver: + image_id: ami-0080e4c5bc078760e + transport: + username: ec2-user + - name: amazon2-x86-64 + driver: + image_id: ami-428aa838 + transport: + username: ec2-user + - name: amazon2-arm64 + driver: + image_id: ami-0c582118883b46f4f + instance_type: c6g.medium + transport: + username: ec2-user + - name: debian10-x86-64 + driver: + image_id: ami-0dedf6a6502877301 + transport: + username: admin + - name: rhel70-x86-64 + driver: + image_id: ami-2051294a + transport: + username: root + - name: rhel80-x86-64 + driver: + image_id: ami-0c322300a1dd5dc79 + transport: + username: ec2-user + - name: rhel82-arm64 + driver: + image_id: ami-029ba835ddd43c34f + instance_type: m6g.medium + transport: + username: ec2-user + - name: suse11-x86-64 + driver: + image_id: ami-7f2e6015 + transport: + username: ec2-user + - name: suse12-x86-64 + driver: + image_id: ami-043eebeabcc4e3d35 + transport: + username: ec2-user + - name: suse15-x86-64 + driver: + image_id: ami-06ea7729e394412c8 + transport: + username: ec2-user + - name: ubuntu1204-x86-64 + driver: + image_id: ami-3fec7956 + transport: + username: ubuntu + - name: ubuntu1404-x86-64 + driver: + image_id: ami-1d8c9574 + transport: + username: ubuntu + - name: ubuntu1604-x86-64 + driver: + image_id: ami-64140d0e + transport: + username: ubuntu + - name: ubuntu1804-x86-64 + driver: + image_id: ami-7ad76705 + transport: + username: ubuntu + - name: ubuntu1804-arm64 + driver: + image_id: ami-01ac7d9c1179d7b74 + instance_type: m6g.medium + - name: ubuntu2004-x86-64 + driver: + image_id: ami-068663a3c619dd892 + transport: + username: ubuntu + - name: ubuntu2004-arm64 + driver: + image_id: ami-00579fbb15b954340 + instance_type: m6g.medium + +transport: + ssh_key: ~/.ssh/kitchen.pem + +suites: + - name: service + run_list: + - recipe[package_test::install_mongodb] + attributes: + artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/kitchen.yml b/buildscripts/package_test/kitchen.yml new file mode 100644 index 00000000000..f27ab0bfd26 --- /dev/null +++ b/buildscripts/package_test/kitchen.yml @@ -0,0 +1,139 @@ +--- +driver: + name: ec2 + region: us-east-1 + vpc_mode: true + vpc_id: <%= ENV['KITCHEN_VPC'] %> + subnet_id: <%= ENV['KITCHEN_SUBNET'] %> + security_group_ids: + - <%= ENV['KITCHEN_SECURITY_GROUP'] %> + aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> + interface: private + associate_public_ip: true + tags: + name: "server package test" + owner: "build" + expire-on: "<%= ENV['KITCHEN_EXPIRE'] %>" + +verifier: + name: inspec + sudo: true + +provisioner: + name: chef_zero + log_level: info + product_name: cinc + product_version: 17.9.52 + download_url: https://omnitruck.cinc.sh/install.sh + +platforms: + - name: amazon-x86-64 + driver: + image_id: ami-0080e4c5bc078760e + transport: + username: ec2-user + - name: amazon2-x86-64 + driver: + image_id: ami-02013ed1a71752ea7 + transport: + username: ec2-user + - name: amazon2-arm64 + driver: + image_id: ami-0c582118883b46f4f + instance_type: c6g.medium + transport: + username: ec2-user + - name: debian71-x86-64 + driver: + image_id: ami-4b124a22 + transport: + username: admin + - name: debian81-x86-64 + driver: + image_id: ami-896d85e2 + transport: + username: admin + - name: debian10-x86-64 + driver: + image_id: ami-0dedf6a6502877301 + transport: + username: admin + - name: debian11-x86-64 + driver: + image_id: ami-06a80441f25333895 + transport: + username: admin + - name: rhel70-x86-64 + driver: + image_id: ami-0051b1b2c5a166c8c + transport: + username: root + - name: rhel80-x86-64 + driver: + image_id: ami-0c322300a1dd5dc79 + transport: + username: ec2-user + - name: rhel82-arm64 + driver: + image_id: ami-029ba835ddd43c34f + instance_type: m6g.medium + transport: + username: ec2-user + - name: suse11-x86-64 + driver: + image_id: ami-7f2e6015 + transport: + username: ec2-user + - name: suse12-x86-64 + driver: + image_id: ami-043eebeabcc4e3d35 + transport: + username: ec2-user + - name: suse15-x86-64 + driver: + image_id: ami-06ea7729e394412c8 + transport: + username: ec2-user + - name: ubuntu1204-x86-64 + driver: + image_id: ami-3fec7956 + transport: + username: ubuntu + - name: ubuntu1404-x86-64 + driver: + image_id: ami-1d8c9574 + transport: + username: ubuntu + - name: ubuntu1604-x86-64 + driver: + image_id: ami-64140d0e + transport: + username: ubuntu + - name: ubuntu1804-x86-64 + driver: + image_id: ami-7ad76705 + transport: + username: ubuntu + - name: ubuntu1804-arm64 + driver: + image_id: ami-01ac7d9c1179d7b74 + instance_type: m6g.medium + - name: ubuntu2004-x86-64 + driver: + image_id: ami-068663a3c619dd892 + transport: + username: ubuntu + - name: ubuntu2004-arm64 + driver: + image_id: ami-00579fbb15b954340 + instance_type: m6g.medium + +transport: + ssh_key: ~/.ssh/kitchen.pem + +suites: + - name: service + run_list: + - recipe[package_test::install_mongodb] + attributes: + artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb new file mode 100644 index 00000000000..3bbfb147acc --- /dev/null +++ b/buildscripts/package_test/metadata.rb @@ -0,0 +1,2 @@ +name 'package_test' +version '0.1.0' diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb new file mode 100644 index 00000000000..d57546ce6b9 --- /dev/null +++ b/buildscripts/package_test/recipes/install_mongodb.rb @@ -0,0 +1,146 @@ +# This Chef task installs MongoDB in a new EC2 instance spun up by Kitchen in +# preparation for running some basic server functionality tests. + +artifacts_tarball = 'artifacts.tgz' +homedir = "/tmp" + +ruby_block 'allow sudo over tty' do + block do + file = Chef::Util::FileEdit.new('/etc/sudoers') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty') + file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw') + file.write_file + end +end + +# This file limits processes to 1024. It therefore interfereres with `ulimit -u` when present. +if platform_family? 'rhel' or platform_family? 'amazon' + file '/etc/security/limits.d/90-nproc.conf' do + action :delete + end +end + +remote_file "#{homedir}/#{artifacts_tarball}" do + source node['artifacts_url'] +end + +execute 'extract artifacts' do + command "tar xzvf #{artifacts_tarball}" + live_stream true + cwd homedir +end + +if platform_family? 'debian' + + # SERVER-40491 Debian 8 sources.list need to point to archive url + if node['platform'] == 'debian' and node['platform_version'] == '8.1' + cookbook_file '/etc/apt/sources.list' do + source 'sources.list.debian8' + owner 'root' + group 'root' + mode '0644' + action :create + end + end + + execute 'apt update' do + command 'apt update' + live_stream true + end + + ENV['DEBIAN_FRONTEND'] = 'noninteractive' + package 'openssl' + + # the ubuntu image does not have some dependencies installed by default + # and it is required for the install_compass script + if node['platform'] == 'ubuntu' and node['platform_version'] == '20.04' + execute 'install dependencies ubuntu 20.04' do + command 'apt-get install -y python3 libsasl2-modules-gssapi-mit' + live_stream true + end + link '/usr/bin/python' do + to '/usr/bin/python3' + end + else + execute 'install dependencies' do + command 'apt-get install -y python libsasl2-modules-gssapi-mit' + live_stream true + end + end + + # dpkg returns 1 if dependencies are not satisfied, which they will not be + # for enterprise builds. We install dependencies in the next block. + execute 'install mongod' do + command 'dpkg -i `find . -name "*server*.deb"`' + live_stream true + cwd homedir + returns [0, 1] + end + + # install the tools so we can test install_compass + execute 'install mongo tools' do + command 'dpkg -i `find . -name "*tools-extra*.deb"`' + live_stream true + cwd homedir + returns [0, 1] + end + + # yum and zypper fetch dependencies automatically, but dpkg does not. + # Installing the dependencies explicitly is fragile, so we reply on apt-get + # to install dependencies after the fact. + execute 'update and fix broken dependencies' do + command 'apt update && apt -y -f install' + live_stream true + end +end + +if platform_family? 'rhel' or platform_family? 'amazon' + bash 'wait for yum updates if they are running' do + code <<-EOH + sleep 120 + EOH + end + execute 'install mongod' do + command 'yum install -y `find . -name "*server*.rpm"`' + live_stream true + cwd homedir + end + + # install the tools so we can test install_compass + execute 'install mongo tools' do + command 'yum install -y `find . -name "*tools-extra*.rpm"`' + live_stream true + cwd homedir + end +end + +if platform_family? 'suse' + bash 'wait for zypper lock to be released' do + code <<-EOD + retry_counter=0 + # We also need to make sure another instance of zypper isn't running while + # we do our install, so just run zypper refresh until it doesn't fail. + # Waiting for 2 minutes is copied from an internal project where we do this. + until [ "$retry_counter" -ge "12" ]; do + zypper refresh && exit 0 + retry_counter=$(($retry_counter + 1)) + [ "$retry_counter" = "12" ] && break + sleep 10 + done + exit 1 + EOD + flags "-x" + end + + execute 'install mongod' do + command 'zypper --no-gpg-checks -n install `find . -name "*server*.rpm"`' + live_stream true + cwd homedir + end + + execute 'install mongo tools' do + command 'zypper --no-gpg-checks -n install `find . -name "*tools-extra*.rpm"`' + live_stream true + cwd homedir + end +end diff --git a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb new file mode 100644 index 00000000000..821f55f1a57 --- /dev/null +++ b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb @@ -0,0 +1,219 @@ +############################################################ +# This section verifies start, stop, and restart after +# installation within a new EC2 instance spun up by Kitchen. +# +# - stop mongod so that we begin testing from a stopped state +# - verify start, stop, and restart +############################################################ + +# service is not in path for commands with sudo on suse +service = os[:name] == 'suse' ? '/sbin/service' : 'service' + +describe command("#{service} mongod stop") do + its('exit_status') { should eq 0 } +end + +describe command("#{service} mongod start") do + its('exit_status') { should eq 0 } +end + +# Inspec treats all amazon linux as upstart, we explicitly make it use +# systemd_service https://github.com/chef/inspec/issues/2639 +if (os[:name] == 'amazon' and os[:release] == '2.0') + describe systemd_service('mongod') do + it { should be_running } + end +else + describe service('mongod') do + it { should be_running } + end +end + +describe command("#{service} mongod stop") do + its('exit_status') { should eq 0 } +end + +describe command("#{service} mongod restart") do + its('exit_status') { should eq 0 } +end + +if (os[:name] == 'amazon' and os[:release] == '2.0') + describe systemd_service('mongod') do + it { should be_running } + end +else + describe service('mongod') do + it { should be_running } + end +end + +if os[:arch] == 'x86_64' + # install_compass does not run Amazon Linux but *does* run on Amazon Linux 2, + # but the 'redhat' family includes both, apparently. We need to specifically + # exclude Amazon Linux from the set of allowed distributions here because the + # version strings would otherwise pass it. + if ((os[:family] == 'redhat' and os[:name] != "amazon" and os[:release].split('.')[0].to_i >= 7) or + (os[:name] == 'ubuntu' and os[:release].split('.')[0].to_i >= 16) or + (os[:name] == 'debian' and os[:release].split('.')[0].to_i >= 9) or + (os[:name] == 'amazon' and os[:release].split('.')[0].to_i == 2)) + describe command("install_compass") do + its('exit_status') { should eq 0 } + its('stderr') { should eq '' } + end + elsif os[:name] == 'suse' + describe command("install_compass") do + its('exit_status') { should eq 1 } + its('stderr') { should match /You are using an unsupported platform/ } + end + else + describe command("install_compass") do + its('exit_status') { should eq 1 } + its('stderr') { should match /You are using an unsupported Linux distribution/ } + end + end +else + describe command("install_compass") do + its('exit_status') { should eq 1 } + its('stderr') { should match /Sorry, MongoDB Compass is only supported on 64-bit Intel platforms/ } + end +end + +############################################################ +# This section verifies files, directories, and users +# - files and directories exist and have correct attributes +# - mongod user exists and has correct attributes +############################################################ + +# convenience variables for init system and package type +upstart = (os[:name] == 'ubuntu' && os[:release][0..1] == '14') || + (os[:name] == 'amazon') +sysvinit = if (os[:name] == 'debian' && os[:release][0] == '7') || + (os[:name] == 'redhat' && os[:release][0] == '6') || + (os[:name] == 'suse' && os[:release][0..1] == '11') || + (os[:name] == 'ubuntu' && os[:release][0..1] == '12') + true + else + false + end +systemd = !(upstart || sysvinit) +rpm = if os[:name] == 'amazon' || os[:name] == 'redhat' || os[:name] == 'suse' + true + else + false + end +deb = !rpm + +# these files should exist on all systems +%w( + /etc/mongod.conf + /usr/bin/mongod + /var/log/mongodb/mongod.log +).each do |filename| + describe file(filename) do + it { should be_file } + end +end + +if sysvinit + describe file('/etc/init.d/mongod') do + it { should be_file } + it { should be_executable } + end +end + +if systemd + unit_file_prefix = '' + if os[:name] == 'suse' + # Putting systemd unit files in /usr, which may be a separate partition + # and therefore not available during isolated startups, is bad practice. + # But it's what SUSE has chosen to do, so we have to deal with it. + unit_file_prefix = '/usr' + end + describe file("#{unit_file_prefix}/lib/systemd/system/mongod.service") do + it { should be_file } + end +end + +if rpm + %w( + /var/lib/mongo + /var/run/mongodb + ).each do |filename| + describe file(filename) do + it { should be_directory } + end + end + + describe user('mongod') do + it { should exist } + its('groups') { should include 'mongod' } + its('home') { should eq '/var/lib/mongo' } + its('shell') { should eq '/bin/false' } + end +end + +if deb + describe file('/var/lib/mongodb') do + it { should be_directory } + end + + describe user('mongodb') do + it { should exist } + its('groups') { should include 'mongodb' } + # All versions of Debian 10 will use /usr/sbin/nologin for service + # account shells + its('shell') { + if ((os[:name] == 'debian' and os[:release].split('.')[0] >= '10') or + (os[:name] == 'ubuntu' and os[:release] == '18.04') or + (os[:name] == 'ubuntu' and os[:release] == '20.04')) + should eq '/usr/sbin/nologin' + else + should eq '/bin/false' + end + } + end +end + +############################################################ +# This section verifies ulimits. +############################################################ + +ulimits = { + 'Max file size' => 'unlimited', + 'Max cpu time' => 'unlimited', + 'Max address space' => 'unlimited', + 'Max open files' => '64000', + 'Max resident set' => 'unlimited', + 'Max processes' => '64000' +} +ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits' + +ulimits.each do |limit, value| + describe command("#{ulimits_cmd} | grep \"#{limit}\"") do + its('stdout') { should match(/#{limit}\s+#{value}/) } + end +end + +############################################################ +# This section verifies uninstall. +############################################################ + +if rpm + describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do + its('exit_status') { should eq 0 } + end +elsif deb + describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do + its('exit_status') { should eq 0 } + end +end + +# make sure we cleaned up +%w( + /lib/systemd/system/mongod.service + /usr/bin/mongod +).each do |filename| + describe file(filename) do + it { should_not exist } + end +end diff --git a/buildscripts/packager.py b/buildscripts/packager.py index 3aab408008f..d90f0c80489 100755 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -44,7 +44,7 @@ import time ARCH_CHOICES = ["x86_64", "arm64", "aarch64", "s390x"] # Made up names for the flavors of distribution we package for. -DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2", "amazon2023"] +DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2"] class Spec(object): @@ -291,8 +291,6 @@ class Distro(object): return "2013.03" elif self.dname == 'amazon2': return "2017.12" - elif self.dname == 'amazon2023': - return "2023.0" elif self.dname == 'ubuntu': if build_os == 'ubuntu1204': return "precise" @@ -304,8 +302,6 @@ class Distro(object): return "bionic" elif build_os == 'ubuntu2004': return "focal" - elif build_os == 'ubuntu2204': - return "jammy" else: raise Exception("unsupported build_os: %s" % build_os) elif self.dname == 'debian': @@ -345,18 +341,8 @@ class Distro(object): if re.search("(suse)", self.dname): return ["suse11", "suse12", "suse15"] elif re.search("(redhat|fedora|centos)", self.dname): - return [ - "rhel90", - "rhel82", - "rhel80", - "rhel70", - "rhel71", - "rhel72", - "rhel62", - "rhel55", - "rhel67", - ] - elif self.dname in ['amazon', 'amazon2', 'amazon2023']: + return ["rhel82", "rhel80", "rhel70", "rhel71", "rhel72", "rhel62", "rhel55", "rhel67"] + elif self.dname in ['amazon', 'amazon2']: return [self.dname] elif self.dname == 'ubuntu': return [ @@ -365,7 +351,6 @@ class Distro(object): "ubuntu1604", "ubuntu1804", "ubuntu2004", - "ubuntu2204", ] elif self.dname == 'debian': return ["debian81", "debian92", "debian10", "debian11"] @@ -384,8 +369,6 @@ class Distro(object): return 'amzn1' elif self.dname == 'amazon2': return 'amzn2' - elif self.dname == 'amazon2023': - return 'amzn2023' return re.sub(r'^rh(el\d).*$', r'\1', build_os) diff --git a/buildscripts/packager_enterprise.py b/buildscripts/packager_enterprise.py index 3e2ec6addd1..fcc260c86be 100755 --- a/buildscripts/packager_enterprise.py +++ b/buildscripts/packager_enterprise.py @@ -44,7 +44,7 @@ import packager # pylint: disable=wrong-import-position ARCH_CHOICES = ["x86_64", "ppc64le", "s390x", "arm64", "aarch64"] # Made up names for the flavors of distribution we package for. -DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2", "amazon2023"] +DISTROS = ["suse", "debian", "redhat", "ubuntu", "amazon", "amazon2"] class EnterpriseSpec(packager.Spec): @@ -119,7 +119,7 @@ class EnterpriseDistro(packager.Distro): def build_os(self, arch): # pylint: disable=too-many-branches """Return the build os label in the binary package to download. - The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80", "rhel90" are for redhat, + The labels "rhel57", "rhel62", "rhel67", "rhel70", "rhel80" are for redhat, the others are delegated to the super class. """ # pylint: disable=too-many-return-statements @@ -139,18 +139,16 @@ class EnterpriseDistro(packager.Distro): return [] if arch == "arm64": if self.dname == 'ubuntu': - return ["ubuntu1804", "ubuntu2004", "ubuntu2204"] + return ["ubuntu1804", "ubuntu2004"] if arch == "aarch64": if self.dname == 'redhat': - return ["rhel82", "rhel90"] + return ["rhel82"] if self.dname == 'amazon2': return ["amazon2"] - if self.dname == 'amazon2023': - return ["amazon2023"] return [] if re.search("(redhat|fedora|centos)", self.dname): - return ["rhel90", "rhel80", "rhel70", "rhel62", "rhel57"] + return ["rhel80", "rhel70", "rhel62", "rhel57"] return super(EnterpriseDistro, self).build_os(arch) # pylint: enable=too-many-return-statements diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py index 0d7fee88496..44af0fca70b 100644 --- a/buildscripts/patch_builds/change_data.py +++ b/buildscripts/patch_builds/change_data.py @@ -87,7 +87,7 @@ def find_changed_files(repo: Repo, revision_map: Optional[RevisionMap] = None) - work_tree_files = _modified_files_for_diff(diff, LOGGER.bind(diff="working tree diff")) commit = repo.index - diff = commit.diff(revision_map.get(repo.git_dir, repo.head.commit), R=True) + diff = commit.diff(revision_map.get(repo.git_dir, repo.head.commit)) index_files = _modified_files_for_diff(diff, LOGGER.bind(diff="index diff")) untracked_files = set(repo.untracked_files) diff --git a/buildscripts/patch_builds/evg_change_data.py b/buildscripts/patch_builds/evg_change_data.py index 6ee3ebb7ca7..4506defa203 100644 --- a/buildscripts/patch_builds/evg_change_data.py +++ b/buildscripts/patch_builds/evg_change_data.py @@ -2,9 +2,6 @@ from typing import List from git import Repo -from requests import HTTPError -import yaml - from evergreen import EvergreenApi from buildscripts.patch_builds.change_data import RevisionMap, generate_revision_map @@ -20,20 +17,11 @@ def generate_revision_map_from_manifest(repos: List[Repo], task_id: str, :param evg_api: Evergreen API object. :return: Map of repositories to revisions """ - try: - manifest = evg_api.manifest_for_task(task_id) - revisions_data = { - module_name: module.revision - for module_name, module in manifest.modules.items() - } - revisions_data["mongo"] = manifest.revision - return generate_revision_map(repos, revisions_data) - except HTTPError as err: - # Manifest unavailable from Evergreen? Just make a fake one: - if err.response.status_code == 404: - with open("../expansions.yml") as file_handle: - return generate_revision_map(repos, - {"mongo": yaml.safe_load(file_handle)["revision"]}) - # Some other Evergreen error? Raise it up the stack! - else: - raise err + manifest = evg_api.manifest_for_task(task_id) + revisions_data = { + module_name: module.revision + for module_name, module in manifest.modules.items() + } + revisions_data["mongo"] = manifest.revision + + return generate_revision_map(repos, revisions_data) diff --git a/buildscripts/promote_silent_failures.py b/buildscripts/promote_silent_failures.py new file mode 100644 index 00000000000..d8d45872685 --- /dev/null +++ b/buildscripts/promote_silent_failures.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Convert silent test failures into non-silent failures. + +Any test files with at least 2 executions in the report.json file that have a "silentfail" status, +this script will change the outputted report to have a "fail" status instead. +""" + +import collections +import json +import optparse +import os +import sys + +# Get relative imports to work when the package is not installed on the PYTHONPATH. +if __name__ == "__main__" and __package__ is None: + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from buildscripts.resmokelib.testing import report + + +def read_json_file(json_file): + """Return contents of a JSON file.""" + with open(json_file) as json_data: + return json.load(json_data) + + +def main(): + """Execute Main program.""" + + usage = "usage: %prog [options] report.json" + parser = optparse.OptionParser(usage=usage) + parser.add_option( + "-o", "--output-file", dest="outfile", default="-", + help=("If '-', then the report file is written to stdout." + " Any other value is treated as the output file name. By default," + " output is written to stdout.")) + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("Requires a single report.json file.") + + report_file_json = read_json_file(args[0]) + test_report = report.TestReport.from_dict(report_file_json) + + # Count number of "silentfail" per test file. + status_dict = collections.defaultdict(int) + for test_info in test_report.test_infos: + if test_info.evergreen_status == "silentfail": + status_dict[test_info.test_id] += 1 + + # For test files with more than 1 "silentfail", convert status to "fail". + for test_info in test_report.test_infos: + if status_dict[test_info.test_id] >= 2: + test_info.evergreen_status = "fail" + + result_report = test_report.as_dict() + if options.outfile != "-": + with open(options.outfile, "w") as fp: + json.dump(result_report, fp) + else: + print(json.dumps(result_report)) + + +if __name__ == "__main__": + main() diff --git a/buildscripts/resmoke_tests_runtime_validate.py b/buildscripts/resmoke_tests_runtime_validate.py index 5c17d1ee5f8..cffcbf48e9d 100644 --- a/buildscripts/resmoke_tests_runtime_validate.py +++ b/buildscripts/resmoke_tests_runtime_validate.py @@ -3,6 +3,7 @@ import json import sys from collections import namedtuple +from datetime import datetime, timedelta from statistics import mean from typing import Dict, List @@ -12,8 +13,7 @@ import structlog from buildscripts.resmokelib.testing.report import TestInfo, TestReport from buildscripts.resmokelib.utils import get_task_name_without_suffix from buildscripts.util.cmdutils import enable_logging - -from buildscripts.util.teststats import HistoricTaskData, HistoricalTestInformation +from evergreen import RetryingEvergreenApi, TestStats LOGGER = structlog.get_logger("buildscripts.resmoke_tests_runtime_validate") @@ -34,12 +34,17 @@ def parse_resmoke_report(report_file: str) -> List[TestInfo]: return [test_info for test_info in test_report.test_infos if "jstests" in test_info.test_file] -def get_historic_stats(project_id: str, task_name: str, - build_variant: str) -> List[HistoricalTestInformation]: +def get_historic_stats(evg_api_config: str, project_id: str, test_files: List[str], task_name: str, + build_variant: str) -> List[TestStats]: """Get historic test stats.""" + evg_api = RetryingEvergreenApi.get_api(config_file=evg_api_config) + before_date = datetime.today() + after_date = before_date - timedelta(days=LOOK_BACK_NUM_DAYS) base_task_name = get_task_name_without_suffix(task_name, build_variant).replace( BURN_IN_PREFIX, "") - return HistoricTaskData.get_stats_from_s3(project_id, base_task_name, build_variant) + return evg_api.test_stats_by_project(project_id=project_id, after_date=after_date, + before_date=before_date, tests=test_files, + tasks=[base_task_name], variants=[build_variant]) def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]: @@ -58,10 +63,13 @@ def make_stats_map(stats: List[_TestData]) -> Dict[str, List[float]]: @click.command() @click.option("--resmoke-report-file", type=str, required=True, help="Location of resmoke's report JSON file.") +@click.option("--evg-api-config", type=str, required=True, + help="Location of evergreen api configuration.") @click.option("--project-id", type=str, required=True, help="Evergreen project id.") @click.option("--build-variant", type=str, required=True, help="Evergreen build variant name.") @click.option("--task-name", type=str, required=True, help="Evergreen task name.") -def main(resmoke_report_file: str, project_id: str, build_variant: str, task_name: str) -> None: +def main(resmoke_report_file: str, evg_api_config: str, project_id: str, build_variant: str, + task_name: str) -> None: """Compare resmoke tests runtime with historic stats.""" enable_logging(verbose=False) @@ -71,9 +79,10 @@ def main(resmoke_report_file: str, project_id: str, build_variant: str, task_nam for test_info in current_test_infos ]) - historic_stats = get_historic_stats(project_id, task_name, build_variant) + historic_stats = get_historic_stats(evg_api_config, project_id, list(current_stats_map.keys()), + task_name, build_variant) historic_stats_map = make_stats_map([ - _TestData(test_stats.test_name, test_stats.avg_duration_pass) + _TestData(test_stats.test_file, test_stats.avg_duration_pass) for test_stats in historic_stats ]) diff --git a/buildscripts/resmokeconfig/loggers/evg.yml b/buildscripts/resmokeconfig/loggers/evg.yml deleted file mode 100644 index 5774de4b013..00000000000 --- a/buildscripts/resmokeconfig/loggers/evg.yml +++ /dev/null @@ -1,27 +0,0 @@ -logging: - executor: - format: '[%(name)s] %(asctime)s %(message)s' - handlers: - - class: logging.StreamHandler - fixture: - format: '[%(name)s] %(message)s' - handlers: - - class: evergreen - tests: - format: '[%(name)s] %(message)s' - handlers: - - class: evergreen - -shorten_logger_name: - remove: - - MongoDFixture - - ReplicaSetFixture - - ShardedClusterFixture - replace: - primary: prim - secondary: sec - node: n - shard: s - configsvr: c - mongos: s - job: j diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml index 6265d19b9e9..e25ee213769 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml @@ -272,43 +272,23 @@ - "multiversion.replica_sets_multiversion_testdata_last_continuous" -- suite_name: aggregation_multiversion_fuzzer_last_lts +- suite_name: generational_fuzzer_last_lts base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" -- suite_name: aggregation_multiversion_fuzzer_last_continuous - base_suite: generational_fuzzer - overrides: - - "multiversion.replica_sets_multiversion_testdata_last_continuous" - -- suite_name: aggregation_expression_multiversion_fuzzer_last_lts - base_suite: generational_fuzzer - overrides: - - "multiversion.replica_sets_multiversion_testdata_last_lts" - -- suite_name: aggregation_expression_multiversion_fuzzer_last_continuous - base_suite: generational_fuzzer - overrides: - - "multiversion.replica_sets_multiversion_testdata_last_continuous" - -- suite_name: update_fuzzer_last_lts - base_suite: generational_fuzzer - overrides: - - "multiversion.replica_sets_multiversion_testdata_last_lts" - -- suite_name: update_fuzzer_last_continuous +- suite_name: generational_fuzzer_last_continuous base_suite: generational_fuzzer overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" -- suite_name: update_fuzzer_replication_last_lts +- suite_name: generational_fuzzer_replication_last_lts base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_lts" -- suite_name: update_fuzzer_replication_last_continuous +- suite_name: generational_fuzzer_replication_last_continuous base_suite: generational_fuzzer_replication overrides: - "multiversion.replica_sets_multiversion_testdata_last_continuous" diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml index 6590d78359d..2ec2f4fe333 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/replica_sets_stepdown_selector.yml @@ -21,7 +21,6 @@ - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js @@ -54,8 +53,6 @@ # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary. - jstests/core/geo_s2ordering.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - name: reconfig_kill_primary_jscore_passthrough_exclude_files @@ -84,7 +81,6 @@ - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # These test run commands using legacy queries, which are not supported on sessions. - jstests/core/comment_field.js @@ -125,8 +121,6 @@ # Inserts enough data that recovery takes more than 8 seconds, so we never get a working primary. - jstests/core/geo_s2ordering.js - - jstests/fle2/**/*.js - - src/mongo/db/modules/*/jstests/fle2/**/*.js - name: kill_primary_jscore_passthrough_exclude_with_any_tags diff --git a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml index f87c2c710cf..36aeaad2531 100644 --- a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml +++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml @@ -110,24 +110,24 @@ evergreen_buildvariants: platform: rhel70 architecture: x86_64 - - name: rhel8 + - name: rhel80 edition: targeted - platform: rhel8 + platform: rhel80 architecture: x86_64 - - name: enterprise-rhel-8-64-bit + - name: enterprise-rhel-80-64-bit edition: enterprise - platform: rhel8 + platform: rhel80 architecture: x86_64 - - name: rhel-8-arm64 + - name: rhel-82-arm64 edition: targeted - platform: rhel8 + platform: rhel82 architecture: arm64 - - name: enterprise-rhel-8-arm64 + - name: enterprise-rhel-82-arm64 edition: enterprise - platform: rhel8 + platform: rhel82 architecture: arm64 - name: enterprise-rhel-71-ppc64le diff --git a/buildscripts/resmokeconfig/suites/benchmarks.yml b/buildscripts/resmokeconfig/suites/benchmarks.yml index 011b16bd01a..5bbd6e460ef 100644 --- a/buildscripts/resmokeconfig/suites/benchmarks.yml +++ b/buildscripts/resmokeconfig/suites/benchmarks.yml @@ -18,10 +18,6 @@ selector: - build/install/bin/simple8b_bm* # Hash table benchmark is really slow, don't run on evergreen - build/install/bin/hash_table_bm* - - build/install/bin/rate_limiting_bm* - # These benchmarks are only run when modifying or upgrading the immutable library. - - build/install/bin/immutable_absl_comparison_bm* - - build/install/bin/immutable_std_comparison_bm* executor: diff --git a/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml index 607a68dc431..51fc059280b 100644 --- a/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/causally_consistent_hedged_reads_jscore_passthrough.yml @@ -3,6 +3,8 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Don't run these tests as transactions can only run on primaries. - jstests/core/txns/**/*.js @@ -45,7 +47,6 @@ selector: - jstests/core/tailable_cursor_invalidation.js - jstests/core/tailable_getmore_batch_size.js - jstests/core/tailable_skip_limit.js - - jstests/core/timeseries/timeseries_lastpoint_top.js - jstests/core/constructors.js - jstests/core/views/views_all_commands.js - jstests/core/or4.js diff --git a/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml b/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml index 5891ea9d392..cb9f747aebb 100644 --- a/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/clustered_collection_passthrough.yml @@ -65,9 +65,6 @@ selector: # TODO (SERVER-61259): $text not supported: "No query solutions" - jstests/core/fts6.js - jstests/core/fts_projection.js - # Assumes there is one collection that is not clustered. - - jstests/core/find_with_resume_after_param.js - - jstests/core/group_lookup_with_canonical_query_prefix.js exclude_with_any_tags: - assumes_standalone_mongod diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml index 8b84c71d5ec..4fedf2ca12b 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_for_export_import.yml @@ -31,8 +31,6 @@ selector: - uses_write_concern # Sharding is not supported for live exports and imports. - requires_sharding - # Workloads that kill random sessions may interrupt the export/import commands. - - kills_random_sessions executor: config: diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml index 65cb93e7f5d..42bc4a7be4d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_replication_multi_stmt_txn_ubsan.yml @@ -54,7 +54,6 @@ selector: - catches_command_failures # time-series collections do not support write transactions - requires_timeseries - - does_not_support_transactions executor: archive: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml index 7e4e48ed066..83467144c4c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml @@ -9,6 +9,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml index 94b95f17b68..3bfbf5175bd 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml @@ -9,6 +9,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js @@ -133,7 +136,6 @@ executor: enableTestCommands: 1 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index 98ee4120781..18452678c0c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -9,6 +9,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js @@ -110,7 +113,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/remove_multiple_documents.js @@ -233,7 +235,6 @@ executor: enableElectionHandoff: 0 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml index 7f0ce597620..a904a527bc6 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn.yml @@ -13,6 +13,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml index 9c321f265bb..b629f4d1971 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer.yml @@ -13,6 +13,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: @@ -185,7 +188,6 @@ executor: enableTestCommands: 1 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml index 66a8a47b4d6..0d711ca421c 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn.yml @@ -13,6 +13,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml index dfcb47c9199..e3280f20994 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml @@ -18,6 +18,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: @@ -178,7 +181,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_ordered_bulk.js - jstests/concurrency/fsm_workloads/indexed_insert_unordered_bulk.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js # Uses non-retryable commands in the same state function as a command not supported in a # transaction. diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml index ca02cc7d9e8..17bf8eb2ccf 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml @@ -18,6 +18,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: @@ -178,7 +181,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_ordered_bulk.js - jstests/concurrency/fsm_workloads/indexed_insert_unordered_bulk.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js # Uses non-retryable commands in the same state function as a command not supported in a # transaction. diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml index ee68077a3e4..84d7c942484 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_balancer.yml @@ -13,6 +13,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: @@ -189,7 +192,6 @@ executor: enableTestCommands: 1 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml index 609de156c2e..88a29447a8e 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml @@ -18,6 +18,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml index 02d2c081cfb..f2e8e9a06b1 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml @@ -9,6 +9,10 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + - jstests/concurrency/fsm_workloads/map_reduce_drop.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml index 60b70c75a31..0d104100f9d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication_with_balancer.yml @@ -9,6 +9,10 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + - jstests/concurrency/fsm_workloads/map_reduce_drop.js + # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js @@ -130,7 +134,6 @@ executor: enableTestCommands: 1 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index caeb219f6f1..41e0539bbc3 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -9,6 +9,9 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js @@ -110,7 +113,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/remove_multiple_documents.js @@ -228,7 +230,6 @@ executor: enableElectionHandoff: 0 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index 420b427f42a..fe148c981c7 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -9,6 +9,10 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + - jstests/concurrency/fsm_workloads/map_reduce_drop.js + # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded # collections'. This bug is problematic for these workloads because they assert on count() # values: @@ -96,7 +100,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index 9c3635e7b78..f4e9fad96fc 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -9,6 +9,10 @@ selector: - jstests/concurrency/fsm_workloads/distinct_noindex.js - jstests/concurrency/fsm_workloads/distinct_projection.js + # SERVER-17397 Drops of sharded namespaces may not fully succeed + - jstests/concurrency/fsm_workloads/create_database.js + - jstests/concurrency/fsm_workloads/map_reduce_drop.js + # SERVER-14669 Multi-removes that use $where miscount removed documents - jstests/concurrency/fsm_workloads/remove_where.js @@ -102,7 +106,6 @@ selector: - jstests/concurrency/fsm_workloads/indexed_insert_upsert.js - jstests/concurrency/fsm_workloads/indexed_insert_where.js - jstests/concurrency/fsm_workloads/list_indexes.js - - jstests/concurrency/fsm_workloads/query_stats_concurrent.js - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js @@ -225,7 +228,6 @@ executor: enableElectionHandoff: 0 roleGraphInvalidationIsFatal: 1 receiveChunkWaitForRangeDeleterTimeoutMS: 90000 - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 3 num_shards: 2 num_mongos: 2 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous.yml index 3bb21c1440e..4e5f0e97357 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous.yml @@ -40,8 +40,6 @@ selector: - requires_replication - requires_sharding - disabled_due_to_server_61671 - - incompatible_with_concurrency_simultaneous - group_size: 10 group_count_multiplier: 2.5 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml index a12675ac315..e6125107252 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml @@ -32,6 +32,13 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js + # This workload kills random sessions and a different FSM workload wouldn't be able to handle + # the error response from the op being killed. + - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js + - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js + # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -49,14 +56,8 @@ selector: - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js - # This workload involves upgrading and downgrading the FCV, and can cause failure on other - # FSM workloads which depend on features which are only available in the latest FCV. - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - exclude_with_any_tags: - requires_sharding - - kills_random_sessions - - incompatible_with_concurrency_simultaneous group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml index 3d677e5aa39..179d44f40ad 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_cursor_sweeps.yml @@ -32,6 +32,13 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js + # This workload kills random sessions and a different FSM workload wouldn't be able to handle + # the error response from the op being killed. + - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js + - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js + # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -54,14 +61,8 @@ selector: - jstests/concurrency/fsm_workloads/reindex_background.js - jstests/concurrency/fsm_workloads/reindex_writeconflict.js - # This workload involves upgrading and downgrading the FCV, and can cause failure on other - # FSM workloads which depend on features which are only available in the latest FCV. - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - exclude_with_any_tags: - requires_sharding - - kills_random_sessions - - incompatible_with_concurrency_simultaneous group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml index 27b9ded5ff5..3bb3060f1cd 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication_wiredtiger_eviction_debug.yml @@ -32,6 +32,13 @@ selector: # TODO: SERVER-39939. - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js + # This workload kills random sessions and a different FSM workload wouldn't be able to handle + # the error response from the op being killed. + - jstests/concurrency/fsm_workloads/multi_statement_transaction_kill_sessions_atomicity_isolation.js + - jstests/concurrency/fsm_workloads/multi_statement_transaction_simple_kill_sessions.js + - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js + - jstests/concurrency/fsm_workloads/internal_transactions_kill_sessions.js + # This workload may restart running transactions on a different client, causing deadlock if # there is a concurrent dropDatabase waiting for the global X lock. # TODO: SERVER-37876 @@ -55,14 +62,8 @@ selector: - jstests/concurrency/fsm_workloads/reindex.js - jstests/concurrency/fsm_workloads/reindex_background.js - # This workload involves upgrading and downgrading the FCV, and can cause failure on other - # FSM workloads which depend on features which are only available in the latest FCV. - - jstests/concurrency/fsm_workloads/internal_transactions_setFCV.js - exclude_with_any_tags: - requires_sharding - - kills_random_sessions - - incompatible_with_concurrency_simultaneous group_size: 10 group_count_multiplier: 1.0 diff --git a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml index fca4ed4e214..09b0a6f7ece 100755 --- a/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/cst_jscore_passthrough.yml @@ -256,6 +256,9 @@ selector: - jstests/core/index_check6.js - jstests/core/index_check7.js - jstests/core/index_decimal.js + - jstests/core/index_elemmatch1.js + - jstests/core/index_elemmatch2.js + - jstests/core/index_elemmatch2.js - jstests/core/index_filter_commands.js - jstests/core/index_filter_on_hidden_index.js - jstests/core/index_multiple_compatibility.js @@ -307,6 +310,7 @@ selector: - jstests/core/not1.js - jstests/core/not2.js - jstests/core/not3.js + - jstests/core/notablescan.js - jstests/core/null_query_semantics.js - jstests/core/null_query_semantics.js - jstests/core/objectfind.js @@ -504,6 +508,8 @@ selector: - jstests/core/in7.js - jstests/core/index13.js - jstests/core/index_check2.js + - jstests/core/index_elemmatch1.js + - jstests/core/index_elemmatch2.js - jstests/core/indexl.js - jstests/core/json_schema/misc_validation.js - jstests/core/ne_array.js @@ -752,6 +758,7 @@ selector: - jstests/core/geonear_key.js - jstests/core/getmore_invalidated_documents.js - jstests/core/hidden_index.js + - jstests/core/index_elemmatch2.js - jstests/core/index_partial_2dsphere.js - jstests/core/json_schema/misc_validation.js - jstests/core/list_collections_filter.js diff --git a/buildscripts/resmokeconfig/suites/free_monitoring.yml b/buildscripts/resmokeconfig/suites/free_monitoring.yml new file mode 100644 index 00000000000..98fb2867c51 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/free_monitoring.yml @@ -0,0 +1,10 @@ +test_kind: js_test + +selector: + roots: + - jstests/free_mon/*.js + +executor: + config: + shell_options: + nodb: '' diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz.yml b/buildscripts/resmokeconfig/suites/jstestfuzz.yml index 45789c08ab8..f70bfae03e6 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz.yml @@ -13,8 +13,6 @@ executor: crashOnInvalidBSONError: "" objcheck: "" hooks: - - class: RunQueryStats - allow_feature_not_supported: true - class: FuzzerRestoreSettings - class: ValidateCollections shell_options: @@ -28,6 +26,4 @@ executor: set_parameters: disableLogicalSessionCacheRefresh: false enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true verbose: '' diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml index 4a872221272..f1ffdb2f014 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_replication.yml @@ -19,8 +19,6 @@ executor: # Other fuzzers test commands against replica sets with logical session ids. disableImplicitSessions: true hooks: - - class: RunQueryStats - allow_feature_not_supported: true - class: FuzzerRestoreSettings # The CheckReplDBHash hook waits until all operations have replicated to and have been applied # on the secondaries, so we run the ValidateCollections hook after it to ensure we're @@ -44,7 +42,5 @@ executor: enableTestCommands: 1 transactionLifetimeLimitSeconds: 1 writePeriodicNoops: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true verbose: '' num_nodes: 2 diff --git a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml index de2863d100a..7ddb9581092 100644 --- a/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml +++ b/buildscripts/resmokeconfig/suites/jstestfuzz_sharded.yml @@ -18,8 +18,6 @@ executor: crashOnInvalidBSONError: "" objcheck: "" hooks: - - class: RunQueryStats - allow_feature_not_supported: true - class: FuzzerRestoreSettings - class: CheckReplDBHash shell_options: @@ -36,8 +34,6 @@ executor: mongos_options: set_parameters: enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true verbose: '' mongod_options: set_parameters: @@ -45,8 +41,6 @@ executor: enableTestCommands: 1 transactionLifetimeLimitSeconds: 1 writePeriodicNoops: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true verbose: '' num_rs_nodes_per_shard: 1 num_shards: 2 diff --git a/buildscripts/resmokeconfig/suites/mqlrun.yml b/buildscripts/resmokeconfig/suites/mqlrun.yml new file mode 100644 index 00000000000..d9f75ff7697 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/mqlrun.yml @@ -0,0 +1,12 @@ +test_kind: js_test + +selector: + roots: + - src/mongo/db/modules/*/jstests/mqlrun/*.js + +# mqlrun tests don't have a test fixture. Instead, they use the shell to spawn an mqlrun subprocess. +executor: + archive: + config: + shell_options: + nodb: '' diff --git a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml index aacae2ac9d6..b30c65e8e4c 100644 --- a/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml +++ b/buildscripts/resmokeconfig/suites/multi_stmt_txn_jscore_passthrough_with_migration.yml @@ -358,6 +358,5 @@ executor: numInitialSyncAttempts: 1 writePeriodicNoops: true coordinateCommitReturnImmediatelyAfterPersistingDecision: true - skipDroppingHashedShardKeyIndex: true num_rs_nodes_per_shard: 2 num_shards: 2 diff --git a/buildscripts/resmokeconfig/suites/multiversion.yml b/buildscripts/resmokeconfig/suites/multiversion.yml index e865729aac1..674aabc6a65 100644 --- a/buildscripts/resmokeconfig/suites/multiversion.yml +++ b/buildscripts/resmokeconfig/suites/multiversion.yml @@ -31,7 +31,3 @@ executor: config: shell_options: nodb: '' - global_vars: - TestData: - setParameters: - bsonTestValidationVersion: 1 diff --git a/buildscripts/resmokeconfig/suites/multiversion_auth.yml b/buildscripts/resmokeconfig/suites/multiversion_auth.yml index aa77936c5a2..d76515abf50 100644 --- a/buildscripts/resmokeconfig/suites/multiversion_auth.yml +++ b/buildscripts/resmokeconfig/suites/multiversion_auth.yml @@ -37,8 +37,6 @@ executor: shell_options: global_vars: TestData: - setParameters: - bsonTestValidationVersion: 1 auth: true # authMechanism: SCRAM-SHA-256 keyFile: *keyFile diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 1fe41d46475..7ab97e1f2ff 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -1,14 +1,8 @@ test_kind: js_test -description: | - "Passthrough" means running a test against different runtime Cluster - configurations, including topology, runtime flags, fault injections, and other - parameters. Most tests by default are able to run in "passthrough" suites. - NoPassthrough is an exception, where tests here only run in the exact - configuration predefined in the tests themselves. selector: roots: - - jstests/noPassthrough/**/*.js + - jstests/noPassthrough/*.js - src/mongo/db/modules/*/jstests/hot_backups/*.js - src/mongo/db/modules/*/jstests/live_import/*.js - src/mongo/db/modules/*/jstests/no_passthrough/*.js @@ -16,10 +10,8 @@ selector: # Self-tests for the Concurrency testing framework are run as part of this test suite. - jstests/concurrency/*.js exclude_files: - - jstests/noPassthrough/libs/*.js # Disable inmem_full as per SERVER-27014 - jstests/noPassthrough/inmem_full.js - exclude_with_any_tags: # noPassthrough tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/parallel.yml b/buildscripts/resmokeconfig/suites/parallel.yml index c4efc723d56..28d13aa2bbc 100644 --- a/buildscripts/resmokeconfig/suites/parallel.yml +++ b/buildscripts/resmokeconfig/suites/parallel.yml @@ -1,3 +1,10 @@ +# If the failure here is due to a test unexpected being run, +# it may be due to the parallel suite not honoring feature flag tags. +# If you want to skip such tests in parallel suite, +# please add them to the exclusion list at +# https://github.com/mongodb/mongo/blob/eb75b6ccc62f7c8ea26a57c1b5eb96a41809396a/jstests/libs/parallelTester.js#L149. + + test_kind: js_test selector: diff --git a/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml deleted file mode 100644 index 70faf6b60e3..00000000000 --- a/buildscripts/resmokeconfig/suites/query_stats_aggregation_passthrough.yml +++ /dev/null @@ -1,35 +0,0 @@ -test_kind: js_test -description: | - This suite enables the collection of query stats metrics on a mongod server, then runs the tests in - aggregation as normal. This should cause each aggregation to compute a query - shape and query stats key, and record in-memory some metrics like execution time and number of - scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of - each test, once with HMAC application enabled and once without. It doesn't assert anything about - the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. - crashing). - -selector: - roots: - - jstests/aggregation/**/*.js - exclude_with_any_tags: - # Running $queryStats will increment these counters which can screw up some test assertions. - - inspects_command_opcounters - -executor: - archive: - hooks: - - ValidateCollections - hooks: - # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. - # That way the fixture restart can't cause any trouble for the other hooks. - - class: RunQueryStats - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true diff --git a/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml deleted file mode 100644 index 7f5ad0390c1..00000000000 --- a/buildscripts/resmokeconfig/suites/query_stats_mongos_aggregation_passthrough.yml +++ /dev/null @@ -1,59 +0,0 @@ -test_kind: js_test -description: | - This suite enables the collection of query stats metrics on a mongos server, then runs the tests in - core and aggregation as normal. This should cause each aggregation to compute a query - shape and query stats key, and record in-memory some metrics like execution time and number of - scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of - each test, once with HMAC application enabled and once without. It doesn't assert anything about - the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. - crashing). - -selector: - roots: - - jstests/aggregation/**/*.js - exclude_files: - - jstests/aggregation/extras/*.js - - jstests/aggregation/data/*.js - # TODO: Remove when SERVER-23229 is fixed. - - jstests/aggregation/bugs/groupMissing.js - # Mongos does not support runtimeConstants. - - jstests/aggregation/accumulators/internal_js_reduce_with_scope.js - - jstests/aggregation/expressions/internal_js_emit_with_scope.js - # $unionWith explain output does not check whether the collection is sharded in a sharded - # cluster. - - jstests/aggregation/sources/unionWith/unionWith_explain.js - exclude_with_any_tags: - # The following tests start their own ShardingTest or ReplSetTest, respectively. - - requires_sharding - - requires_replication - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # system.profile collection doesn't exist on mongos. - - requires_profiling - # Running $queryStats will increment these counters which can screw up some test assertions. - - inspects_command_opcounters - -executor: - archive: - hooks: - - ValidateCollections - hooks: - # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. - # That way the fixture restart can't cause any trouble for the other hooks. - - class: RunQueryStats - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - mongos_options: - set_parameters: - enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true - mongod_options: - set_parameters: - enableTestCommands: 1 - num_rs_nodes_per_shard: 1 - enable_sharding: - - test diff --git a/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml deleted file mode 100644 index 8721dd2a940..00000000000 --- a/buildscripts/resmokeconfig/suites/query_stats_mongos_passthrough.yml +++ /dev/null @@ -1,77 +0,0 @@ -test_kind: js_test -description: | - This suite enables the collection of query stats metrics on a mongos server, then runs the tests in - core and aggregation as normal. This should cause each query to compute a query - shape and query stats key, and record in-memory some metrics like execution time and number of - scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of - each test, once with HMAC application enabled and once without. It doesn't assert anything about - the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. - crashing). - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - - jstests/core/txns/**/*.js - # The following tests fail because a certain command or functionality is not supported on - # mongos. This command or functionality is placed in a comment next to the failing test. - - jstests/core/**/apitest_db.js # serverStatus output doesn't have storageEngine. - - jstests/core/**/check_shard_index.js # checkShardingIndex. - - jstests/core/**/collection_truncate.js # emptycapped. - - jstests/core/**/compact_keeps_indexes.js # compact. - - jstests/core/**/currentop.js # uses fsync. - - jstests/core/**/dbhash.js # dbhash. - - jstests/core/**/dbhash2.js # dbhash. - - jstests/core/**/fsync.js # uses fsync. - - jstests/core/**/geo_s2cursorlimitskip.js # profiling. - - jstests/core/**/geo_update_btree2.js # notablescan. - - jstests/core/**/index9.js # "local" database. - - jstests/core/**/queryoptimizera.js # "local" database. - - jstests/core/**/stages*.js # stageDebug. - - jstests/core/**/startup_log.js # "local" database. - - jstests/core/**/top.js # top. - # The following tests fail because mongos behaves differently from mongod when testing certain - # functionality. The differences are in a comment next to the failing test. - - jstests/core/**/explain_missing_database.js # Behavior with no db different on mongos. - - jstests/core/**/geo_2d_explain.js # executionSuccess in different spot in explain(). - - jstests/core/**/geo_s2explain.js # inputStage in different spot in explain(). - - jstests/core/**/geo_s2sparse.js # keysPerIndex in different spot in validate(). - - jstests/core/**/operation_latency_histogram.js # Stats are counted differently on mongos, SERVER-24880. - - jstests/core/**/killop_drop_collection.js # Uses fsyncLock. - - jstests/core/**/or_to_in.js # queryPlanner in different spot in explain() - # The following tests fail because of divergent dropCollection behavior between standalones and - # sharded clusters. These tests expect a second drop command to error, whereas in sharded clusters - # we expect a second drop to return status OK. - - jstests/core/**/explain_upsert.js - exclude_with_any_tags: - - assumes_standalone_mongod - - assumes_against_mongod_not_mongos - # system.profile collection doesn't exist on mongos. - - requires_profiling - # Running $queryStats will increment these counters which can screw up some test assertions. - - inspects_command_opcounters - -executor: - archive: - hooks: - - ValidateCollections - hooks: - # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. - # That way the fixture restart can't cause any trouble for the other hooks. - - class: RunQueryStats - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: ShardedClusterFixture - mongos_options: - set_parameters: - enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true - mongod_options: - set_parameters: - enableTestCommands: 1 - num_rs_nodes_per_shard: 1 - enable_sharding: - - test diff --git a/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml b/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml deleted file mode 100644 index c378ed81167..00000000000 --- a/buildscripts/resmokeconfig/suites/query_stats_passthrough.yml +++ /dev/null @@ -1,39 +0,0 @@ -test_kind: js_test -description: | - This suite enables the collection of query stats metrics on a mongod server, then runs the tests in - core and aggregation as normal. This should cause each query to compute a query - shape and query stats key, and record in-memory some metrics like execution time and number of - scanned documents. Then it uses the 'RunQueryStats' hook to collect the query stats at the end of - each test, once with HMAC application enabled and once without. It doesn't assert anything about - the collected query stats, it is just meant to make sure nothing is going seriously awry (e.g. - crashing). - -selector: - roots: - - jstests/core/**/*.js - exclude_files: - # Transactions are not supported on MongoDB standalone nodes, so we do not run these tests. - - jstests/core/txns/**/*.js - - jstests/core/timeseries/timeseries_union_with.js - exclude_with_any_tags: - # Running $queryStats will increment these counters which can screw up some test assertions. - - inspects_command_opcounters - -executor: - archive: - hooks: - - ValidateCollections - hooks: - # Be sure to run the hooks which depend on the fixture being alive before the CleanEveryN hook. - # That way the fixture restart can't cause any trouble for the other hooks. - - class: RunQueryStats - - class: ValidateCollections - - class: CleanEveryN - n: 20 - fixture: - class: MongoDFixture - mongod_options: - set_parameters: - enableTestCommands: 1 - internalQueryStatsRateLimit: -1 - internalQueryStatsErrorsAreCommandFatal: true diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml index 8408255eea1..b6316ac186a 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -41,6 +41,7 @@ selector: - jstests/core/json_schema/json_schema.js - jstests/core/mr_bigobject.js - jstests/core/not2.js + - jstests/core/notablescan.js - jstests/core/null_query_semantics.js - jstests/core/or1.js - jstests/core/or2.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml index d885199b493..bc58d35542c 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough.yml @@ -223,7 +223,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Does not support tojson of command objects. - jstests/core/SERVER-23626.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml index 1b27ca1e5c3..daff4e411c8 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough.yml @@ -221,7 +221,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Does not support tojson of command objects. - jstests/core/SERVER-23626.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml index 769a7daf339..d292c3ae69d 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_reconfig_jscore_stepdown_passthrough.yml @@ -7,6 +7,8 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -31,7 +33,6 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # These test run commands using legacy queries, which are not supported on sessions. - jstests/core/comment_field.js diff --git a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml index 7f91860f3b8..1288ffb0a86 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_terminate_primary_jscore_passthrough.yml @@ -3,6 +3,8 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -22,7 +24,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml index 0ef725251e6..b3d3d510dae 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_downgrade.yml @@ -24,7 +24,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml index dd0743c6a23..3f8a241c4bc 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_passthrough.yml @@ -30,7 +30,6 @@ selector: - jstests/core/find_and_modify_pipeline_update.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js - - jstests/core/project_with_collation.js # These tests rely on the assumption that an update command is run only once. - jstests/core/find_and_modify_metrics.js diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml index b382dde715a..985f095650d 100644 --- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml @@ -3,6 +3,8 @@ test_kind: js_test selector: roots: - jstests/core/**/*.js + - jstests/fle2/**/*.js + - src/mongo/db/modules/*/jstests/fle2/*.js exclude_files: # Transactions do not support retryability of individual operations. # TODO: Remove this once it is supported (SERVER-33952). @@ -23,7 +25,6 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/sasl.yml b/buildscripts/resmokeconfig/suites/sasl.yml index d7888475e08..4d1a49cc95d 100644 --- a/buildscripts/resmokeconfig/suites/sasl.yml +++ b/buildscripts/resmokeconfig/suites/sasl.yml @@ -3,8 +3,6 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/sasl/*.js - exclude_files: - - src/mongo/db/modules/enterprise/jstests/sasl/sasl_plugins.js # sasl tests start their own mongod's. executor: diff --git a/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml b/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml deleted file mode 100644 index 0c1cf6fc6ce..00000000000 --- a/buildscripts/resmokeconfig/suites/sasl_windows_cyrussasl.yml +++ /dev/null @@ -1,11 +0,0 @@ -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/sasl/sasl_plugins.js - -# sasl tests start their own mongod's. -executor: - config: - shell_options: - nodb: "" diff --git a/buildscripts/resmokeconfig/suites/search.yml b/buildscripts/resmokeconfig/suites/search.yml index 4c003db89d1..61ba50f7715 100644 --- a/buildscripts/resmokeconfig/suites/search.yml +++ b/buildscripts/resmokeconfig/suites/search.yml @@ -3,9 +3,6 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/search/*.js - - src/mongo/db/modules/*/jstests/mongot/*.js - exclude_with_any_tags: - - requires_auth executor: config: diff --git a/buildscripts/resmokeconfig/suites/search_auth.yml b/buildscripts/resmokeconfig/suites/search_auth.yml index e73b0474840..edf79042a40 100644 --- a/buildscripts/resmokeconfig/suites/search_auth.yml +++ b/buildscripts/resmokeconfig/suites/search_auth.yml @@ -7,10 +7,6 @@ test_kind: js_test selector: roots: - src/mongo/db/modules/*/jstests/search/*.js - - src/mongo/db/modules/*/jstests/mongot/*.js - exclude_files: - # Skip any tests that run with auth explicitly. - - src/mongo/db/modules/*/jstests/search/auth_list_search_indexes_agg.js executor: config: diff --git a/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml b/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml deleted file mode 100644 index 6b6b0e07bdc..00000000000 --- a/buildscripts/resmokeconfig/suites/search_pinned_connections_auth.yml +++ /dev/null @@ -1,27 +0,0 @@ -config_variables: -- &keyFile jstests/libs/authTestsKey -- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly - -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/search/*.js - - src/mongo/db/modules/*/jstests/mongot/*.js - exclude_files: - # Skip any tests that run with auth explicitly. - - src/mongo/db/modules/*/jstests/search/auth_list_search_indexes_agg.js - -executor: - config: - shell_options: - global_vars: - TestData: - auth: true - authMechanism: SCRAM-SHA-256 - keyFile: *keyFile - keyFileData: *keyFileData - roleGraphInvalidationIsFatal: true - setParameters: - pinTaskExecCursorConns: true - nodb: '' diff --git a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml index 3fab3de86db..6419782211b 100644 --- a/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml +++ b/buildscripts/resmokeconfig/suites/sharded_retryable_writes_downgrade.yml @@ -24,7 +24,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/sharding_auth.yml b/buildscripts/resmokeconfig/suites/sharding_auth.yml index 02e731e6352..964d03ce993 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth.yml @@ -32,7 +32,6 @@ selector: - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 # Runs with auth enabled. - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js - - jstests/sharding/cluster_time_across_add_shard.js # Skip because this suite implicitly authenticates as __system, which allows bypassing user write # blocking. - jstests/sharding/set_user_write_block_mode.js diff --git a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml index a6c193f0e91..630863957be 100644 --- a/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml +++ b/buildscripts/resmokeconfig/suites/sharding_auth_audit.yml @@ -32,7 +32,6 @@ selector: - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 # Runs with auth enabled. - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js - - jstests/sharding/cluster_time_across_add_shard.js # Skip because this suite implicitly authenticates as __system, which allows bypassing user write # blocking. - jstests/sharding/set_user_write_block_mode.js diff --git a/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml new file mode 100644 index 00000000000..0d7391ad637 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/sharding_batched_deletes_passthrough.yml @@ -0,0 +1,37 @@ +# This passthrough runs all sharding JS tests and automatically batches multi-deletes. +test_kind: js_test + +selector: + roots: + - jstests/sharding/**/*.js + exclude_files: + - jstests/sharding/libs/*.js + # TODO (SERVER-64506): these tests use transactions under the hood to modify user roles. + - jstests/sharding/api_params_nontransaction_sharded.js + - jstests/sharding/api_params_nontransaction_unsharded.js + # Expects DELETE stage + - jstests/sharding/query/explain_cmd.js + # TODO: (SERVER-64972): add change stream support for batched deletes. + - jstests/sharding/change_stream_no_orphans.js + + exclude_with_any_tags: + - assumes_standalone_mongod + # TODO (SERVER-64506): make groupOplogEntries WUOW's nestable (e.g. inside multi-doc txns). + - uses_multi_shard_transaction + - uses_prepare_transaction + - uses_transactions + +executor: + archive: + hooks: + - CheckReplDBHashInBackground + - CheckReplDBHash + - ValidateCollections + config: + shell_options: + nodb: '' + global_vars: + TestData: + setParameters: + enableTestCommands: 1 + failpoint.batchDeletesByDefault: "{mode: 'alwaysOn'}" diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml index 095ca918119..20749832032 100644 --- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml +++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml @@ -10,8 +10,6 @@ selector: - jstests/sharding/*[aA]uth*.js - jstests/sharding/query/*[aA]uth*.js - jstests/sharding/change_streams/*[aA]uth*.js - - jstests/sharding/cluster_time_across_add_shard.js - - jstests/sharding/internal_txns/internal_client_restrictions.js - jstests/sharding/internal_txns/non_retryable_writes_during_migration.js - jstests/sharding/internal_txns/retry_on_transient_error_validation.js @@ -31,6 +29,7 @@ selector: - jstests/sharding/addshard1.js - jstests/sharding/addshard2.js - jstests/sharding/autosplit.js + - jstests/sharding/auto_rebalance_parallel.js - jstests/sharding/basic_merge.js - jstests/sharding/count1.js - jstests/sharding/count2.js @@ -52,10 +51,12 @@ selector: - jstests/sharding/limit_push.js - jstests/sharding/merge_with_drop_shard.js - jstests/sharding/merge_with_move_primary.js + - jstests/sharding/migrateBig_balancer.js - jstests/sharding/move_chunk_basic.js - jstests/sharding/movePrimary1.js - jstests/sharding/names.js - jstests/sharding/prefix_shard_key.js + - jstests/sharding/presplit.js - jstests/sharding/query_config.js - jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js - jstests/sharding/remove1.js @@ -63,8 +64,11 @@ selector: - jstests/sharding/shard2.js - jstests/sharding/shard3.js - jstests/sharding/shard_collection_basic.js + - jstests/sharding/shard_existing_coll_chunk_count.js + - jstests/sharding/sharding_balance1.js - jstests/sharding/sharding_balance2.js - jstests/sharding/sharding_balance3.js + - jstests/sharding/sharding_balance4.js - jstests/sharding/sharding_migrate_cursor1.js - jstests/sharding/tag_range.js - jstests/sharding/top_chunk_autosplit.js @@ -230,16 +234,11 @@ selector: # TODO SERVER-51495: Re-enable these tests after reshardCollection is resilient to config server # primary failovers. - - jstests/sharding/**/*reshard*.js + - jstests/sharding/*reshard*.js # SERVER-51805 splitChunk op is not idempotent - jstests/sharding/mongos_get_shard_version.js - # Test will fail if it is unable to lock the config server primary successfully. - - jstests/sharding/fsync_lock_unlock.js - - jstests/sharding/fsync_deadlock.js - - jstests/sharding/fsync_lock_fails_with_in_progress_ddl_op.js - exclude_with_any_tags: - does_not_support_stepdowns diff --git a/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml b/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml index 790d06a5f0c..312f8249245 100644 --- a/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml +++ b/buildscripts/resmokeconfig/suites/simulate_crash_concurrency_replication.yml @@ -30,7 +30,6 @@ executor: config: {} hooks: - class: SimulateCrash - - class: CleanupConcurrencyWorkloads fixture: class: ReplicaSetFixture mongod_options: diff --git a/buildscripts/resmokeconfig/suites/ssl_linear.yml b/buildscripts/resmokeconfig/suites/ssl_linear.yml deleted file mode 100644 index 3c3eacb1f78..00000000000 --- a/buildscripts/resmokeconfig/suites/ssl_linear.yml +++ /dev/null @@ -1,23 +0,0 @@ -test_kind: js_test - -# Similar to the `ssl` suite, but configured by evergreen to run sequentially, so that the -# contents of the Windows system certificate store are only affected by one test at a time. -selector: - roots: - - jstests/ssl_linear/*.js - -# ssl tests start their own mongod's. -executor: - config: - shell_options: - nodb: '' - ssl: '' - sslAllowInvalidCertificates: '' - sslCAFile: jstests/libs/ca.pem - sslPEMKeyFile: jstests/libs/client.pem - global_vars: - TestData: - setParameters: - logComponentVerbosity: "{network:2}" - setParametersMongos: - logComponentVerbosity: "{network:2}" diff --git a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml index 3bc42f1f487..7a1f8d9f47a 100644 --- a/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/talk_directly_to_shardsvrs_kill_primary_jscore_passthrough.yml @@ -34,7 +34,6 @@ selector: - jstests/core/find_and_modify.js - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - - jstests/core/project_with_collation.js - jstests/core/bench_test*.js # benchRun() used for writes - jstests/core/benchrun_pipeline_updates.js # benchRun() used for writes diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml index c6a2225f44f..36433f7ceca 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_kill_primary_jscore_passthrough.yml @@ -43,7 +43,6 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml index 6fd164a5025..fc02fb23677 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_stepdown_jscore_passthrough.yml @@ -43,7 +43,6 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml index 6e139307b27..94d46e70994 100644 --- a/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/tenant_migration_terminate_primary_jscore_passthrough.yml @@ -43,7 +43,6 @@ selector: - jstests/core/find_and_modify2.js - jstests/core/find_and_modify_server6865.js - jstests/core/fts_find_and_modify.js - - jstests/core/project_with_collation.js # Stepdown commands during fsync lock will fail. - jstests/core/currentop.js diff --git a/buildscripts/resmokeconfig/suites/unittests_first_quarter.yml b/buildscripts/resmokeconfig/suites/unittests_first_quarter.yml deleted file mode 100644 index ca8f5af0de0..00000000000 --- a/buildscripts/resmokeconfig/suites/unittests_first_quarter.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_kind: cpp_unit_test - -selector: - root: build/first_quarter_unittests.txt - -executor: - config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_fourth_quarter.yml b/buildscripts/resmokeconfig/suites/unittests_fourth_quarter.yml deleted file mode 100644 index 177672b8726..00000000000 --- a/buildscripts/resmokeconfig/suites/unittests_fourth_quarter.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_kind: cpp_unit_test - -selector: - root: build/fourth_quarter_unittests.txt - -executor: - config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_second_quarter.yml b/buildscripts/resmokeconfig/suites/unittests_second_quarter.yml deleted file mode 100644 index 52a96c1384c..00000000000 --- a/buildscripts/resmokeconfig/suites/unittests_second_quarter.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_kind: cpp_unit_test - -selector: - root: build/second_quarter_unittests.txt - -executor: - config: {} diff --git a/buildscripts/resmokeconfig/suites/unittests_third_quarter.yml b/buildscripts/resmokeconfig/suites/unittests_third_quarter.yml deleted file mode 100644 index 60b997bb403..00000000000 --- a/buildscripts/resmokeconfig/suites/unittests_third_quarter.yml +++ /dev/null @@ -1,7 +0,0 @@ -test_kind: cpp_unit_test - -selector: - root: build/third_quarter_unittests.txt - -executor: - config: {} diff --git a/buildscripts/resmokeconfig/suites/vector_search.yml b/buildscripts/resmokeconfig/suites/vector_search.yml deleted file mode 100644 index effd621be8f..00000000000 --- a/buildscripts/resmokeconfig/suites/vector_search.yml +++ /dev/null @@ -1,12 +0,0 @@ -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/vector_search/*.js - exclude_with_any_tags: - - requires_auth - -executor: - config: - shell_options: - nodb: '' diff --git a/buildscripts/resmokeconfig/suites/vector_search_auth.yml b/buildscripts/resmokeconfig/suites/vector_search_auth.yml deleted file mode 100644 index c025afe3619..00000000000 --- a/buildscripts/resmokeconfig/suites/vector_search_auth.yml +++ /dev/null @@ -1,21 +0,0 @@ -config_variables: -- &keyFile jstests/libs/authTestsKey -- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly - -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/vector_search/*.js - -executor: - config: - shell_options: - global_vars: - TestData: - auth: true - authMechanism: SCRAM-SHA-256 - keyFile: *keyFile - keyFileData: *keyFileData - roleGraphInvalidationIsFatal: true - nodb: '' diff --git a/buildscripts/resmokeconfig/suites/vector_search_ssl.yml b/buildscripts/resmokeconfig/suites/vector_search_ssl.yml deleted file mode 100644 index e234ea490fd..00000000000 --- a/buildscripts/resmokeconfig/suites/vector_search_ssl.yml +++ /dev/null @@ -1,15 +0,0 @@ -test_kind: js_test - -selector: - roots: - - src/mongo/db/modules/*/jstests/vector_search/ssl/*.js - -executor: - config: - shell_options: - nodb: '' - ssl: '' - tlsAllowInvalidHostnames: '' - tlsAllowInvalidCertificates: '' - tlsCAFile: jstests/libs/ca.pem - tlsCertificateKeyFile: jstests/libs/client.pem diff --git a/buildscripts/resmokelib/cli.py b/buildscripts/resmokelib/cli.py index 3dc6a0a522d..2cb85721e9e 100644 --- a/buildscripts/resmokelib/cli.py +++ b/buildscripts/resmokelib/cli.py @@ -18,6 +18,6 @@ def main(argv): usage="Resmoke is MongoDB's correctness testing orchestrator.\n" "For more information, see the help message for each subcommand.\n" "For example: resmoke.py run -h\n" - "Note: bisect, setup-multiversion and symbolize subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" + "Note: bisect and setup-multiversion subcommands have been moved to db-contrib-tool (https://github.com/10gen/db-contrib-tool#readme).\n" ) subcommand.execute() diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index f3ab984756c..c1cc1048585 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -55,7 +55,7 @@ DEFAULTS = { "archive_limit_tests": 10, "base_port": 20000, "backup_on_restart_dir": None, - "buildlogger_url": "https://logkeeper2.build.10gen.cc", + "buildlogger_url": "https://logkeeper.mongodb.org", "continue_on_failure": False, "dbpath_prefix": None, "dbtest_executable": None, @@ -88,11 +88,11 @@ DEFAULTS = { "repeat_tests_min": None, "repeat_tests_secs": None, "replay_file": None, + "report_failure_status": "fail", "report_file": None, "run_all_feature_flag_tests": False, - "run_no_feature_flag_tests": False, + "run_all_feature_flags_no_tests": False, "additional_feature_flags": None, - "additional_feature_flags_file": None, "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, @@ -101,7 +101,6 @@ DEFAULTS = { "stagger_jobs": None, "majority_read_concern": "on", "storage_engine": "wiredTiger", - "enable_enterprise_tests": "on", "storage_engine_cache_size_gb": None, "suite_files": "with_server", "tag_files": [], @@ -132,7 +131,6 @@ DEFAULTS = { "task_doc": None, "variant_name": None, "version_id": None, - "work_dir": None, # WiredTiger options. "wt_coll_config": None, @@ -168,6 +166,7 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "num_repeat_tests_max", "num_repeat_tests_min", "time_repeat_tests_secs", + "report_failure_status", ]) @@ -234,6 +233,7 @@ class SuiteOptions(_SuiteOptions): REPEAT_TESTS_MAX, REPEAT_TESTS_MIN, REPEAT_TESTS_SECS, + REPORT_FAILURE_STATUS, ]))) options = self._asdict() @@ -295,9 +295,6 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None -# if set, enables enterprise jstest to automatically be included -ENABLE_ENTERPRISE_TESTS = None - # URL to connect to the Evergreen service. EVERGREEN_URL = None @@ -340,12 +337,6 @@ EVERGREEN_VARIANT_NAME = None # the commit hash. EVERGREEN_VERSION_ID = None -# The Evergreen task's working directory. -EVERGREEN_WORK_DIR = None - -# Path to evergreen project configuration yaml file -EVERGREEN_PROJECT_CONFIG_PATH = None - # If set, then any jstests that have any of the specified tags will be excluded from the suite(s). EXCLUDE_WITH_ANY_TAGS = None @@ -384,11 +375,8 @@ INSTALL_DIR = None # Whether to run tests for feature flags. RUN_ALL_FEATURE_FLAG_TESTS = None -# Whether to run the tests with enabled feature flags -RUN_NO_FEATURE_FLAG_TESTS = None - -# the path to a file containing feature flags -ADDITIONAL_FEATURE_FLAGS_FILE = None +# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true. +RUN_ALL_FEATURE_FLAGS = None # List of enabled feature flags. ENABLED_FEATURE_FLAGS = [] @@ -445,6 +433,9 @@ REPEAT_TESTS_MIN = None # If set, then each test is repeated the specified time (seconds) inside the suites. REPEAT_TESTS_SECS = None +# Controls if the test failure status should be reported as failed or be silently ignored. +REPORT_FAILURE_STATUS = None + # If set, then resmoke.py will write out a report file with the status of each test that ran. REPORT_FILE = None @@ -565,17 +556,12 @@ DEFAULT_BENCHMARK_TEST_LIST = "build/benchmarks.txt" DEFAULT_UNIT_TEST_LIST = "build/unittests.txt" DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt" DEFAULT_LIBFUZZER_TEST_LIST = "build/libfuzzer_tests.txt" -DEFAULT_PRETTY_PRINTER_TEST_LIST = "build/pretty_printer_tests.txt" -SPLIT_UNITTESTS_LISTS = [ - f"build/{test_group}_quarter_unittests.txt" - for test_group in ['first', 'second', 'third', 'fourth'] -] + # External files or executables, used as suite selectors, that are created during the build and # therefore might not be available when creating a test membership map. EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE, - DEFAULT_LIBFUZZER_TEST_LIST, DEFAULT_PRETTY_PRINTER_TEST_LIST, - *SPLIT_UNITTESTS_LISTS) + DEFAULT_LIBFUZZER_TEST_LIST) # Where to look for logging and suite configuration files CONFIG_DIR = None @@ -590,12 +576,3 @@ SHORTEN_LOGGER_NAME_CONFIG: dict = {} # that get loaded for multiversion tests can behave correctly on master and on v5.0; the latter # case runs 5.0 and 4.4 binaries and has this value set to True. Can be removed after 6.0. USE_LEGACY_MULTIVERSION = True - -# Expansions file location -# in CI, the expansions file is located in the ${workdir}, one dir up -# from src, the checkout directory -EXPANSIONS_FILE = "../expansions.yml" if 'CI' in os.environ else "expansions.yml" - -# Symbolizer secrets -SYMBOLIZER_CLIENT_SECRET = None -SYMBOLIZER_CLIENT_ID = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index ed659d2f79b..db1319b8343 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -1,6 +1,5 @@ """Configure the command line input for the resmoke 'run' subcommand.""" -import argparse import collections import configparser import datetime @@ -16,21 +15,18 @@ import shlex import pymongo.uri_parser -from buildscripts.idl import gen_all_feature_flag_list from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils from buildscripts.resmokelib import mongod_fuzzer_configs from buildscripts.resmokelib.suitesconfig import SuiteFinder -from buildscripts.util.read_config import read_config_file def validate_and_update_config(parser, args): """Validate inputs and update config module.""" _validate_options(parser, args) _update_config_vars(args) - _update_symbolizer_secrets() _validate_config(parser) _set_logging_config() @@ -55,9 +51,14 @@ def _validate_options(parser, args): "Cannot use --replayFile with additional test files listed on the command line invocation." ) - if args.additional_feature_flags_file and not os.path.isfile( - args.additional_feature_flags_file): - parser.error("The specified additional feature flags file does not exist.") + if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests: + if not os.path.isfile(ALL_FEATURE_FLAG_FILE): + parser.error( + "To run tests with all feature flags, the %s file must exist and be placed in" + " your working directory. The file can be downloaded from the artifacts tarball" + " in Evergreen. Alternatively, if you know which feature flags you want to enable," + " you can use the --additionalFeatureFlags command line argument" % + ALL_FEATURE_FLAG_FILE) def get_set_param_errors(process_params): agg_set_params = collections.defaultdict(list) @@ -183,36 +184,28 @@ be invoked as either: - buildscripts/resmoke.py --installDir {shlex.quote(user_config['install_dir'])}""") raise RuntimeError(err) - def process_feature_flag_file(path): - with open(path) as fd: - return fd.read().split() - def setup_feature_flags(): _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests") - _config.RUN_NO_FEATURE_FLAG_TESTS = config.pop("run_no_feature_flag_tests") - _config.ADDITIONAL_FEATURE_FLAGS_FILE = config.pop("additional_feature_flags_file") + _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests") + # Running all feature flag tests implies running the fixtures with feature flags. if _config.RUN_ALL_FEATURE_FLAG_TESTS: - print("Generating: ", ALL_FEATURE_FLAG_FILE) - gen_all_feature_flag_list.gen_all_feature_flags_file() + _config.RUN_ALL_FEATURE_FLAGS = True all_ff = [] enabled_feature_flags = [] try: - all_ff = process_feature_flag_file(ALL_FEATURE_FLAG_FILE) + with open(ALL_FEATURE_FLAG_FILE) as fd: + all_ff = fd.read().split() except FileNotFoundError: # If we ask resmoke to run with all feature flags, the feature flags file # needs to exist. - if _config.RUN_ALL_FEATURE_FLAG_TESTS or _config.RUN_NO_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS: raise - if _config.RUN_ALL_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS: enabled_feature_flags = all_ff[:] - if _config.ADDITIONAL_FEATURE_FLAGS_FILE: - enabled_feature_flags.extend( - process_feature_flag_file(_config.ADDITIONAL_FEATURE_FLAGS_FILE)) - # Specify additional feature flags from the command line. # Set running all feature flag tests to True if this options is specified. additional_feature_flags = _tags_from_list(config.pop("additional_feature_flags")) @@ -236,7 +229,7 @@ be invoked as either: _config.EXCLUDE_WITH_ANY_TAGS.extend( utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - if _config.RUN_NO_FEATURE_FLAG_TESTS: + if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS: # Don't run any feature flag tests. _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags) else: @@ -254,7 +247,6 @@ be invoked as either: _config.JOBS = config.pop("jobs") _config.LINEAR_CHAIN = config.pop("linear_chain") == "on" _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on" - _config.ENABLE_ENTERPRISE_TESTS = config.pop("enable_enterprise_tests") _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions") if _config.MIXED_BIN_VERSIONS is not None: _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-") @@ -310,8 +302,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG, _config.WT_COLL_CONFIG, \ _config.WT_INDEX_CONFIG = mongod_fuzzer_configs.fuzz_set_parameters( _config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS) - _config.EXCLUDE_WITH_ANY_TAGS.extend(["uses_compact"]) - _config.EXCLUDE_WITH_ANY_TAGS.extend(["requires_emptycapped"]) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) mongos_set_parameters = config.pop("mongos_set_parameters") @@ -332,6 +322,7 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") + _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") _config.EXPORT_MONGOD_CONFIG = config.pop("export_mongod_config") @@ -362,7 +353,6 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py _config.EVERGREEN_TASK_DOC = config.pop("task_doc") _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") _config.EVERGREEN_VERSION_ID = config.pop("version_id") - _config.EVERGREEN_WORK_DIR = config.pop("work_dir") # Archival options. Archival is enabled only when running on evergreen. if not _config.EVERGREEN_TASK_ID: @@ -512,36 +502,3 @@ def _tags_from_list(tags_list): tags.extend([t for t in tag.split(",") if t != ""]) return tags return None - - -def _update_symbolizer_secrets(): - """Open `expansions.yml`, get values for symbolizer secrets and update their values inside config.py .""" - if not _config.EVERGREEN_TASK_ID: - # not running on Evergreen - return - yml_data = utils.load_yaml_file(_config.EXPANSIONS_FILE) - _config.SYMBOLIZER_CLIENT_SECRET = yml_data.get("symbolizer_client_secret") - _config.SYMBOLIZER_CLIENT_ID = yml_data.get("symbolizer_client_id") - - -def detect_evergreen_config(parsed_args: argparse.Namespace, - expansions_file: str = "../expansions.yml"): - """Detect evergreen expansions.""" - if not os.path.exists(expansions_file): - return - - expansions = read_config_file(expansions_file) - - parsed_args.build_id = expansions.get("build_id", None) - parsed_args.distro_id = expansions.get("distro_id", None) - parsed_args.execution_number = expansions.get("execution", None) - parsed_args.project_name = expansions.get("project", None) - parsed_args.git_revision = expansions.get("revision", None) - parsed_args.revision_order_id = expansions.get("revision_order_id", None) - parsed_args.task_id = expansions.get("task_id", None) - parsed_args.task_name = expansions.get("task_name", None) - parsed_args.variant_name = expansions.get("build_variant", None) - parsed_args.version_id = expansions.get("version_id", None) - parsed_args.work_dir = expansions.get("workdir", None) - parsed_args.evg_project_config_path = expansions.get("evergreen_config_file_path", None) - parsed_args.requester = expansions.get("requester", None) diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index fa9a2e98ebd..19354d4c314 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -6,9 +6,7 @@ Handles all the nitty-gritty parameter conversion. import json import os import os.path -import re import stat -from packaging import version from buildscripts.resmokelib import config from buildscripts.resmokelib import utils @@ -45,31 +43,6 @@ def get_path_env_var(env_vars): return path -def get_binary_version(executable): - """Return the string for the binary version of the given executable.""" - - # pylint: disable=wrong-import-position - from buildscripts.resmokelib.multiversionconstants import LATEST_FCV - - split_executable = executable.split("-") - version_regex = re.compile(version.VERSION_PATTERN, re.VERBOSE | re.IGNORECASE) - if len(split_executable) > 1 and version_regex.match(split_executable[-1]): - return split_executable[-1] - return LATEST_FCV - - -def remove_set_parameter_if_before_version(set_parameters, parameter_name, bin_version, - required_bin_version): - """ - Used for removing a server parameter that does not exist prior to a specified version. - - Remove 'parameter_name' from the 'set_parameters' dictionary if 'bin_version' is older than - 'required_bin_version'. - """ - if version.parse(bin_version) < version.parse(required_bin_version): - set_parameters.pop(parameter_name, None) - - def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): """ Return a Process instance that starts mongod arguments constructed from 'mongod_options'. @@ -81,17 +54,12 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): @param mongod_options - A HistoryDict describing the various options to pass to the mongod. """ - bin_version = get_binary_version(executable) args = [executable] mongod_options = mongod_options.copy() if "port" not in mongod_options: mongod_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongod_options.get("set_parameters", {}) - remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", - bin_version, "6.0") - remove_set_parameter_if_before_version( - suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongod_options.pop("set_parameters") @@ -110,7 +78,6 @@ def mongod_program(logger, job_num, executable, process_kwargs, mongod_options): def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos_options=None): # pylint: disable=too-many-arguments """Return a Process instance that starts a mongos with arguments constructed from 'kwargs'.""" - bin_version = get_binary_version(executable) args = [executable] mongos_options = mongos_options.copy() @@ -118,10 +85,6 @@ def mongos_program(logger, job_num, executable=None, process_kwargs=None, mongos if "port" not in mongos_options: mongos_options["port"] = network.PortAllocator.next_fixture_port(job_num) suite_set_parameters = mongos_options.get("set_parameters", {}) - remove_set_parameter_if_before_version(suite_set_parameters, "internalQueryStatsRateLimit", - bin_version, "6.0") - remove_set_parameter_if_before_version( - suite_set_parameters, "internalQueryStatsErrorsAreCommandFatal", bin_version, "6.0") _apply_set_parameters(args, suite_set_parameters) mongos_options.pop("set_parameters") @@ -288,11 +251,6 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches # Load a callback to check that all orphans are deleted before shutting down a ShardingTest. eval_sb.append("load('jstests/libs/override_methods/check_orphans_are_deleted.js');") - if config.FUZZ_MONGOD_CONFIGS is not None and config.FUZZ_MONGOD_CONFIGS is not False: - # Prevent commands from running with the config fuzzer. - eval_sb.append( - "load('jstests/libs/override_methods/config_fuzzer_incompatible_commands.js');") - # Load this file to retry operations that fail due to in-progress background operations. eval_sb.append( "load('jstests/libs/override_methods/implicitly_retry_on_background_op_in_progress.js');") diff --git a/buildscripts/resmokelib/flags.py b/buildscripts/resmokelib/flags.py deleted file mode 100644 index 6aff6961666..00000000000 --- a/buildscripts/resmokelib/flags.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Global flags used by resmoke.""" - -import threading - -HANG_ANALYZER_CALLED = threading.Event() diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 97788b0469b..e5112b870b3 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -5,14 +5,12 @@ import logging import os import sys import tempfile -from datetime import datetime from abc import ABCMeta, abstractmethod from collections import namedtuple from distutils import spawn # pylint: disable=no-name-in-module from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program from buildscripts.resmokelib.hang_analyzer.process_list import Pinfo -from buildscripts.resmokelib import config as resmoke_config Dumpers = namedtuple('Dumpers', ['dbg', 'jstack']) @@ -332,20 +330,6 @@ class LLDBDumper(Dumper): class GDBDumper(Dumper): """GDBDumper class.""" - def __init__(self, root_logger: logging.Logger, dbg_output: str, - timeout_seconds_for_gdb_process=720): - """Initialize GDBDumper.""" - if resmoke_config.EVERGREEN_TASK_ID is None: - # Set 24 hours time out for hang analyzer being run in locally - timeout_seconds_for_gdb_process = 86400 - #Timeout for hang analyzer, default timeout is 12mins(out of total 15mins) in Evergreen - self._timeout_seconds_for_gdb_process = timeout_seconds_for_gdb_process - super().__init__(root_logger, dbg_output) - - def _reduce_timeout_for_gdb_process(self, timeout_period: int): - """Reduce timeout for remaining gdb processes.""" - self._timeout_seconds_for_gdb_process -= timeout_period - def _find_debugger(self, debugger): """Find the installed debugger.""" return find_program(debugger, ['/opt/mongodbtoolchain/v3/bin', '/usr/bin']) @@ -461,19 +445,12 @@ class GDBDumper(Dumper): debugger = "gdb" dbg = self._find_debugger(debugger) logger = _get_process_logger(self._dbg_output, pinfo.name) - _start_time = datetime.now() if dbg is None: self._root_logger.warning("Debugger %s not found, skipping dumping of %s", debugger, str(pinfo.pidv)) return - if self._timeout_seconds_for_gdb_process <= 0: - self._root_logger.warning( - "Skipping dumping of %s processes with PIDs %s because the time limit expired", - pinfo.name, str(pinfo.pidv)) - return - self._root_logger.info("Debugger %s, analyzing %s processes with PIDs %s", dbg, pinfo.name, str(pinfo.pidv)) @@ -481,20 +458,9 @@ class GDBDumper(Dumper): cmds = self._prefix() + self._process_specific(pinfo, take_dump, logger) + self._postfix() - # gcore is both a command within GDB and a script packaged alongside gdb. The gcore script - # invokes the gdb binary with --readnever to avoid spending time loading the debug symbols - # prior to taking the core dump. The debug symbols are unneeded to generate the core dump. - # - # For reference - # https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/gcore.in;h=34860de630cf0ee766e102eb82f7a3fddba6b368#l101 - skip_reading_symbols_on_take_dump = ["--readnever"] if take_dump else [] - - call([dbg, "--quiet", "--nx"] + skip_reading_symbols_on_take_dump + list( - itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger, - self._timeout_seconds_for_gdb_process, pinfo) + call([dbg, "--quiet", "--nx"] + list( + itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) - time_period = (datetime.now() - _start_time).total_seconds() - self._reduce_timeout_for_gdb_process(time_period) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, str(pinfo.pidv)) diff --git a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py index ba259955444..44d0b23e54e 100755 --- a/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py +++ b/buildscripts/resmokelib/hang_analyzer/hang_analyzer.py @@ -68,13 +68,7 @@ class HangAnalyzer(Subcommand): self._configure_processes() self._setup_logging(logger) - def kill_rogue_processes(self): - """Kill any processes that are currently being analyzed.""" - processes = process_list.get_processes(self.process_ids, self.interesting_processes, - self.options.process_match, self.root_logger) - process.teardown_processes(self.root_logger, processes, dump_pids={}) - - def execute(self): # pylint: disable=too-many-branches + def execute(self): # pylint: disable=too-many-branches,too-many-locals,too-many-statements """ Execute hang analysis. diff --git a/buildscripts/resmokelib/hang_analyzer/process.py b/buildscripts/resmokelib/hang_analyzer/process.py index ee528aa6de6..629f78a2594 100644 --- a/buildscripts/resmokelib/hang_analyzer/process.py +++ b/buildscripts/resmokelib/hang_analyzer/process.py @@ -22,7 +22,7 @@ if _IS_WINDOWS: PROCS_TIMEOUT_SECS = 60 -def call(args, logger, timeout_seconds=None, pinfo=None): +def call(args, logger): """Call subprocess on args list.""" logger.info(str(args)) @@ -31,16 +31,7 @@ def call(args, logger, timeout_seconds=None, pinfo=None): logger_pipe = core.pipe.LoggerPipe(logger, logging.INFO, process.stdout) logger_pipe.wait_until_started() - try: - ret = process.wait(timeout=timeout_seconds) - except subprocess.TimeoutExpired: - logger.error("Killing %s processes with PIDs %s because time limit expired", pinfo.name, - str(pinfo.pidv)) - process.kill() - process.wait() - logger_pipe.wait_until_finished() - return - + ret = process.wait() logger_pipe.wait_until_finished() if ret != 0: diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index 8aa6b6bd983..6325ecabb89 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -14,7 +14,6 @@ CREATE_BUILD_ENDPOINT = "/build" APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" -PARSLEY_LOGS_URL = "https://parsley.mongodb.com/resmoke/%(build_id)s/test/%(test_id)s" _BUILDLOGGER_CONFIG = os.getenv("BUILDLOGGER_CREDENTIALS", "mci.buildlogger") @@ -299,7 +298,6 @@ class BuildloggerServer(object): "builder": builder, "buildnum": build_num, "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -318,7 +316,6 @@ class BuildloggerServer(object): "command": test_command, "phase": self.config.get("build_phase", "unknown"), "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -344,8 +341,3 @@ class BuildloggerServer(object): base_url = _config.BUILDLOGGER_URL.rstrip("/") endpoint = APPEND_TEST_LOGS_ENDPOINT % {"build_id": build_id, "test_id": test_id} return "%s/%s" % (base_url, endpoint.strip("/")) - - @staticmethod - def get_parsley_log_url(build_id, test_id): - """Return the parsley log URL.""" - return PARSLEY_LOGS_URL % {"build_id": build_id, "test_id": test_id} diff --git a/buildscripts/resmokelib/logging/formatters.py b/buildscripts/resmokelib/logging/formatters.py index b4ee2abd4b5..2998756a27e 100644 --- a/buildscripts/resmokelib/logging/formatters.py +++ b/buildscripts/resmokelib/logging/formatters.py @@ -19,17 +19,3 @@ class TimestampFormatter(logging.Formatter): formatted_time = time.strftime("%H:%M:%S", converted_time) return "%s.%03dZ" % (formatted_time, record.msecs) - - -class EvergreenLogFormatter(logging.Formatter): - """Log line formatter for Evergreen log messages. - - See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` - for more info. - """ - - def format(self, record): - """Return formatted line.""" - ts = int(record.created * 1e9) - - return "\n".join([f"{ts} {line}" for line in super().format(record).split("\n")]) diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 07bc214517a..692b4532c47 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -161,29 +161,6 @@ class BufferedHandler(logging.Handler): logging.Handler.close(self) -class BufferedFileHandler(BufferedHandler): - """File handler with in-memory buffering.""" - - def __init__(self, filename, capacity=2000, interval_secs=600): - """Initialize the handler with the filename and buffer capacity and flush interval.""" - super().__init__(capacity, interval_secs) - self.file = open(filename, "a", encoding="utf-8") - - def process_record(self, record): - """Return the formatted record message appended with a newline.""" - return self.format(record) + "\n" - - def _flush_buffer_with_lock(self, buf, close_called): - """Write the buffered log lines to the destination file.""" - self.file.writelines(buf) - - def close(self): - """Close the handler and the file descriptor.""" - super().close() - - self.file.close() - - class HTTPHandler(object): """A class which sends data to a web server using POST requests.""" diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index 8775c830669..6d5fee06912 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -1,19 +1,16 @@ """Module to hold the logger instances themselves.""" import logging -import os import re import shutil import subprocess import sys -import yaml from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import redirect as redirect_lib from buildscripts.resmokelib.logging import buildlogger from buildscripts.resmokelib.logging import formatters -from buildscripts.resmokelib.logging.handlers import BufferedFileHandler _DEFAULT_FORMAT = "[%(name)s] %(message)s" @@ -97,13 +94,6 @@ def configure_loggers(): global ROOT_EXECUTOR_LOGGER # pylint: disable=global-statement ROOT_EXECUTOR_LOGGER = new_root_logger(EXECUTOR_LOGGER_NAME) - _write_evergreen_log_spec() - - -def get_evergreen_log_name(job_num, test_id=None): - """Return the log name, relative to the reserved test log directory, on the Evergreen task host.""" - return f"job{job_num}/" + (f"{test_id}.log" if test_id else "global.log") - def new_root_logger(name): """ @@ -180,7 +170,6 @@ def new_fixture_logger(fixture_class, job_num): logger = FixtureLogger(_shorten(full_name), full_name) logger.parent = ROOT_FIXTURE_LOGGER _add_build_logger_handler(logger, job_num) - _add_evergreen_handler(logger, job_num) _FIXTURE_LOGGER_REGISTRY[job_num] = logger return logger @@ -210,7 +199,6 @@ def new_test_logger(test_shortname, test_basename, command, parent, job_num, tes name = "%s:%s" % (parent.name, test_shortname) logger = logging.Logger(name) logger.parent = parent - _add_evergreen_handler(logger, job_num, test_id) def _get_test_endpoint(job_num, test_basename, command, meta_logger): """Get a new test endpoint for the buildlogger server.""" @@ -268,8 +256,6 @@ def _add_handler(logger, handler_info, formatter): handler = logging.StreamHandler(sys.stdout) elif handler_class == "buildlogger": return # Buildlogger handlers are applied when creating specific child loggers - elif handler_class == "evergreen": - return # Evergreen handlers are applied when creating specific child loggers else: raise ValueError("Unknown handler class '%s'" % handler_class) handler.setFormatter(formatter) @@ -339,55 +325,3 @@ def _shorten(logger_name): logger_name = re.sub(r"(^[:_]+|[:_]+$)", "", logger_name) return logger_name - - -# Utility functions for Evergreen file system logging. -# See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` -# for more information. - - -def _write_evergreen_log_spec(): - """Configure file system logging for Evergreen tasks.""" - if not config.EVERGREEN_WORK_DIR: - return - - fp = f"{_get_evergreen_log_dirname()}/log_spec.yaml" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - ROOT_EXECUTOR_LOGGER.info("Writing Evergreen test log spec to %s.", fp) - - log_spec = { - "version": 0, - "format": "text-timestamp", - } - with open(fp, "w") as fd: - yaml.dump(log_spec, fd) - - -def _add_evergreen_handler(logger, job_num, test_id=None): - """Add a new evergreen handler to a logger.""" - logger_info = config.LOGGING_CONFIG[TESTS_LOGGER_NAME] - evergreen_handler_info = None - for handler_info in logger_info["handlers"]: - if handler_info["class"] == "evergreen": - evergreen_handler_info = handler_info - break - - if evergreen_handler_info: - fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - handler = BufferedFileHandler(fp) - handler.setFormatter( - formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT))) - logger.addHandler(handler) - - if test_id: - ROOT_EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp) - else: - ROOT_EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp) - - -def _get_evergreen_log_dirname(): - """Return the reserved directory for test logs on the Evergreen task host.""" - return f"{config.EVERGREEN_WORK_DIR}/build/TestLogs" diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongod_fuzzer_configs.py index 874bc7764cf..ec84d6c5a4e 100644 --- a/buildscripts/resmokelib/mongod_fuzzer_configs.py +++ b/buildscripts/resmokelib/mongod_fuzzer_configs.py @@ -11,16 +11,13 @@ def generate_eviction_configs(rng): eviction_trigger = rng.randint(eviction_target + 1, 99) # Fuzz eviction_dirty_target and trigger both as relative and absolute values - target_bytes_min = 50 * 1024 * 1024 # 50MB # 5% of 1GB default cache size on Evergreen + target_bytes_min = 10 * 1024 * 1024 # 10MB target_bytes_max = 256 * 1024 * 1024 # 256MB # 1GB default cache size on Evergreen eviction_dirty_target = rng.choice( [rng.randint(5, 50), rng.randint(target_bytes_min, target_bytes_max)]) trigger_max = 75 if eviction_dirty_target <= 50 else target_bytes_max eviction_dirty_trigger = rng.randint(eviction_dirty_target + 1, trigger_max) - assert eviction_dirty_trigger > eviction_dirty_target - assert eviction_dirty_trigger <= trigger_max - close_idle_time_secs = rng.randint(1, 100) close_handle_minimum = rng.randint(0, 1000) close_scan_interval = rng.randint(1, 100) @@ -92,8 +89,6 @@ def generate_independent_parameters(rng): # The old retryable writes format is used by other variants. Weight towards turning on the # new retryable writes format on in this one. ret["storeFindAndModifyImagesInSideCollection"] = True - # TODO (SERVER-75632): Uncomment this to enable passthrough testing. - # ret["lockCodeSegmentsInMemory"] = rng.choice([True, False]) return ret diff --git a/buildscripts/resmokelib/multiversion/__init__.py b/buildscripts/resmokelib/multiversion/__init__.py index 7160d1ff026..2c9bae45766 100644 --- a/buildscripts/resmokelib/multiversion/__init__.py +++ b/buildscripts/resmokelib/multiversion/__init__.py @@ -1,5 +1,4 @@ """Subcommand for multiversion config.""" -import argparse from typing import List, Optional import yaml @@ -37,19 +36,10 @@ class MultiversionConfig(BaseModel): class MultiversionConfigSubcommand(Subcommand): """Subcommand for discovering multiversion configuration.""" - def __init__(self, options: argparse.Namespace) -> None: - """Initialize the class.""" - self.config_file_output = options.config_file_output - def execute(self): """Execute the subcommand.""" mv_config = self.determine_multiversion_config() - yaml_output = yaml.safe_dump(mv_config.dict()) - print(yaml_output) - - if self.config_file_output: - with open(self.config_file_output, "w") as file: - file.write(yaml_output) + print(yaml.safe_dump(mv_config.dict())) @staticmethod def determine_multiversion_config() -> MultiversionConfig: @@ -59,14 +49,14 @@ class MultiversionConfigSubcommand(Subcommand): mongo_version=MongoVersion.from_yaml_file(multiversionconstants.MONGO_VERSION_YAML), mongo_releases=MongoReleases.from_yaml_file(multiversionconstants.RELEASES_YAML), ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() return MultiversionConfig( last_versions=multiversionconstants.OLD_VERSIONS, - requires_fcv_tag=version_constants.get_fcv_tag_list(), - requires_fcv_tag_lts=version_constants.get_lts_fcv_tag_list(), - requires_fcv_tag_continuous=version_constants.get_continuous_fcv_tag_list(), - last_lts_fcv=version_constants.get_last_lts_fcv(), - last_continuous_fcv=version_constants.get_last_continuous_fcv(), + requires_fcv_tag=fcv_constants.get_fcv_tag_list(), + requires_fcv_tag_lts=fcv_constants.get_lts_fcv_tag_list(), + requires_fcv_tag_continuous=fcv_constants.get_continuous_fcv_tag_list(), + last_lts_fcv=fcv_constants.get_last_lts_fcv(), + last_continuous_fcv=fcv_constants.get_last_continuous_fcv(), ) @@ -79,14 +69,10 @@ class MultiversionPlugin(PluginInterface): :param subparsers: argparse subparsers """ - parser = subparsers.add_parser(MULTIVERSION_SUBCOMMAND, - help="Display configuration for multiversion testing") - - parser.add_argument("--config-file-output", '-f', action="store", type=str, default=None, - help="File to write the multiversion config to.") + subparsers.add_parser(MULTIVERSION_SUBCOMMAND, + help="Display configuration for multiversion testing") - def parse(self, subcommand: str, parser: argparse.ArgumentParser, - parsed_args: argparse.Namespace, **kwargs) -> Optional[Subcommand]: + def parse(self, subcommand, parser, parsed_args, **kwargs) -> Optional[Subcommand]: """ Resolve command-line options to a Subcommand or None. @@ -96,8 +82,7 @@ class MultiversionPlugin(PluginInterface): :param kwargs: additional args. :return: None or a Subcommand. """ + configure_resmoke.validate_and_update_config(parser, parsed_args) if subcommand == MULTIVERSION_SUBCOMMAND: - configure_resmoke.detect_evergreen_config(parsed_args) - configure_resmoke.validate_and_update_config(parser, parsed_args) - return MultiversionConfigSubcommand(parsed_args) + return MultiversionConfigSubcommand() return None diff --git a/buildscripts/resmokelib/multiversion/multiversion_service.py b/buildscripts/resmokelib/multiversion/multiversion_service.py index 621d5ade9ae..3597cc2177f 100644 --- a/buildscripts/resmokelib/multiversion/multiversion_service.py +++ b/buildscripts/resmokelib/multiversion/multiversion_service.py @@ -25,9 +25,9 @@ def version_str(version: Version) -> str: return f"{version.major}.{version.minor}" -class VersionConstantValues(NamedTuple): +class FcvConstantValues(NamedTuple): """ - Object to hold the calculated Version constants. + Object to hold the calculated FCV constants. * latest: Latest FCV. * last_continuous: Last continuous FCV. @@ -36,7 +36,6 @@ class VersionConstantValues(NamedTuple): * requires_fcv_tag_list_continuous: List of FCVs that we need to generate a tag for against continuous versions. * fcvs_less_than_latest: List of all FCVs that are less than latest, starting from v4.0. - * eols: List of stable MongoDB versions since v2.0 that have been EOL'd. """ latest: Version @@ -45,7 +44,6 @@ class VersionConstantValues(NamedTuple): requires_fcv_tag_list: List[Version] requires_fcv_tag_list_continuous: List[Version] fcvs_less_than_latest: List[Version] - eols: List[Version] def get_fcv_tag_list(self) -> str: """Get a comma joined string of all the fcv tags.""" @@ -98,10 +96,6 @@ class VersionConstantValues(NamedTuple): last_continuous = self.get_last_continuous_fcv() return f"{base_name}-{last_continuous}" - def get_eols(self) -> List[str]: - """Get EOL'd versions as list of strings.""" - return [version_str(eol) for eol in self.eols] - class MongoVersion(BaseModel): """ @@ -138,14 +132,12 @@ class MongoReleases(BaseModel): * feature_compatibility_version: All FCVs starting with 4.0. * long_term_support_releases: All LTS releases starting with 4.0. - * eol_versions: List of stable MongoDB versions since 2.0 that have been EOL'd. * generate_fcv_lower_bound_override: Extend FCV generation down to the previous value of last LTS. """ feature_compatibility_versions: List[str] = Field(alias="featureCompatibilityVersions") long_term_support_releases: List[str] = Field(alias="longTermSupportReleases") - eol_versions: List[str] = Field(alias="eolVersions") generate_fcv_lower_bound_override: Optional[str] = Field(None, alias="generateFCVLowerBoundOverride") @@ -167,11 +159,7 @@ class MongoReleases(BaseModel): def get_lts_versions(self) -> List[Version]: """Get the Version representation of the lts versions.""" - return [Version(lts) for lts in self.long_term_support_releases] - - def get_eol_versions(self) -> List[Version]: - """Get the Version representation of the EOL versions.""" - return [Version(eol) for eol in self.eol_versions] + return [Version(fcv) for fcv in self.long_term_support_releases] class MultiversionService: @@ -187,12 +175,11 @@ class MultiversionService: self.mongo_version = mongo_version self.mongo_releases = mongo_releases - def calculate_version_constants(self) -> VersionConstantValues: + def calculate_fcv_constants(self) -> FcvConstantValues: """Calculate multiversion constants from data files.""" latest = self.mongo_version.get_version() fcvs = self.mongo_releases.get_fcv_versions() lts = self.mongo_releases.get_lts_versions() - eols = self.mongo_releases.get_eol_versions() lower_bound_override = self.mongo_releases.generate_fcv_lower_bound_override # Highest release less than latest. @@ -213,12 +200,7 @@ class MultiversionService: # All FCVs less than latest. fcvs_less_than_latest = fcvs[:bisect_left(fcvs, latest)] - return VersionConstantValues( - latest=latest, - last_continuous=last_continuous, - last_lts=last_lts, - requires_fcv_tag_list=requires_fcv_tag_list, - requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, - fcvs_less_than_latest=fcvs_less_than_latest, - eols=eols, - ) + return FcvConstantValues(latest=latest, last_continuous=last_continuous, last_lts=last_lts, + requires_fcv_tag_list=requires_fcv_tag_list, + requires_fcv_tag_list_continuous=requires_fcv_tag_list_continuous, + fcvs_less_than_latest=fcvs_less_than_latest) diff --git a/buildscripts/resmokelib/multiversionconstants.py b/buildscripts/resmokelib/multiversionconstants.py index 6d31e11dc47..0cac4fbc95b 100644 --- a/buildscripts/resmokelib/multiversionconstants.py +++ b/buildscripts/resmokelib/multiversionconstants.py @@ -2,27 +2,26 @@ import os import shutil from subprocess import DEVNULL, STDOUT, CalledProcessError, call, check_output -import http -import requests -from retry import retry import structlog -from buildscripts.resmokelib.multiversion.multiversion_service import ( - MongoReleases, MongoVersion, MultiversionService, MONGO_VERSION_YAML, RELEASES_YAML) -from buildscripts.resmokelib.multiversionsetupconstants import \ - USE_EXISTING_RELEASES_FILE - -LAST_LTS = "last_lts" -LAST_CONTINUOUS = "last_continuous" - -# We use the "releases.yml" file from "master" because it is guaranteed to be up-to-date -# with the latest EOL versions. If a "last-continuous" version is EOL, we don't include -# it in the multiversion config and therefore don't test against it. -MASTER_RELEASES_REMOTE_FILE = "https://raw.githubusercontent.com/mongodb/mongo/master/src/mongo/util/version/releases.yml" +try: + # when running resmoke + from buildscripts.resmokelib.multiversion.multiversion_service import ( + MongoReleases, MongoVersion, MultiversionService) + from buildscripts.resmokelib.multiversionsetupconstants import \ + USE_EXISTING_RELEASES_FILE +except ImportError: + # when running db-contrib-tool + from multiversion.multiversion_service import (MongoReleases, MongoVersion, MultiversionService) + from multiversionsetupconstants import USE_EXISTING_RELEASES_FILE LOGGER = structlog.getLogger(__name__) +# These values must match the include paths for artifacts.tgz in evergreen.yml. +MONGO_VERSION_YAML = ".resmoke_mongo_version.yml" +RELEASES_YAML = ".resmoke_mongo_release_values.yml" + def generate_mongo_version_file(): """Generate the mongo version data file. Should only be called in the root of the mongo directory.""" @@ -38,16 +37,17 @@ def generate_mongo_version_file(): mongo_version_fh.write("mongo_version: " + res) -@retry(tries=5, delay=3) def generate_releases_file(): """Generate the releases constants file.""" # Copy the 'releases.yml' file from the source tree. - with open(RELEASES_YAML, "wb") as file: - response = requests.get(MASTER_RELEASES_REMOTE_FILE) - if response.status_code != http.HTTPStatus.OK: - raise RuntimeError("Http response for releases yml file was not 200 but was " + - response.status_code) - file.write(response.content) + releases_yaml_path = os.path.join("src", "mongo", "util", "version", "releases.yml") + if not os.path.isfile(releases_yaml_path): + LOGGER.info( + 'Skipping yml file generation because file .resmoke_mongo_release_values.yml does not exist at path {}.' + .format(releases_yaml_path)) + return + + shutil.copyfile(releases_yaml_path, RELEASES_YAML) def in_git_root_dir(): @@ -86,34 +86,33 @@ multiversion_service = MultiversionService( mongo_releases=MongoReleases.from_yaml_file(RELEASES_YAML), ) -version_constants = multiversion_service.calculate_version_constants() +fcv_constants = multiversion_service.calculate_fcv_constants() -LAST_LTS_BIN_VERSION = version_constants.get_last_lts_fcv() -LAST_CONTINUOUS_BIN_VERSION = version_constants.get_last_continuous_fcv() +LAST_LTS_BIN_VERSION = fcv_constants.get_last_lts_fcv() +LAST_CONTINUOUS_BIN_VERSION = fcv_constants.get_last_continuous_fcv() -LAST_LTS_FCV = version_constants.get_last_lts_fcv() -LAST_CONTINUOUS_FCV = version_constants.get_last_continuous_fcv() -LATEST_FCV = version_constants.get_latest_fcv() +LAST_LTS_FCV = fcv_constants.get_last_lts_fcv() +LAST_CONTINUOUS_FCV = fcv_constants.get_last_continuous_fcv() +LATEST_FCV = fcv_constants.get_latest_fcv() -LAST_CONTINUOUS_MONGO_BINARY = version_constants.build_last_continuous_binary("mongo") -LAST_CONTINUOUS_MONGOD_BINARY = version_constants.build_last_continuous_binary("mongod") -LAST_CONTINUOUS_MONGOS_BINARY = version_constants.build_last_continuous_binary("mongos") +LAST_CONTINUOUS_MONGO_BINARY = fcv_constants.build_last_continuous_binary("mongo") +LAST_CONTINUOUS_MONGOD_BINARY = fcv_constants.build_last_continuous_binary("mongod") +LAST_CONTINUOUS_MONGOS_BINARY = fcv_constants.build_last_continuous_binary("mongos") -LAST_LTS_MONGO_BINARY = version_constants.build_last_lts_binary("mongo") -LAST_LTS_MONGOD_BINARY = version_constants.build_last_lts_binary("mongod") -LAST_LTS_MONGOS_BINARY = version_constants.build_last_lts_binary("mongos") +LAST_LTS_MONGO_BINARY = fcv_constants.build_last_lts_binary("mongo") +LAST_LTS_MONGOD_BINARY = fcv_constants.build_last_lts_binary("mongod") +LAST_LTS_MONGOS_BINARY = fcv_constants.build_last_lts_binary("mongos") -REQUIRES_FCV_TAG_LATEST = version_constants.get_latest_tag() +REQUIRES_FCV_TAG_LATEST = fcv_constants.get_latest_tag() # Generate tags for all FCVS in (lastLTS, latest], or (lowerBoundOverride, latest] if requested. # All multiversion tests should be run with these tags excluded. -REQUIRES_FCV_TAG = version_constants.get_fcv_tag_list() +REQUIRES_FCV_TAG = fcv_constants.get_fcv_tag_list() # Generate evergreen project names for all FCVs less than latest. EVERGREEN_PROJECTS = ['mongodb-mongo-master'] -EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in version_constants.fcvs_less_than_latest]) +EVERGREEN_PROJECTS.extend([evg_project_str(fcv) for fcv in fcv_constants.fcvs_less_than_latest]) -OLD_VERSIONS = [ - LAST_LTS -] if LAST_CONTINUOUS_FCV == LAST_LTS_FCV or LAST_CONTINUOUS_FCV in version_constants.get_eols( -) else [LAST_LTS, LAST_CONTINUOUS] +OLD_VERSIONS = ["last_lts"] +if LAST_LTS_FCV != LAST_CONTINUOUS_FCV: + OLD_VERSIONS.append("last_continuous") diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index e450540f293..5f5bb7f3765 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -11,6 +11,7 @@ from buildscripts.resmokelib.hang_analyzer import HangAnalyzerPlugin from buildscripts.resmokelib.multiversion import MultiversionPlugin from buildscripts.resmokelib.powercycle import PowercyclePlugin from buildscripts.resmokelib.run import RunPlugin +from buildscripts.resmokelib.symbolizer import SymbolizerPlugin from buildscripts.resmokelib.undodb import UndoDbPlugin _PLUGINS = [ @@ -18,6 +19,7 @@ _PLUGINS = [ HangAnalyzerPlugin(), UndoDbPlugin(), PowercyclePlugin(), + SymbolizerPlugin(), GenerateFCVConstantsPlugin(), DiscoveryPlugin(), MultiversionPlugin(), diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index e18099de679..20ff72c2c79 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -175,8 +175,7 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes def generate_multiversion_exclude_tags(self): """Generate multiversion exclude tags file.""" generate_multiversion_exclude_tags.generate_exclude_yaml( - config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, config.EXPANSIONS_FILE, - self._resmoke_logger) + config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, self._resmoke_logger) @staticmethod def _find_suites_by_test(suites): @@ -386,12 +385,25 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ + UNRELIABLE_TAG = _TagInfo( + tag_name="unreliable", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + report_failure_status="silentfail")) + RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) + RETRY_ON_FAILURE_TAG = _TagInfo( + tag_name="retry_on_failure", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, + report_failure_status="silentfail")) + @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -426,8 +438,29 @@ class TestRunnerEvg(TestRunner): combinations = [] - combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) - combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) + if config.EVERGREEN_PATCH_BUILD: + combinations.append(("unreliable and resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("unreliable and not resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) + combinations.append(("reliable and resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("reliable and not resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + else: + combinations.append(("retry on failure and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("retry on failure and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + combinations.append(("run once and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("run once and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) return combinations @@ -707,19 +740,15 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", - help=("Do not run any tests tagged with enabled feature flags." - " This argument has precedence over --runAllFeatureFlagTests" - "; used for multiversion suites")) + "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", + action="store_true", help= + "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" + ) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") - parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", - action="store", metavar="FILE", - help="The path to a file with feature flags, delimited by newlines.") - parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -761,11 +790,6 @@ class RunPlugin(PluginInterface): metavar="ON|OFF", help=("Enable or disable majority read concern support." " Defaults to %(default)s.")) - mongodb_server_options.add_argument( - "--enableEnterpriseTests", action="store", dest="enable_enterprise_tests", default="on", - choices=("on", "off"), metavar="ON|OFF", - help=("Enable or disable enterprise tests. Defaults to 'on'.")) - mongodb_server_options.add_argument("--flowControl", action="store", dest="flow_control", choices=("on", "off"), metavar="ON|OFF", help=("Enable or disable flow control.")) @@ -855,6 +879,13 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( + "--reportFailureStatus", action="store", dest="report_failure_status", + choices=("fail", "silentfail"), metavar="STATUS", + help="Controls if the test failure status should be reported as failed" + " or be silently ignored (STATUS=silentfail). Dynamic test failures will" + " never be silently ignored. Defaults to STATUS=%(default)s.") + + internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") @@ -952,9 +983,6 @@ class RunPlugin(PluginInterface): evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") - evergreen_options.add_argument("--taskWorkDir", dest="work_dir", metavar="TASK_WORK_DIR", - help="Sets the working directory of the task.") - benchmark_options = parser.add_argument_group( title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests") @@ -1068,6 +1096,15 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) + # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on + # all_feature_flags.txt which may not exist in the local dev environment. + run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) + if run_all_feature_flag_tests is not None: + setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) + del parsed_args.run_all_feature_flag_tests + + del parsed_args.run_all_feature_flags_no_tests + # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py index d5c904bb2c3..98be3db0fa8 100755 --- a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py +++ b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py @@ -5,8 +5,6 @@ import re import tempfile from collections import defaultdict from subprocess import check_output -from typing import Optional -from github import GithubIntegration import requests @@ -15,35 +13,13 @@ from buildscripts.resmokelib.config import MultiversionOptions from buildscripts.resmokelib.core.programs import get_path_env_var from buildscripts.resmokelib.utils import is_windows from buildscripts.util.fileops import read_yaml_file -from buildscripts.util.read_config import read_config_file BACKPORT_REQUIRED_TAG = "backport_required_multiversion" # The directory in which BACKPORTS_REQUIRED_FILE resides. ETC_DIR = "etc" BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml" -BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo" - - -def get_installation_access_token(app_id: int, private_key: str, - installation_id: int) -> Optional[str]: # noqa: D406,D407,D413 - """ - Obtain an installation access token using JWT. - - Args: - - app_id: The application ID for GitHub App. - - private_key: The private key associated with the GitHub App. - - installation_id: The installation ID of the GitHub App for a particular account. - - Returns: - - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token. - """ - integration = GithubIntegration(app_id, private_key) - auth = integration.get_access_token(installation_id) - if auth: - return auth.token - else: - raise Exception("Error obtaining installation token") +BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo" def get_backports_required_hash_for_shell_version(mongo_shell_path=None): @@ -75,24 +51,10 @@ def get_backports_required_hash_for_shell_version(mongo_shell_path=None): f"Could not find a valid commit hash from the {mongo_shell_path} mongo binary.") -def get_old_yaml(commit_hash, expansions_file): +def get_old_yaml(commit_hash): """Download BACKPORTS_REQUIRED_FILE from the old commit and return the yaml.""" - - if not os.path.exists(expansions_file): - raise FileNotFoundError(f"The specified file does not exist: {expansions_file}") - expansions = read_config_file(expansions_file) - - # Obtain installation access tokens using app credentials - access_token_10gen_mongo = get_installation_access_token( - expansions["app_id_10gen_mongo"], expansions["private_key_10gen_mongo"], - expansions["installation_id_10gen_mongo"]) - response = requests.get( - f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}', - headers={ - 'Authorization': f'token {access_token_10gen_mongo}', - }) - + f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}') # If the response was successful, no exception will be raised. response.raise_for_status() @@ -106,8 +68,7 @@ def get_old_yaml(commit_hash, expansions_file): return backports_required_old -def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: str, - logger: logging.Logger) -> None: +def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Logger) -> None: """ Create a tag file associating multiversion tests to tags for exclusion. @@ -138,7 +99,7 @@ def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: st # Get the yaml contents from the old commit. logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}") - backports_required_old = get_old_yaml(old_version_commit_hash, expansions_file) + backports_required_old = get_old_yaml(old_version_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 5194c097f62..b9a06e0d742 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -19,8 +19,6 @@ from buildscripts.resmokelib import utils from buildscripts.resmokelib.utils import globstar from buildscripts.resmokelib.utils import jscomment -ENTERPRISE_TEST_DIR = os.path.normpath("src/mongo/db/modules/enterprise/jstests") - ######################## # Test file explorer # ######################## @@ -217,13 +215,6 @@ class _TestList(object): format(path))) self._filtered.discard(path) - def filter_enterprise_tests(self): - """Exclude tests that start with the enterprise module directory from the test list.""" - self._filtered = { - test - for test in self._filtered if not os.path.normpath(test).startswith(ENTERPRISE_TEST_DIR) - } - def match_tag_expression(self, tag_expression, get_tags): """Filter the test list to only include tests that match the tag expression. @@ -462,9 +453,6 @@ class _Selector(object): # 5. Apply the include files last with force=True to take precedence over the tags. if self._tests_are_files and selector_config.include_files: test_list.include_files(selector_config.include_files, force=True) - # 6: Apply the enterprise tests filter - if self.get_enterprise_tests_status() == "off": - test_list.filter_enterprise_tests() return self.sort_tests(*test_list.get_tests()) @@ -480,11 +468,6 @@ class _Selector(object): """Retrieve the tags associated with the give test file.""" return [] - @staticmethod - def get_enterprise_tests_status() -> str: - """Get the status of enterprise tests from the configuration.""" - return config.ENABLE_ENTERPRISE_TESTS - class _JSTestSelectorConfig(_SelectorConfig): """_SelectorConfig subclass for JavaScript tests.""" diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py index 609ea8a6e3e..5df67812d06 100644 --- a/buildscripts/resmokelib/sighandler.py +++ b/buildscripts/resmokelib/sighandler.py @@ -10,10 +10,10 @@ import traceback import psutil -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib import reportfile from buildscripts.resmokelib import testing from buildscripts.resmokelib import config +from buildscripts.resmokelib.hang_analyzer import hang_analyzer from buildscripts.resmokelib import parser _IS_WINDOWS = (sys.platform == "win32") @@ -32,8 +32,8 @@ def register(logger, suites, start_time): log suite summaries. """ - HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to SIGUSR1 signal" + _dump_and_log(header_msg) def _handle_set_event(event_handle): @@ -53,7 +53,6 @@ def register(logger, suites, start_time): except win32event.error as err: logger.error("Exception from win32event.WaitForSingleObject with error: %s" % err) else: - HANG_ANALYZER_CALLED.set() header_msg = "Dumping stacks due to signal from win32event.SetEvent" _dump_and_log(header_msg) @@ -160,26 +159,4 @@ def _analyze_pids(logger, pids): if not os.getenv('ASAN_OPTIONS'): hang_analyzer_args.append('-c') _hang_analyzer = parser.parse_command_line(hang_analyzer_args, logger=logger) - - # Evergreen has a 15 minute timeout for task timeout commands - # Limit the hang analyzer to 12 minutes so there is time for other tasks. - hang_analyzer_hard_timeout = None - if config.EVERGREEN_TASK_ID: - hang_analyzer_hard_timeout = 60 * 12 - logger.info( - "Limit the resmoke invoked hang analyzer to 12 minutes so there is time for resmoke to finish up." - ) - - hang_analyzer_thread = threading.Thread(target=_hang_analyzer.execute, daemon=True) - hang_analyzer_thread.start() - hang_analyzer_thread.join(hang_analyzer_hard_timeout) - - if hang_analyzer_thread.is_alive(): - logger.warning( - "Resmoke invoked hang analyzer thread did not finish, but will continue running in the background. The thread may be disruputed and may show extraneous output." - ) - logger.warning("Cleaning up resmoke child processes so that resmoke can fail gracefully.") - _hang_analyzer.kill_rogue_processes() - - else: - logger.info("Done running resmoke invoked hang analyzer thread.") + _hang_analyzer.execute() diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 55c23923ecc..394c025d510 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -4,8 +4,6 @@ import os import os.path import time import shutil -import uuid - import yaml import pymongo @@ -25,9 +23,6 @@ class MongoDFixture(interface.Fixture): self.mongod_options = self.fixturelib.make_historic( self.fixturelib.default_if_none(mongod_options, {})) - if "set_parameters" not in self.mongod_options: - self.mongod_options["set_parameters"] = {} - if add_feature_flags: for ff in self.config.ENABLED_FEATURE_FLAGS: self.mongod_options["set_parameters"][ff] = "true" @@ -57,11 +52,6 @@ class MongoDFixture(interface.Fixture): self.port = port or fixturelib.get_next_port(job_num) self.mongod_options["port"] = self.port - # Always log backtraces to a file in the dbpath in our testing. - backtrace_log_file_name = os.path.join(self.get_dbpath_prefix(), - uuid.uuid4().hex + ".stacktrace") - self.mongod_options["set_parameters"]["backtraceLogFile"] = backtrace_log_file_name - def setup(self): """Set up the mongod.""" if not self.preserve_dbpath and os.path.lexists(self._dbpath): @@ -71,8 +61,7 @@ class MongoDFixture(interface.Fixture): launcher = MongodLauncher(self.fixturelib) # Second return val is the port, which we ignore because we explicitly created the port above. - # The port is used to set other mongod_option's here: - # https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 + # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136 mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num, executable=self.mongod_executable, mongod_options=self.mongod_options) @@ -179,7 +168,7 @@ class MongoDFixture(interface.Fixture): def get_driver_connection_url(self): """Return the driver connection URL.""" - return "mongodb://" + self.get_internal_connection_string() + "/?directConnection=true" + return "mongodb://" + self.get_internal_connection_string() # The below parameters define the default 'logComponentVerbosity' object passed to mongod processes @@ -191,16 +180,15 @@ class MongoDFixture(interface.Fixture): # The default verbosity setting for any tests that are not started with an Evergreen task id. This # will apply to any tests run locally. DEFAULT_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, - "transaction": 4, "tenantMigration": 4 + "replication": {"rollback": 2}, "sharding": {"migration": 2}, "transaction": 4, + "tenantMigration": 4 } # The default verbosity setting for any mongod processes running in Evergreen i.e. started with an # Evergreen task id. DEFAULT_EVERGREEN_MONGOD_LOG_COMPONENT_VERBOSITY = { - "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, - "rollback": 2}, "sharding": {"migration": 2, "rangeDeleter": 2}, - "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 + "replication": {"election": 4, "heartbeats": 2, "initialSync": 2, "rollback": 2}, + "sharding": {"migration": 2}, "storage": {"recovery": 2}, "transaction": 4, "tenantMigration": 4 } @@ -254,15 +242,6 @@ class MongodLauncher(object): if "shardsvr" in mongod_options and "orphanCleanupDelaySecs" not in suite_set_parameters: suite_set_parameters["orphanCleanupDelaySecs"] = 1 - # receiveChunkWaitForRangeDeleterTimeoutMS controls the amount of time an incoming migration - # will wait for an intersecting range with data in it to be cleared up before failing. The - # default is 10 seconds, but in some slower variants this is not enough time for the range - # deleter to finish so we increase it here to 90 seconds. Setting a value for this parameter - # in the .yml file overrides this. - if (("shardsvr" in mongod_options or "configsvr" in mongod_options) - and "receiveChunkWaitForRangeDeleterTimeoutMS" not in suite_set_parameters): - suite_set_parameters["receiveChunkWaitForRangeDeleterTimeoutMS"] = 90000 - # The LogicalSessionCache does automatic background refreshes in the server. This is # race-y for tests, since tests trigger their own immediate refreshes instead. Turn off # background refreshing for tests. Set in the .yml file to override this. @@ -313,13 +292,6 @@ class MongodLauncher(object): "mode": "alwaysOn", "data": {"numTickets": self.config.FLOW_CONTROL_TICKETS} } - # The internalQueryForceClassicEngine parameter was renamed to - # internalQueryFrameworkControl starting in version 6.1. - if "internalQueryFrameworkControl" in suite_set_parameters and suite_set_parameters[ - "internalQueryFrameworkControl"] == "forceClassicEngine": - del suite_set_parameters["internalQueryFrameworkControl"] - suite_set_parameters["internalQueryForceClassicEngine"] = True - _add_testing_set_parameters(suite_set_parameters) shortcut_opts = { @@ -396,7 +368,3 @@ def _add_testing_set_parameters(suite_set_parameters): """ suite_set_parameters.setdefault("testingDiagnosticsEnabled", True) suite_set_parameters.setdefault("enableTestCommands", True) - # The exact file location is on a per-process basis, so it'll have to be determined when the process gets spun up. - # Set it to true for now as a placeholder that will error if no further processing is done. - # The placeholder is needed so older versions don't have this option won't have this value set. - suite_set_parameters.setdefault("backtraceLogFile", True) diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index 23ebfda8de3..38909056ce3 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -6,7 +6,6 @@ import threading from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib import utils -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED from buildscripts.resmokelib.utils import globstar @@ -106,9 +105,5 @@ class HookTestArchival(object): else: logger.info("Archive succeeded for %s: %s", test_name, message) - if HANG_ANALYZER_CALLED.is_set(): - logger.info("Hang Analyzer has been called. Fixtures will not be restarted.") - raise errors.StopExecution( - "Hang analyzer has been called. Stopping further execution of tests.") - elif not manager.setup_fixture(logger): + if not manager.setup_fixture(logger): raise errors.StopExecution("Error while restarting test fixtures after archiving.") diff --git a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py index c45f93192eb..d392c6ae0b8 100644 --- a/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py +++ b/buildscripts/resmokelib/testing/hooks/aggregate_metrics_background.py @@ -4,70 +4,70 @@ This hook runs continuously, but the run_aggregate_metrics_background.js file it internally sleep for 1 second between runs. """ -import random -import pymongo +import os.path -from buildscripts.resmokelib.testing.hooks.bghook import BGHook +from buildscripts.resmokelib import errors +from buildscripts.resmokelib.testing.hooks import jsfile +from buildscripts.resmokelib.testing.hooks.background_job import _BackgroundJob, _ContinuousDynamicJSTestCase -def verify_metrics(doc): - """Checks whether the output from $operatiomMetrics has the schema we expect.""" +class AggregateResourceConsumptionMetricsInBackground(jsfile.JSHook): + """A hook to run $operationMetrics stage in the background.""" - top_level_fields = [ - "docBytesWritten", "docUnitsWritten", "idxEntryBytesWritten", "idxEntryUnitsWritten", - "totalUnitsWritten", "cpuNanos", "db", "primaryMetrics", "secondaryMetrics" - ] - read_fields = [ - "docBytesRead", "docUnitsRead", "idxEntryBytesRead", "idxEntryUnitsRead", "keysSorted", - "docUnitsReturned" - ] + IS_BACKGROUND = True - for key in top_level_fields: - assert key in doc, ("The metrics output is missing the property: " + key) + def __init__(self, hook_logger, fixture, shell_options=None): + """Initialize AggregateResourceConsumptionMetricsInBackground.""" + description = "Run background $operationMetrics on all mongods while a test is running" + js_filename = os.path.join("jstests", "hooks", "run_aggregate_metrics_background.js") + jsfile.JSHook.__init__(self, hook_logger, fixture, js_filename, description, + shell_options=shell_options) + self._background_job = None - primary_metrics = doc["primaryMetrics"] - for key in read_fields: - assert key in primary_metrics, ( - "The metrics output is missing the property: primaryMetrics." + key) + def before_suite(self, test_report): + """Start the background thread.""" + self._background_job = _BackgroundJob("AggregateResourceConsumptionMetricsInBackground") + self.logger.info("Starting the background aggregate metrics thread.") + self._background_job.start() - secondary_metrics = doc["secondaryMetrics"] - for key in read_fields: - assert key in secondary_metrics, ( - "The metrics output is missing the property: secondaryMetrics." + key) + def after_suite(self, test_report, teardown_flag=None): + """Signal the background aggregate metrics thread to exit, and wait until it does.""" + if self._background_job is None: + return + self.logger.info("Stopping the background aggregate metrics thread.") + self._background_job.stop() -class AggregateResourceConsumptionMetricsInBackground(BGHook): - """A hook to run $operationMetrics stage in the background.""" + def before_test(self, test, test_report): + """Instruct the background aggregate metrics thread to run while 'test' is also running.""" + if self._background_job is None: + return - def __init__(self, hook_logger, fixture): - """Initialize AggregateResourceConsumptionMetricsInBackground.""" + hook_test_case = _ContinuousDynamicJSTestCase.create_before_test( + test.logger, test, self, self._js_filename, self._shell_options) + hook_test_case.configure(self.fixture) - description = "Run background $operationMetrics on all mongods while a test is running" - super().__init__(hook_logger, fixture, description, tests_per_cycle=None, - loop_delay_ms=1000) - - def run_action(self): - """Collects $operationMetrics on all non-arbiter nodes in the fixture.""" - for node_info in self.fixture.get_node_info(): - conn = pymongo.MongoClient(port=node_info.port) - # Filter out arbiters. - if "arbiterOnly" in conn.admin.command({"isMaster": 1}): - self.logger.info( - "Skipping background aggregation against test node: %s because it is an " + - "arbiter and has no data.", node_info.full_name) - return - - # Clear the metrics about 10% of the time. - clear_metrics = random.random() < 0.1 - self.logger.info("Running $operationMetrics with {clearMetrics: %s} on host: %s", - clear_metrics, node_info.full_name) - with conn.admin.aggregate( - [{"$operationMetrics": {"clearMetrics": clear_metrics}}]) as cursor: - for doc in cursor: - try: - verify_metrics(doc) - except: - self.logger.info( - "caught exception while verifying that all expected fields are in the" + - " metrics output: ", doc) - raise + self.logger.info("Resuming the background aggregate metrics thread.") + self._background_job.resume(hook_test_case, test_report) + + def after_test(self, test, test_report): # noqa: D205,D400 + """Instruct the background aggregate metrics thread to stop running now that 'test' has + finished running. + """ + if self._background_job is None: + return + + self.logger.info("Pausing the background aggregate metrics thread.") + self._background_job.pause() + + if self._background_job.exc_info is not None: + if isinstance(self._background_job.exc_info[1], errors.TestFailure): + # If the mongo shell process running the JavaScript file exited with a non-zero + # return code, then we raise an errors.ServerFailure exception to cause resmoke.py's + # test execution to stop. + raise errors.ServerFailure(self._background_job.exc_info[1].args[0]) + else: + self.logger.error( + "Encountered an error inside the background aggregate metrics thread.", + exc_info=self._background_job.exc_info) + raise self._background_job.exc_info[1] diff --git a/buildscripts/resmokelib/testing/hooks/background_job.py b/buildscripts/resmokelib/testing/hooks/background_job.py index cd775aabd4a..92dbb5437a4 100644 --- a/buildscripts/resmokelib/testing/hooks/background_job.py +++ b/buildscripts/resmokelib/testing/hooks/background_job.py @@ -3,7 +3,6 @@ import sys import threading -from buildscripts.resmokelib import errors from buildscripts.resmokelib.testing.hooks import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/bghook.py b/buildscripts/resmokelib/testing/hooks/bghook.py index 1b3fed3b104..b61f37b4e2d 100644 --- a/buildscripts/resmokelib/testing/hooks/bghook.py +++ b/buildscripts/resmokelib/testing/hooks/bghook.py @@ -13,13 +13,11 @@ class BGJob(threading.Thread): BGJob will call 'run_action' without any delay and expects the 'run_action' function to add some form of delay. """ - def __init__(self, hook, loop_delay_ms=None): + def __init__(self, hook): """Initialize the background job.""" threading.Thread.__init__(self, name=f"BGJob-{hook.__class__.__name__}") - self._loop_delay_ms = loop_delay_ms self.daemon = True self._hook = hook - self._interrupt_event = threading.Event() self.__is_alive = True self.err = None @@ -31,14 +29,6 @@ class BGJob(threading.Thread): try: self._hook.run_action() - if self._loop_delay_ms is not None: - # The configured loop delay asked us to wait before running the action again. Do - # that wait, but listen to see if we finish running the test or are killed in - # the meantime. - interrupted = self._interrupt_event.wait(self._loop_delay_ms / 1000.0) - if interrupted: - self._hook.logger.info("interrupted") - break except Exception as err: # pylint: disable=broad-except self._hook.logger.error("Background thread caught exception: %s.", err) self.err = err @@ -47,7 +37,6 @@ class BGJob(threading.Thread): def kill(self): """Kill the background job.""" self.__is_alive = False - self._interrupt_event.set() class BGHook(interface.Hook): @@ -57,13 +46,8 @@ class BGHook(interface.Hook): # By default, we continuously run the background hook for the duration of the suite. DEFAULT_TESTS_PER_CYCLE = math.inf - def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None, loop_delay_ms=None): - """ - Initialize the background hook. - - 'tests_per_cycle' or 'loop_delay_ms' can be used to configure how often the background job - is restarted, and how often run_action() is called, respectively. - """ + def __init__(self, hook_logger, fixture, desc, tests_per_cycle=None): + """Initialize the background hook.""" interface.Hook.__init__(self, hook_logger, fixture, desc) self.logger = hook_logger @@ -73,21 +57,15 @@ class BGHook(interface.Hook): self._test_num = 0 # The number of tests we execute before restarting the background hook. self._tests_per_cycle = self.DEFAULT_TESTS_PER_CYCLE if tests_per_cycle is None else tests_per_cycle - self._loop_delay_ms = loop_delay_ms def run_action(self): - """ - Perform an action. This function will be called continuously in the BgJob. - - If a sleep_delay_ms was given, that many milliseconds of sleep will happen between each - invocation. - """ + """Perform an action. This function will be called continuously in the BgJob.""" raise NotImplementedError def before_suite(self, test_report): """Start the background thread.""" self.logger.info("Starting the background thread.") - self._background_job = BGJob(self, self._loop_delay_ms) + self._background_job = BGJob(self) self._background_job.start() def after_suite(self, test_report, teardown_flag=None): @@ -108,7 +86,7 @@ class BGHook(interface.Hook): return self.logger.info("Restarting the background thread.") - self._background_job = BGJob(self, self._loop_delay_ms) + self._background_job = BGJob(self) self._background_job.start() def after_test(self, test, test_report): diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index 5278819da8f..b0ff858c5dc 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -276,15 +276,16 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): secondary.await_ready() client = secondary.mongo_client() + minvalid_doc = client.local["replset.minvalid"].find_one() oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one() recovery_timestamp_res = client.admin.command("replSetTest", getLastStableRecoveryTimestamp=True) latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural", pymongo.DESCENDING)]) - self.logger.info("Checking replication invariants. oplogTruncateAfterPoint: {}," + self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {}," " stable recovery timestamp: {}, latest oplog doc: {}".format( - oplog_truncate_after_doc, recovery_timestamp_res, + minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res, latest_oplog_doc)) null_ts = bson.Timestamp(0, 0) @@ -298,6 +299,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): raise errors.ServerFailure( "Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc)) + # The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default + # it to null. + oplog_truncate_after_ts = null_ts + if oplog_truncate_after_doc is not None: + oplog_truncate_after_ts = oplog_truncate_after_doc.get( + "oplogTruncateAfterPoint", null_ts) + # The "lastStableRecoveryTimestamp" field is present if the storage engine supports # "recover to a timestamp". If it's a null timestamp on a durable storage engine, that # means we do not yet have a stable checkpoint timestamp and must be restarting at the @@ -320,6 +328,94 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): recovery_timestamp, latest_oplog_entry_ts, recovery_timestamp_res, latest_oplog_doc)) + if minvalid_doc is not None: + applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts) + minvalid_ts = minvalid_doc.get("ts", null_ts) + + # The "appliedThrough" value should always equal the "last stable recovery + # timestamp", AKA the stable checkpoint for durable engines, on server restart. + # + # The written "appliedThrough" time is updated with the latest timestamp at the end + # of each batch application, and batch boundaries are the only valid stable + # timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must + # equal the checkpoint timestamp, because any stable timestamp that the checkpoint + # could use includes an equal persisted appliedThrough timestamp. + if (recovery_timestamp != null_ts and applied_through_ts != null_ts + and (not recovery_timestamp == applied_through_ts)): + raise errors.ServerFailure( + "The condition last stable recovery timestamp ({}) == appliedThrough ({})" + " doesn't hold: minValid document={}," + " getLastStableRecoveryTimestamp result={}, last oplog entry={}".format( + recovery_timestamp, applied_through_ts, minvalid_doc, + recovery_timestamp_res, latest_oplog_doc)) + + if applied_through_ts == null_ts: + # We clear "appliedThrough" to represent having applied through the top of the + # oplog in PRIMARY state or immediately after "rollback via refetch". + # If we are using a storage engine that supports "recover to a timestamp," + # then we will have a "last stable recovery timestamp" and we should use that + # as our "appliedThrough" (similarly to why we assert their equality above). + # If both are null, then we are in PRIMARY state on a storage engine that does + # not support "recover to a timestamp" or in RECOVERING immediately after + # "rollback via refetch". Since we do not update "minValid" in PRIMARY state, + # we leave "appliedThrough" as null so that the invariants below hold, rather + # than substituting the latest oplog entry for the "appliedThrough" value. + applied_through_ts = recovery_timestamp + + if minvalid_ts == null_ts: + # The server treats the "ts" field in the minValid document as missing when its + # value is the null timestamp. + minvalid_ts = applied_through_ts + + if latest_oplog_entry_ts == null_ts: + # If the oplog is empty, we treat the "minValid" as the latest oplog entry. + latest_oplog_entry_ts = minvalid_ts + + if oplog_truncate_after_ts == null_ts: + # The server treats the "oplogTruncateAfterPoint" field as missing when its + # value is the null timestamp. When it is null, the oplog is complete and + # should not be truncated, so it is effectively the top of the oplog. + oplog_truncate_after_ts = latest_oplog_entry_ts + + # Check the ordering invariants before the secondary has reconciled the end of + # its oplog. + # The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of + # oplog entries before they are written to the oplog. Thus, it can be ahead + # of the top of the oplog before any oplog entries are written, and behind it + # after some are written. Thus, we cannot compare it to the top of the oplog. + + # appliedThrough <= minValid + # appliedThrough represents the end of the previous batch, so it is always the + # earliest. + if applied_through_ts > minvalid_ts: + raise errors.ServerFailure( + "The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid" + " document={}, latest oplog entry={}".format( + applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc)) + + # minValid <= oplogTruncateAfterPoint + # This is true because this hook is never run after a rollback. Thus, we only + # move "minValid" to the end of each batch after the batch is written to the oplog. + # We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from + # the end of the previous batch to the end of the current batch. Thus "minValid" + # must be less than or equal to the "oplogTruncateAfterPoint". + if minvalid_ts > oplog_truncate_after_ts: + raise errors.ServerFailure( + "The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't" + " hold: minValid document={}, oplogTruncateAfterPoint document={}," + " latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts, + minvalid_doc, oplog_truncate_after_doc, + latest_oplog_doc)) + + # minvalid <= latest oplog entry + # "minValid" is set to the end of a batch after the batch is written to the oplog. + # Thus it is always less than or equal to the top of the oplog. + if minvalid_ts > latest_oplog_entry_ts: + raise errors.ServerFailure( + "The condition minValid <= top of oplog ({} <= {}) doesn't" + " hold: minValid document={}, latest oplog entry={}".format( + minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc)) + try: secondary.teardown() except errors.ServerFailure: diff --git a/buildscripts/resmokelib/testing/hooks/run_query_stats.py b/buildscripts/resmokelib/testing/hooks/run_query_stats.py deleted file mode 100644 index 736c5e261c7..00000000000 --- a/buildscripts/resmokelib/testing/hooks/run_query_stats.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Test hook for verifying $queryStats collects expected metrics and can redact query shapes. - -This runs in the background as other tests are ongoing. -""" - -from bson import binary -import pymongo.errors -from buildscripts.resmokelib.testing.hooks.interface import Hook - -QUERY_STATS_NOT_ENABLED_CODES = [224, 7373500, 6579000] - - -class RunQueryStats(Hook): - """Runs $queryStats after every test, and clears the query stats store before every test.""" - - IS_BACKGROUND = False - - def __init__(self, hook_logger, fixture, allow_feature_not_supported=False): - """Initialize the RunQueryStats hook. - - Args: - hook_logger: the logger instance for this hook. - fixture: the target fixture (replica sets or a sharded cluster). - allow_feature_not_supported: absorb 'QueryFeatureNotAllowed' errors when calling - $queryStats. This is to support fuzzer suites that may manipulate the FCV. - """ - description = "Read query stats data after each test." - super().__init__(hook_logger, fixture, description) - self.client = self.fixture.mongo_client() - self.hmac_key = binary.Binary(("0" * 32).encode('utf-8'), 8) - self.allow_feature_not_supported = allow_feature_not_supported - - def verify_query_stats(self, querystats_spec): - """Verify a $queryStats call has all the right properties.""" - query_stats_pipeline = [ - {"$queryStats": querystats_spec}, - # SERVER-90921: The pymongo version we use on this branch has trouble parsing invalid - # DBRefs, which can be produced by the 'key' field. We'll overwrite that field with a - # dummy one, since the contents aren't important for this test/check. - {"$set": {"key": "Redacted due to issues with DBRefs"}} - ] - try: - with self.client.admin.aggregate(query_stats_pipeline) as cursor: - nreturned = 0 - for operation in cursor: - assert "key" in operation - assert "metrics" in operation - assert "asOf" in operation - nreturned += 1 - self.logger.info("Found %d query stats entries.", nreturned) - except pymongo.errors.OperationFailure as err: - if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: - self.logger.info("Encountered an error while running $queryStats. " - "$queryStats will not be run for this test.") - else: - raise err - - def after_test(self, test, test_report): - """After the test, make sure we can ingest the query stats, with and without hmac.""" - self.verify_query_stats({}) - self.verify_query_stats( - {"transformIdentifiers": {"algorithm": "hmac-sha-256", "hmacKey": self.hmac_key}}) - - # Log the number of evictions we encountered. - server_status = self.client.admin.command({"serverStatus": 1}) - num_evicted_entries = server_status["metrics"]["queryStats"]["numEvicted"] - if num_evicted_entries > 0: - self.logger.info("Evicted %d query stats entries during test execution.", - num_evicted_entries) - - def before_test(self, test, test_report): - """Before the test, reset the contents of the query stats store.""" - try: - # Clear out all existing entries, then reset the size cap. - self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="0%") - self.client.admin.command("setParameter", 1, internalQueryStatsCacheSize="1%") - except pymongo.errors.OperationFailure as err: - if self.allow_feature_not_supported and err.code in QUERY_STATS_NOT_ENABLED_CODES: - self.logger.info("Encountered an error while configuring the query stats store. " - "Query stats will not be collected for this test.") - else: - raise err diff --git a/buildscripts/resmokelib/testing/hooks/simulate_crash.py b/buildscripts/resmokelib/testing/hooks/simulate_crash.py index 68407a66192..c7f8bc4026c 100644 --- a/buildscripts/resmokelib/testing/hooks/simulate_crash.py +++ b/buildscripts/resmokelib/testing/hooks/simulate_crash.py @@ -94,14 +94,7 @@ class SimulateCrash(bghook.BGHook): rel = fqfn[len(root):] os.makedirs(new_root + "/journal", exist_ok=True) out_fd = os.open(new_root + rel, os.O_WRONLY | os.O_CREAT) - - total_bytes_sent = 0 - while total_bytes_sent < in_bytes: - bytes_sent = os.sendfile(out_fd, in_fd, total_bytes_sent, in_bytes - total_bytes_sent) - if bytes_sent == 0: - raise ValueError("Unexpectedly reached EOF copying file") - total_bytes_sent += bytes_sent - + os.sendfile(out_fd, in_fd, 0, in_bytes) os.close(out_fd) os.close(in_fd) diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index 8fb565cd1d8..c09602baee8 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -363,7 +363,7 @@ class FileBasedStepdownLifecycle(object): # We remove the "permitted" file to revoke permission for the stepdown thread to continue # performing stepdowns. - fs.remove_if_exists(self.__stepdown_files.permitted) + os.remove(self.__stepdown_files.permitted) class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-attributes @@ -627,10 +627,6 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at break except pymongo.errors.NotMasterError: pass - except pymongo.errors.OperationFailure as ex: - if ex.code == 166: # CommandNotSupportedOnView - # listCollections return also views and collStats is not supported on views - break retarget_time = time.time() - start_time if retarget_time >= 60: raise RuntimeError( diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index e2655e8bf81..38fe409b656 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -4,14 +4,12 @@ This is used to support additional test status and timing information for the re """ import copy -import os import threading import time import unittest from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import logging -from buildscripts.resmokelib.testing.symbolizer_service import ResmokeSymbolizer # pylint: disable=attribute-defined-outside-init @@ -56,9 +54,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for report in reports: if not isinstance(report, TestReport): - raise TypeError( - f"reports must be a list of TestReport instances, current report is {type(report)}" - ) + raise TypeError("reports must be a list of TestReport instances") with report._lock: # pylint: disable=protected-access for test_info in report.test_infos: @@ -110,7 +106,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr unittest.TestResult.startTest(self, test) test_info = TestInfo(test.id(), test.test_name, test.dynamic) - test_info.group_id = f"job{self.job_num}" basename = test.basename() command = test.as_command() @@ -129,53 +124,40 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test.basename(), command, test.logger, self.job_num, test.id(), self.job_logger) - test_info.log_info = { - "log_name": logging.loggers.get_evergreen_log_name(self.job_num, test.id()), - "logs_to_merge": [logging.loggers.get_evergreen_log_name(self.job_num)], - "rendering_type": "resmoke", "version": 0 - } test_info.url_endpoint = url_endpoint if self.logging_prefix is not None: test_logger.info(self.logging_prefix) # Set job_num in test. test.job_num = self.job_num + test.override_logger(test_logger) test_info.start_time = time.time() def stopTest(self, test): # pylint: disable=invalid-name """Call after 'test' has run.""" - try: - # check if there are stacktrace files, if so, invoke the symbolizer here. - # log symbolized output to test.logger.info() - symbolizer = ResmokeSymbolizer() - symbolizer.symbolize_test_logs(test) - - unittest.TestResult.stopTest(self, test) - - with self._lock: - test_info = self.find_test_info(test) - test_info.end_time = time.time() - test_status = "no failures detected" if test_info.status == "pass" else "failed" - - time_taken = test_info.end_time - test_info.start_time - self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, - test_status) - - finally: - # This is a failsafe. In the event that 'stopTest' fails, - # any rogue logger handlers will be removed from this test. - # If not cleaned up, these will trigger 'setup failures' -- - # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE. - for handler in test.logger.handlers: - # We ignore the cancellation token returned by close_later() since we always want the - # logs to eventually get flushed. - logging.flush.close_later(handler) - - # Restore the original logger for the test. - test.reset_logger() - - def addError(self, test, err): + unittest.TestResult.stopTest(self, test) + + with self._lock: + test_info = self.find_test_info(test) + test_info.end_time = time.time() + test_status = "no failures detected" if test_info.status == "pass" else "failed" + + time_taken = test_info.end_time - test_info.start_time + self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken, + test_status) + + # Asynchronously closes the buildlogger test handler to avoid having too many threads open + # on 32-bit systems. + for handler in test.logger.handlers: + # We ignore the cancellation token returned by close_later() since we always want the + # logs to eventually get flushed. + logging.flush.close_later(handler) + + # Restore the original logger for the test. + test.reset_logger() + + def addError(self, test, err): # pylint: disable=invalid-name """Call when a non-failureException was raised during the execution of 'test'.""" unittest.TestResult.addError(self, test, err) @@ -219,7 +201,12 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = self.find_test_info(test) test_info.status = "fail" - test_info.evergreen_status = "fail" + if test_info.dynamic: + # Dynamic tests are used for data consistency checks, so the failures are never + # silenced. + test_info.evergreen_status = "fail" + else: + test_info.evergreen_status = self.suite_options.report_failure_status test_info.return_code = test.return_code def setFailure(self, test, return_code=1): # pylint: disable=invalid-name @@ -231,7 +218,12 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr raise ValueError("stopTest was not called on %s" % (test.basename())) test_info.status = "fail" - test_info.evergreen_status = "fail" + if test_info.dynamic: + # Dynamic tests are used for data consistency checks, so the failures are never + # silenced. + test_info.evergreen_status = "fail" + else: + test_info.evergreen_status = self.suite_options.report_failure_status test_info.return_code = return_code # Recompute number of success, failures, and errors. @@ -294,18 +286,19 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for test_info in self.test_infos: result = { "test_file": test_info.test_file, - "group_id": test_info.group_id, "status": test_info.evergreen_status, "exit_code": test_info.return_code, "start": test_info.start_time, "end": test_info.end_time, "elapsed": test_info.end_time - test_info.start_time, - "log_info": test_info.log_info, } if test_info.display_test_name is not None: result["display_test_name"] = test_info.display_test_name + if test_info.group_id is not None: + result["group_id"] = test_info.group_id + if test_info.url_endpoint is not None: result["url"] = test_info.url_endpoint result["url_raw"] = test_info.url_endpoint + "?raw=1" @@ -335,7 +328,6 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr test_info = TestInfo(test_file, test_file, is_dynamic) test_info.display_test_name = result.get("display_test_name") test_info.group_id = result.get("group_id") - test_info.log_info = result.get("log_info") test_info.url_endpoint = result.get("url") test_info.status = result["status"] test_info.evergreen_status = test_info.status @@ -389,16 +381,15 @@ class TestInfo(object): # pylint: disable=too-many-instance-attributes self.test_id = test_id self.test_file = test_file - self.group_id = None self.display_test_name = None self.dynamic = dynamic + self.group_id = None self.start_time = None self.end_time = None self.status = None self.evergreen_status = None self.return_code = None - self.log_info = None self.url_endpoint = None diff --git a/buildscripts/resmokelib/testing/symbolizer_service.py b/buildscripts/resmokelib/testing/symbolizer_service.py deleted file mode 100644 index 9e78b0dd6b7..00000000000 --- a/buildscripts/resmokelib/testing/symbolizer_service.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Symbolize stacktraces inside test logs.""" -from __future__ import annotations - -import os -import subprocess -import sys -import time -from datetime import timedelta -from threading import Lock - -from typing import List, Optional, NamedTuple - -from buildscripts.resmokelib import config as _config -from buildscripts.resmokelib.flags import HANG_ANALYZER_CALLED -from buildscripts.resmokelib.testing.testcases.interface import TestCase - -# This lock prevents different resmoke jobs from symbolizing stacktraces concurrently, -# which includes downloading the debug symbols, that can be reused by other resmoke jobs -_lock = Lock() - -STACKTRACE_FILE_EXTENSION = ".stacktrace" -SYMBOLIZE_RETRY_TIMEOUT_SECS = timedelta(minutes=4).total_seconds() - - -class ResmokeSymbolizerConfig(NamedTuple): - """ - Resmoke symbolizer config. - - * evg_task_id: evergreen task ID resmoke runs on - * client_id: symbolizer client ID - * client_secret: symbolizer client secret - """ - - evg_task_id: Optional[str] - client_id: Optional[str] - client_secret: Optional[str] - - @classmethod - def from_resmoke_config(cls) -> ResmokeSymbolizerConfig: - """ - Make resmoke symbolizer config from a global resmoke config. - - :return: resmoke symbolizer config - """ - return cls( - evg_task_id=_config.EVERGREEN_TASK_ID, - client_id=_config.SYMBOLIZER_CLIENT_ID, - client_secret=_config.SYMBOLIZER_CLIENT_SECRET, - ) - - @staticmethod - def is_windows() -> bool: - """ - Whether we are on Windows. - - :return: True if on Windows - """ - return sys.platform == "win32" or sys.platform == "cygwin" - - @staticmethod - def is_macos() -> bool: - """ - Whether we are on MacOS. - - :return: True if on MacOS. - """ - return sys.platform == "darwin" - - -class ResmokeSymbolizer: - """Symbolize stacktraces inside test logs.""" - - def __init__(self, config: Optional[ResmokeSymbolizerConfig] = None, - symbolizer_service: Optional[SymbolizerService] = None, - file_service: Optional[FileService] = None): - """Initialize instance.""" - - self.config = config if config is not None else ResmokeSymbolizerConfig.from_resmoke_config( - ) - self.symbolizer_service = symbolizer_service if symbolizer_service is not None else SymbolizerService( - ) - self.file_service = file_service if file_service is not None else FileService() - - def symbolize_test_logs(self, test: TestCase, - symbolize_retry_timeout: float = SYMBOLIZE_RETRY_TIMEOUT_SECS) -> None: - """ - Perform all necessary actions to symbolize and write output to test logs. - - :param test: resmoke test case - :param symbolize_retry_timeout: the timeout for symbolizer retries - """ - if not self.should_symbolize(test): - return - - dbpath = self.get_stacktrace_dir(test) - if dbpath is None: - return - - test.logger.info("Looking for stacktrace files in '%s'", dbpath) - files = self.collect_stacktrace_files(dbpath) - if not files: - test.logger.info("No failure logs/stacktrace files found, skipping symbolization") - return - - with _lock: - test.logger.info("Found stacktrace files. \nBEGIN Symbolization") - test.logger.info("Stacktrace files: %s", files) - - start_time = time.perf_counter() - for file_path in files: - test.logger.info("Working on: %s", file_path) - symbolizer_script_timeout = int(symbolize_retry_timeout - - (time.perf_counter() - start_time)) - symbolized_out = self.symbolizer_service.run_symbolizer_script( - file_path, symbolizer_script_timeout) - test.logger.info(symbolized_out) - if time.perf_counter() - start_time > symbolize_retry_timeout: - break - - # To avoid performing the same actions on these files again, we remove them - self.file_service.remove_all(files) - - test.logger.info("\nEND Symbolization \nSymbolization process completed. ") - - def should_symbolize(self, test: TestCase) -> bool: - """ - Check whether we should perform symbolization process. - - :param test: resmoke test case - :return: whether we should symbolize - """ - if self.config.evg_task_id is None: - test.logger.info("Not running in Evergreen, skipping symbolization") - return False - - if self.config.client_id is None or self.config.client_secret is None: - test.logger.info("Symbolizer client secret and/or client ID are absent," - " skipping symbolization") - return False - - if self.config.is_windows(): - test.logger.info("Running on Windows, skipping symbolization") - return False - - if self.config.is_macos(): - test.logger.info("Running on MacOS, skipping symbolization") - return False - - if HANG_ANALYZER_CALLED.is_set(): - test.logger.info( - "Hang analyzer has been called, skipping symbolization to meet timeout constraints." - ) - return False - - return True - - def get_stacktrace_dir(self, test: TestCase) -> Optional[str]: - """ - Get dbpath from test case. - - :param test: resmoke test case - :return: dbpath or None - """ - if not hasattr(test, "fixture") or test.fixture is None: - test.logger.info("Test fixture is not available, could not get dbpath") - return None - - dbpath = test.fixture.get_dbpath_prefix() - if not self.file_service.check_path_exists(dbpath): - test.logger.info("dbpath '%s' directory not found", dbpath) - return None - - return dbpath - - def collect_stacktrace_files(self, dir_path: str) -> List[str]: - """ - Collect all stacktrace files which are not empty and return their full paths. - - :param dir_path: directory to look into - :return: list of stacktrace files paths - """ - - files = self.file_service.find_all_children_recursively(dir_path) - files = self.file_service.filter_by_extension(files, STACKTRACE_FILE_EXTENSION) - self.file_service.remove_empty(files) - files = self.file_service.filter_out_non_files(files) - - return files - - -class FileService: - """A service for working with files.""" - - @staticmethod - def find_all_children_recursively(dir_path: str) -> List[str]: - """ - Find all children files in directory recursively. - - :param dir_path: directory path - :return: list of all children files - """ - children_in_dir = [] - for parent, _, children in os.walk(dir_path): - children_in_dir.extend(os.path.join(parent, child) for child in children) - return children_in_dir - - @staticmethod - def filter_by_extension(files: List[str], extension: str) -> List[str]: - """ - Filter files by extension. - - :param files: list of file paths - :param extension: file extension - :return: filtered list of file paths - """ - return [f for f in files if f.endswith(extension)] - - @staticmethod - def filter_out_non_files(files: List[str]) -> List[str]: - """ - Filter out non files. - - :param files: list of paths - :return: filtered list of file paths - """ - return [f for f in files if os.path.isfile(f)] - - @staticmethod - def remove_empty(files: List[str]) -> None: - """ - Delete files that are empty. - - :param files: list of paths - """ - for file in [f for f in files if os.stat(f).st_size == 0]: - os.remove(file) - - @staticmethod - def remove_all(files: List[str]) -> None: - """ - Delete all files. - - :param files: list of paths - """ - for file in files: - os.remove(file) - - @staticmethod - def check_path_exists(path: str) -> bool: - """ - Check that file or directory exists. - - :param path: file or directory path - :return: whether path exists - """ - return os.path.exists(path) - - -class SymbolizerService: - """Wrapper around symbolizer script.""" - - @staticmethod - def run_symbolizer_script(full_file_path: str, retry_timeout_secs: int) -> str: - """ - Symbolize given file and return symbolized output as string. - - :param full_file_path: stacktrace file path - :param retry_timeout_secs: the timeout for symbolizer to retry - :return: symbolized output as string - """ - - symbolizer_args = [ - "db-contrib-tool", - "symbolize", - "--client-secret", - _config.SYMBOLIZER_CLIENT_SECRET, - "--client-id", - _config.SYMBOLIZER_CLIENT_ID, - "--total-seconds-for-retries", - str(retry_timeout_secs), - ] - - with open(full_file_path) as file_obj: - symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True, - stdin=file_obj, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - try: - output, _ = symbolizer_process.communicate(timeout=retry_timeout_secs) - except subprocess.TimeoutExpired: - symbolizer_process.kill() - output, _ = symbolizer_process.communicate() - - return output.strip().decode() diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 81202daceb8..7658a415fa6 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -85,7 +85,7 @@ def get_task_name_without_suffix(task_name, variant_name): """Return evergreen task name without suffix added to the generated task. Remove evergreen variant name, numerical suffix and underscores between them from evergreen task name. - Example: "noPassthrough_0_enterprise-rhel-8-64-bit-dynamic-required" -> "noPassthrough" + Example: "noPassthrough_0_enterprise-rhel-80-64-bit-dynamic-required" -> "noPassthrough" """ task_name = task_name if task_name else "" return re.sub(fr"(_[0-9]+)?(_{variant_name})?$", "", task_name) diff --git a/buildscripts/resmokelib/utils/check_has_tag.py b/buildscripts/resmokelib/utils/check_has_tag.py index 18ae19bc21b..aebdd5644fe 100755 --- a/buildscripts/resmokelib/utils/check_has_tag.py +++ b/buildscripts/resmokelib/utils/check_has_tag.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """CLI interface for jscomment.""" -import re import sys import jscomment @@ -17,21 +16,11 @@ try: print(' 3 if any error happened during check') print('Usage:') print(' check_has_tag.py <jsfile> <tag>') - print('Notice: <tag> is a regex, not search string') sys.exit(2) else: tags = jscomment.get_tags(sys.argv[1]) print(sys.argv[1], "has tags:", tags) - - prog = re.compile(sys.argv[2]) - for tag in tags: - if prog.match(tag): - print("tag matches:", tag) - sys.exit(0) - - print("no tags match", sys.argv[2]) - sys.exit(1) - + sys.exit(0 if sys.argv[2] in tags else 1) except Exception as err: # pylint: disable=W0703 print(err) sys.exit(3) diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py index 6153349b6af..5857870e627 100644 --- a/buildscripts/resmokelib/utils/globstar.py +++ b/buildscripts/resmokelib/utils/globstar.py @@ -1,9 +1,11 @@ """Filename globbing utility.""" import glob as _glob +import os import os.path import re +_GLOBSTAR = "**" _CONTAINS_GLOB_PATTERN = re.compile("[*?[]") @@ -33,6 +35,155 @@ def iglob(globbed_pathname): expanded to match zero or more subdirectories. """ - for pathname in _glob.iglob(globbed_pathname, recursive=True): - # Normalize 'pathname' so exact string comparison can be used later. - yield os.path.normpath(pathname) + parts = _split_path(globbed_pathname) + parts = _canonicalize(parts) + + index = _find_globstar(parts) + if index == -1: + for pathname in _glob.iglob(globbed_pathname): + # Normalize 'pathname' so exact string comparison can be used later. + yield os.path.normpath(pathname) + return + + # **, **/, or **/a + if index == 0: + expand = _expand_curdir + + # a/** or a/**/ or a/**/b + else: + expand = _expand + + prefix_parts = parts[:index] + suffix_parts = parts[index + 1:] + + prefix = os.path.join(*prefix_parts) if prefix_parts else os.curdir + suffix = os.path.join(*suffix_parts) if suffix_parts else "" + + for (kind, path) in expand(prefix): + if not suffix_parts: + yield path + + # Avoid following symlinks to avoid an infinite loop + elif suffix_parts and kind == "dir" and not os.path.islink(path): + path = os.path.join(path, suffix) + for pathname in iglob(path): + yield pathname + + +def _split_path(pathname): + """Return 'pathname' as a list of path components.""" + + parts = [] + + while True: + (dirname, basename) = os.path.split(pathname) + parts.append(basename) + if pathname == dirname: + parts.append(dirname) + break + if not dirname: + break + pathname = dirname + + parts.reverse() + return parts + + +def _canonicalize(parts): + """Return a copy of 'parts' with consecutive "**"s coalesced. + + Raise a ValueError for unsupported uses of "**". + """ + + res = [] + + prev_was_globstar = False + for part in parts: + if part == _GLOBSTAR: + # Skip consecutive **'s + if not prev_was_globstar: + prev_was_globstar = True + res.append(part) + elif _GLOBSTAR in part: # a/b**/c or a/**b/c + raise ValueError("Can only specify glob patterns of the form a/**/b") + else: + prev_was_globstar = False + res.append(part) + + return res + + +def _find_globstar(parts): + """Return the index of the first occurrence of "**" in 'parts'. + + Return -1 if "**" is not found in the list. + """ + + for (idx, part) in enumerate(parts): + if part == _GLOBSTAR: + return idx + return -1 + + +def _list_dir(pathname): + """Return a pair of subdirectory names and filenames contained within the 'pathname' directory. + + If 'pathname' does not exist, then None is returned. + """ + + try: + (_root, dirs, files) = next(os.walk(pathname)) + return (dirs, files) + except StopIteration: + return None # 'pathname' directory does not exist + + +def _expand(pathname): + """Emit tuples of the form ("dir", dirname) and ("file", filename). + + The result is for all directories and files contained within the 'pathname' directory. + """ + + res = _list_dir(pathname) + if res is None: + return + + (dirs, files) = res + + # Zero expansion + if os.path.basename(pathname): + yield ("dir", os.path.join(pathname, "")) + + for fname in files: + path = os.path.join(pathname, fname) + yield ("file", path) + + for dname in dirs: + path = os.path.join(pathname, dname) + for xpath in _expand(path): + yield xpath + + +def _expand_curdir(pathname): + """Emit tuples of the form ("dir", dirname) and ("file", filename). + + The result is for all directories and files contained within the 'pathname' directory. + + The returned pathnames omit a "./" prefix. + """ + + res = _list_dir(pathname) + if res is None: + return + + (dirs, files) = res + + # Zero expansion + yield ("dir", "") + + for fname in files: + yield ("file", fname) + + for dname in dirs: + for xdir in _expand(dname): + yield xdir diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py index d7c2c295492..7af28d11ed8 100644 --- a/buildscripts/resmokelib/utils/jscomment.py +++ b/buildscripts/resmokelib/utils/jscomment.py @@ -36,18 +36,6 @@ def get_tags(pathname): tags = yaml.safe_load(_strip_jscomments(match.group(1))) if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) - - for tag in tags: - if '//' in tag: - raise ValueError(("Found a JS line comment '%s'. "\ - "Use '#' YAML style comments instead in a tags array %s") - % (tag, pathname)) - - if ' ' in tag: - raise ValueError(("Found an empty space in tag '%s'. "\ - "This is not permitted and may indicate a missing comma in %s") - % (tag, pathname)) - return tags except yaml.YAMLError as err: raise ValueError( diff --git a/buildscripts/scons_metrics/metrics.py b/buildscripts/scons_metrics/metrics.py index 6e9949f770c..1ba1f7825ad 100644 --- a/buildscripts/scons_metrics/metrics.py +++ b/buildscripts/scons_metrics/metrics.py @@ -1,6 +1,5 @@ """SCons metrics.""" import re -import os from typing import Optional, NamedTuple, List, Pattern, AnyStr from buildscripts.util.cedar_report import CedarMetric, CedarTestReport @@ -106,11 +105,8 @@ class SconsMetrics: # pylint: disable=too-many-instance-attributes self.total_command_execution_time = self._parse_float( TOTAL_COMMAND_EXECUTION_TIME_REGEX, self.raw_report) - if os.path.exists(cache_debug_log_file): - with open(cache_debug_log_file, "r") as fh: - self.final_cache_hit_ratio = self._parse_float(CACHE_HIT_RATIO_REGEX, fh.read()) - else: - self.final_cache_hit_ratio = 0.0 + with open(cache_debug_log_file, "r") as fh: + self.final_cache_hit_ratio = self._parse_float(CACHE_HIT_RATIO_REGEX, fh.read()) def make_cedar_report(self) -> List[dict]: """Format the data to look like a cedar report json.""" diff --git a/buildscripts/scons_metrics/report.py b/buildscripts/scons_metrics/report.py index c016492c966..61b60ccc186 100644 --- a/buildscripts/scons_metrics/report.py +++ b/buildscripts/scons_metrics/report.py @@ -31,6 +31,10 @@ def main(scons_stdout_log_file: str, scons_cache_debug_log_file: str, print(f"Could not find SCons stdout log file '{scons_stdout_log_file}'.") sys.exit(1) + if not os.path.exists(scons_cache_debug_log_file): + print(f"Could not find SCons cache debug log file '{scons_cache_debug_log_file}'.") + sys.exit(1) + scons_metrics = SconsMetrics(scons_stdout_log_file, scons_cache_debug_log_file) if not scons_metrics.raw_report: print( diff --git a/buildscripts/setup_spawnhost_coredump b/buildscripts/setup_spawnhost_coredump index 11191d53c4a..865dae12282 100755 --- a/buildscripts/setup_spawnhost_coredump +++ b/buildscripts/setup_spawnhost_coredump @@ -61,7 +61,7 @@ else # Write this file that gets cat'ed on login to communicate to users logging in if this setup script is still running. echo '+-----------------------------------------------------------------------------------+' > ~/.setup_spawnhost_coredump_progress - echo "| The setup script is still setting up data files for inspection on a [${machine}] host. |" >> ~/.setup_spawnhost_coredump_progress + echo '| The setup script is still setting up data files for inspection on a [${machine}] host. |' >> ~/.setup_spawnhost_coredump_progress echo '+-----------------------------------------------------------------------------------+' >> ~/.setup_spawnhost_coredump_progress cat >> ~/.profile <<EOF @@ -157,44 +157,3 @@ EOF # paths/environment variables will be set as intended. wall "The setup_spawnhost_coredump script has completed, please relogin to ensure the right environment variables are set." fi - -# Send a Slack notification as the very last thing the setup_spawnhost_coredump script does. -# This way a Server engineer can temporarily forget about the Evergreen host they spawned until the -# paths and environment variables are configured as intended for when they first connect. -if [[ "${machine}" = "Cygwin" ]]; then - # The setup_spawnhost_coredump script runs as the mci-exec user on Windows hosts. However, - # Server engineers log in as the Administrator user. - ssh_user="Administrator" - # The Evergreen binary only expects a Windows path. The rest of Cygwin is flexible about it - # being a Cygwin path or a Windows path so we do the conversion here. - evg_credentials_pathname=$(cygpath -w ~Administrator/.evergreen.yml) - evg_binary_pathname=~Administrator/cli_bin/evergreen -else - ssh_user=$(whoami) - evg_credentials_pathname=~/.evergreen.yml - evg_binary_pathname=evergreen -fi - -slack_user=$(awk '{if ($1 == "user:") print $2}' "$evg_credentials_pathname") -# Refer to the https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html -# documentation for more information on the AWS instance metadata endpoints. -aws_metadata_svc="http://169.254.169.254" -aws_token=$(curl -s -X PUT "$aws_metadata_svc/latest/api/token" -H 'X-aws-ec2-metadata-token-ttl-seconds: 60') -ssh_host=$(curl -s -H "X-aws-ec2-metadata-token: $aws_token" "$aws_metadata_svc/latest/meta-data/public-hostname") -if [[ "${machine}" = "Cygwin" ]]; then - slack_message=$(printf "The setup_spawnhost_coredump script has finished setting things up. \ -Please use Windows Remote Desktop with\n\ -1. PC name: $ssh_host\n\ -2. User account: $ssh_user\n\ -3. The RDP password configured under the edit dialog at https://spruce.mongodb.com/spawn/host\n\ -to log in.") -else - slack_message="The setup_spawnhost_coredump script has finished setting things up. Please run "'```'"ssh $ssh_user@$ssh_host"'```'" to log in." -fi - -# The Evergreen spawn host is expected to be provisioned with the user's .evergreen.yml credentials. -# But in case something unexpected happens we don't want the setup_spawnhost_coredump script itself -# to error. -if [[ -n "${slack_user}" ]]; then - "$evg_binary_pathname" --config "$evg_credentials_pathname" notify slack -t "@$slack_user" -m "$slack_message" -fi diff --git a/buildscripts/sign_macos_binaries_for_testing.py b/buildscripts/sign_macos_binaries_for_testing.py deleted file mode 100644 index 360dd1b35d8..00000000000 --- a/buildscripts/sign_macos_binaries_for_testing.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Signs all of the known testing binaries with insecure development entitlements. - -Specifically the `Get Task Allow` is what we are looking for. -Adding the `Get Task Allow` entitlement allows us to attach to -the mongo processes and get core dumps/debug in any way we need. -You can view some more documentation on this topic here: -https://developer.apple.com/documentation/bundleresources/entitlements/com_apple_security_cs_debugger#discussion -""" - -import os -import subprocess -import sys - - -def main(): - """Signs all of the known testing binaries with insecure development entitlements.""" - if sys.platform != "darwin": - print("Non-macos system detected, do not need to sign binaries.") - sys.exit(0) - - build_bin_dir = os.path.join("build", "install", "bin") - binary_directories = ["/data/multiversion", os.path.join("dist-test", "bin"), build_bin_dir] - entitlements_file = os.path.abspath(os.path.join("etc", "macos_dev_entitlements.xml")) - assert os.path.exists(entitlements_file), f"{entitlements_file} does not exist" - - for binary_dir in binary_directories: - if not os.path.exists(binary_dir): - continue - - for binary in os.listdir(binary_dir): - binary_path = os.path.join(binary_dir, binary) - if not os.path.isfile(binary_path): - continue - - print(f"Signing {binary}") - subprocess.run([ - "/usr/bin/codesign", "-s", "-", "-f", "--entitlements", entitlements_file, - binary_path - ], check=True) - - -if __name__ == '__main__': - main() diff --git a/buildscripts/simple_report.py b/buildscripts/simple_report.py deleted file mode 100644 index da12326b562..00000000000 --- a/buildscripts/simple_report.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" -import json -import pathlib -import os -from typing import List, Dict, Optional -from typing_extensions import TypedDict -import click - - -class Result(TypedDict, total=False): - """Evergreen test result.""" - - status: str - exit_code: int - test_file: str - start: float - end: float - elapsed: float - log_raw: str - - -class Report(TypedDict): - """Evergreen report.""" - - failures: int - results: List[Result] - - -def _open_and_truncate_log_lines(log_file: pathlib.Path) -> List[str]: - with open(log_file) as fh: - lines = fh.read().splitlines() - for i, line in enumerate(lines): - if line == "scons: done reading SConscript files.": - offset = i - # if possible, also shave off the current and next line - # as they contain: - # scons: done reading SConscript files. - # scons: Building targets ... - # which is superfluous. - if len(lines) > i + 2: - offset = i + 2 - return lines[offset:] - - return lines - - -def _clean_log_file(log_file: pathlib.Path, dedup_lines: bool) -> str: - lines = _open_and_truncate_log_lines(log_file) - if dedup_lines: - lines = _dedup_lines(lines) - return os.linesep.join(lines) - - -def make_report(test_name: str, log_file_contents: str, exit_code: int) -> Report: # noqa: D103 - # pylint: disable=missing-function-docstring - status = "pass" if exit_code == 0 else "fail" - return Report({ - 'failures': - 0 if exit_code == 0 else 1, "results": [ - Result({ - "status": status, "exit_code": exit_code, "test_file": test_name, - "log_raw": log_file_contents - }) - ] - }) - - -def try_combine_reports(out: Report): # noqa: D103 - # pylint: disable=missing-function-docstring - try: - with open("report.json") as fh: - report = json.load(fh) - out["results"] += report["results"] - out["failures"] += report["failures"] - except NameError: - pass - except IOError: - pass - - -def _dedup_lines(lines: List[str]) -> List[str]: - return list(set(lines)) - - -def put_report(out: Report): # noqa: D103 - # pylint: disable=missing-function-docstring - with open("report.json", "w") as fh: - json.dump(out, fh) - - -@click.command() -@click.option("--test-name", required=True, type=str) -@click.option("--log-file", required=True, type=pathlib.Path) -@click.option("--exit-code", required=True, type=int) -@click.option("--dedup-lines", is_flag=True) -def main(test_name: str, log_file: pathlib.Path, exit_code: int, dedup_lines: bool): - """Given a test name, path to log file and exit code, generate/append an Evergreen report.json.""" - log_file_contents = _clean_log_file(log_file, dedup_lines) - report = make_report(test_name, log_file_contents, exit_code) - try_combine_reports(report) - put_report(report) - - -if __name__ == "__main__": - main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/sync_repo_with_copybara.py b/buildscripts/sync_repo_with_copybara.py deleted file mode 100644 index 37f9bf385bb..00000000000 --- a/buildscripts/sync_repo_with_copybara.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Module for syncing a repo with Copybara and setting up configurations.""" -import argparse -import subprocess -import os -import sys - -from typing import Optional -from github import GithubIntegration - -from buildscripts.util.read_config import read_config_file -from evergreen.api import RetryingEvergreenApi - - -def run_command(command): # noqa: D406 - """ - Execute a shell command and return its standard output (`stdout`). - - Args: - command (str): The shell command to be executed. - - Returns: - str: The standard output of the executed command. - - Raises: - subprocess.CalledProcessError: If the command execution fails. - - """ - try: - return subprocess.run(command, shell=True, check=True, text=True, - capture_output=True).stdout - except subprocess.CalledProcessError as error: - print(f"Error while executing: '{command}'.\n{error}\nStandard Error: {error.stderr}") - raise - - -def create_mongodb_bot_gitconfig(): - """Create the mongodb-bot.gitconfig file with the desired content.""" - - content = """ - [user] - name = MongoDB Bot - email = mongo-bot@mongodb.com - """ - - gitconfig_path = os.path.expanduser("~/mongodb-bot.gitconfig") - - with open(gitconfig_path, 'w') as file: - file.write(content) - - print("mongodb-bot.gitconfig file created.") - - -def get_installation_access_token(app_id: int, private_key: str, - installation_id: int) -> Optional[str]: # noqa: D406,D407,D413 - """ - Obtain an installation access token using JWT. - - Args: - - app_id (int): The application ID for GitHub App. - - private_key (str): The private key associated with the GitHub App. - - installation_id (int): The installation ID of the GitHub App for a particular account. - - Returns: - - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token. - - """ - integration = GithubIntegration(app_id, private_key) - auth = integration.get_access_token(installation_id) - - if auth: - return auth.token - else: - print("Error obtaining installation token") - return None - - -def send_failure_message_to_slack(expansions): - """ - Send a failure message to a specific Slack channel when the Copybara task fails. - - :param expansions: Dictionary containing various expansion data. - """ - current_version_id = expansions.get("version_id", None) - error_msg = ( - "Evergreen task '* Copybara Sync Between Repos' failed\n" - f"For more details: <https://spruce.mongodb.com/version/{current_version_id}|here>.") - - evg_api = RetryingEvergreenApi.get_api(config_file=".evergreen.yml") - evg_api.send_slack_message( - target="#sdp-triager", - msg=error_msg, - ) - - -def main(): - """Clone the Copybara repo, build its Docker image, and set up and run migrations.""" - parser = argparse.ArgumentParser() - parser.add_argument("--expansion-file", dest="expansion_file", type=str, - help="Location of expansions file generated by evergreen.") - - args = parser.parse_args() - - # Check if the copybara directory already exists - if os.path.exists('copybara'): - print("Copybara directory already exists.") - else: - run_command("git clone https://github.com/10gen/copybara.git") - - # Navigate to the Copybara directory and build the Copybara Docker image - run_command("cd copybara && docker build --rm -t copybara .") - - # Read configurations - expansions = read_config_file(args.expansion_file) - - access_token_copybara_syncer = get_installation_access_token( - expansions["app_id_copybara_syncer"], expansions["private_key_copybara_syncer"], - expansions["installation_id_copybara_syncer"]) - - # Create the mongodb-bot.gitconfig file as necessary. - create_mongodb_bot_gitconfig() - - current_dir = os.getcwd() - git_destination_url_with_token = f"https://x-access-token:{access_token_copybara_syncer}@github.com/mongodb/mongo.git" - - # Set up the Docker command and execute it - docker_cmd = [ - "docker run", - "-v ~/.ssh:/root/.ssh", - "-v ~/mongodb-bot.gitconfig:/root/.gitconfig", - f'-v "{current_dir}/copy.bara.sky":/usr/src/app/copy.bara.sky', - "-e COPYBARA_CONFIG='copy.bara.sky'", - "-e COPYBARA_SUBCOMMAND='migrate'", - f"-e COPYBARA_OPTIONS='-v --git-destination-url={git_destination_url_with_token}'", - "copybara copybara", - ] - - try: - run_command(" ".join(docker_cmd)) - except subprocess.CalledProcessError as err: - error_message = str(err.stderr) - acceptable_error_messages = [ - # Indicates the two repositories are identical - "No new changes to import for resolved ref", - # Indicates differences exist but no changes affect the destination, for example: exclusion rules - "Iterative workflow produced no changes in the destination for resolved ref", - ] - - if any(acceptable_message in error_message - for acceptable_message in acceptable_error_messages): - return - - # Send a failure message to #sdp-triager if the Copybara sync task fails. - send_failure_message_to_slack(expansions) - - -if __name__ == "__main__": - main() diff --git a/buildscripts/task_generation/suite_split.py b/buildscripts/task_generation/suite_split.py index a65fcae68f0..5e1e9d32115 100644 --- a/buildscripts/task_generation/suite_split.py +++ b/buildscripts/task_generation/suite_split.py @@ -201,15 +201,25 @@ class SuiteSplitService: if self.config.default_to_fallback: return self.calculate_fallback_suites(params) - evg_stats = HistoricTaskData.from_s3(self.config.evg_project, params.task_name, - params.build_variant) - - if evg_stats: + try: + evg_stats = HistoricTaskData.from_evg(self.evg_api, self.config.evg_project, + self.config.start_date, self.config.end_date, + params.task_name, params.build_variant) + if not evg_stats: + LOGGER.debug("No test history, using fallback suites") + # This is probably a new suite, since there is no test history, just use the + # fallback values. + return self.calculate_fallback_suites(params) return self.calculate_suites_from_evg_stats(evg_stats, params) - - LOGGER.debug("No test history, using fallback suites") - # Since there is no test history this is probably a new suite, just use the fallback values. - return self.calculate_fallback_suites(params) + except requests.HTTPError as err: + if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE: + # Evergreen may return a 503 when the service is degraded. + # We fall back to splitting the tests into a fixed number of suites. + LOGGER.warning("Received 503 from Evergreen, " + "dividing the tests evenly among suites") + return self.calculate_fallback_suites(params) + else: + raise def calculate_fallback_suites(self, params: SuiteSplitParameters) -> GeneratedSuite: """Divide tests into a fixed number of suites.""" diff --git a/buildscripts/testmatrix/getdisplaytaskname.py b/buildscripts/testmatrix/getdisplaytaskname.py index e8ff49d6a39..c0a08b422bc 100644 --- a/buildscripts/testmatrix/getdisplaytaskname.py +++ b/buildscripts/testmatrix/getdisplaytaskname.py @@ -1,7 +1,7 @@ """ Get the display task name from the execution task and the variant. -Get an execution task name like this: multiversion_auth_0_enterprise-rhel-8-64-bit-dynamic-all-feature-flags-required +Get an execution task name like this: multiversion_auth_0_enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required Into a display task name like this: multiversion_auth """ diff --git a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp index 24b0b42fed3..834d2052a98 100644 --- a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp +++ b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp @@ -27,9 +27,4 @@ LOGV2_ERROR(25, "more words"); LOGV2_ERROR(26, "words", - "comma, more words words words words words words words words words words words words "); -iassert(27, "words"); -iasserted(28, "words"); -iassertNoTrace(29, "words"); -iassertedNoTrace(30, "words"); -MONGO_UNREACHABLE_TASSERT(31); + "comma, more words words words words words words words words words words words words " diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 0cba8c8d82f..82434ee25c4 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -4,7 +4,6 @@ import logging import json import os import os.path -import subprocess import sys import time import unittest @@ -172,7 +171,7 @@ class TestTimeout(_ResmokeSelftest): ] self.execute_resmoke(resmoke_args) - archival_dirs_to_expect = 8 # 2 tests * 2 nodes + archival_dirs_to_expect = 4 # 2 tests * 2 nodes self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) @@ -190,11 +189,9 @@ class TestTimeout(_ResmokeSelftest): self.execute_resmoke(resmoke_args, sleep_secs=25) - archival_dirs_to_expect = 4 + archival_dirs_to_expect = 2 # 2 tests * 2 nodes / 2 data_file directories self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) - archival_inner_dirs_to_expect = 2 - self.assert_dir_file_count(self.test_dir_inner, self.archival_file, - archival_inner_dirs_to_expect) + self.assert_dir_file_count(self.test_dir_inner, self.archival_file, archival_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) self.assert_dir_file_count(self.test_dir, self.analysis_file, analysis_pids_to_expect) @@ -428,21 +425,3 @@ class TestSetParameters(_ResmokeSelftest): """--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}""", """--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}""" ]).wait()) - - -class TestMultiversionConfig(unittest.TestCase): - def test_valid_yaml(self): - file_name = "multiversion-config.yml" - subprocess.run([ - sys.executable, "buildscripts/resmoke.py", "multiversion-config", - "--config-file-output", file_name - ], check=True) - with open(file_name, "r") as file: - file_contents = file.read() - - try: - yaml.safe_load(file_contents) - except Exception: # pylint: disable=broad-except - self.fail(msg="`resmoke.py multiversion-config` does not output valid yaml.") - - os.remove(file_name) diff --git a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py index 274b7e5ffbe..2f685f36c97 100644 --- a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py +++ b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py @@ -38,8 +38,6 @@ class TestCalculateFcvConstants(TestCase): "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0"], - "eolVersions": - ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -47,15 +45,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() - self.assertEqual(version_constants.latest, Version("6.0")) - self.assertEqual(version_constants.last_continuous, Version("5.3")) - self.assertEqual(version_constants.last_lts, Version("5.0")) - self.assertEqual(version_constants.requires_fcv_tag_list, + self.assertEqual(fcv_constants.latest, Version("6.0")) + self.assertEqual(fcv_constants.last_continuous, Version("5.3")) + self.assertEqual(fcv_constants.last_lts, Version("5.0")) + self.assertEqual(fcv_constants.requires_fcv_tag_list, [Version(v) for v in ["5.1", "5.2", "5.3", "6.0"]]) - self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) - self.assertEqual(version_constants.fcvs_less_than_latest, [ + self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) + self.assertEqual(fcv_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3"] ]) @@ -69,8 +67,6 @@ class TestCalculateFcvConstants(TestCase): "6.1", "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0", "6.0"], - "eolVersions": - ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -78,15 +74,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() - self.assertEqual(version_constants.latest, Version("100.0")) - self.assertEqual(version_constants.last_continuous, Version("6.1")) - self.assertEqual(version_constants.last_lts, Version("6.0")) - self.assertEqual(version_constants.requires_fcv_tag_list, + self.assertEqual(fcv_constants.latest, Version("100.0")) + self.assertEqual(fcv_constants.last_continuous, Version("6.1")) + self.assertEqual(fcv_constants.last_lts, Version("6.0")) + self.assertEqual(fcv_constants.requires_fcv_tag_list, [Version(v) for v in ["6.1", "100.0"]]) - self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) - self.assertEqual(version_constants.fcvs_less_than_latest, [ + self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) + self.assertEqual(fcv_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1"] ]) diff --git a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py index ec98f233840..decc81f4433 100644 --- a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py +++ b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py @@ -11,7 +11,6 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags as un from buildscripts.util.fileops import read_yaml_file EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml" -EXPANSIONS_FILE = "expansions.yml" class TestGenerateExcludeYaml(unittest.TestCase): @@ -43,7 +42,6 @@ class TestGenerateExcludeYaml(unittest.TestCase): output = os.path.join(self._tmpdir.name, EXCLUDE_TAGS_FILE) under_test.generate_exclude_yaml(old_bin_version=old_bin_version, output=output, - expansions_file=EXPANSIONS_FILE, logger=MagicMock()) mock_read_yaml.assert_called_once() diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py index c1ba97c6056..b9fc74f088b 100644 --- a/buildscripts/tests/resmokelib/test_parser.py +++ b/buildscripts/tests/resmokelib/test_parser.py @@ -248,6 +248,7 @@ class TestLocalCommandLine(unittest.TestCase): cmdline = to_local_args([ "run", "--suites=my_suite", + "--reportFailureStatus=fail", "--reportFile=report.json", "--perfReportFile=perf.json", "--storageEngine=my_storage_engine", diff --git a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py b/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py deleted file mode 100644 index caeb81e1e4b..00000000000 --- a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Unit tests for buildscripts/resmokelib/testing/symbolizer_service.py.""" -# pylint: disable=missing-docstring -import os -import unittest -from pathlib import Path -from tempfile import TemporaryDirectory -from unittest.mock import MagicMock - -from buildscripts.resmokelib.testing import symbolizer_service as under_test - - -def mock_resmoke_symbolizer_config(): - config_mock: under_test.ResmokeSymbolizerConfig = MagicMock( - spec_set=under_test.ResmokeSymbolizerConfig) - config_mock.evg_task_id = "evg_task_id" - config_mock.client_id = "client_id" - config_mock.client_secret = "client_secret" - config_mock.is_windows.return_value = False - config_mock.is_macos.return_value = False - return config_mock - - -class TestResmokeSymbolizer(unittest.TestCase): - def setUp(self) -> None: - self.config_mock = mock_resmoke_symbolizer_config() - self.symbolizer_service_mock: under_test.SymbolizerService = MagicMock( - spec_set=under_test.SymbolizerService) - self.file_service_mock: under_test.FileService = MagicMock(spec_set=under_test.FileService) - self.resmoke_symbolizer = under_test.ResmokeSymbolizer( - self.config_mock, self.symbolizer_service_mock, self.file_service_mock) - - def test_symbolize_test_logs_process_all_files(self): - stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] - self.file_service_mock.filter_out_non_files.return_value = stacktrace_files - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - - self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 5) - for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): - self.assertEqual(call.args[0], f"file{i}.stacktrace") - self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) - - def test_symbolize_test_logs_hit_timeout(self): - stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] - self.file_service_mock.filter_out_non_files.return_value = stacktrace_files - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock(), 0) - - self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 1) - for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): - self.assertEqual(call.args[0], f"file{i}.stacktrace") - self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) - - def test_symbolize_test_logs_should_not_symbolize(self): - self.config_mock.is_windows.return_value = True - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_symbolize_test_logs_could_not_get_dbpath(self): - self.file_service_mock.check_path_exists.return_value = False - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_symbolize_test_logs_did_not_find_files(self): - self.file_service_mock.find_all_children_recursively.return_value = [] - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_should_not_symbolize_if_not_in_evergreen(self): - self.config_mock.evg_task_id = None - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_secrets_are_absent(self): - self.config_mock.client_id = None - self.config_mock.client_secret = None - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_on_windows(self): - self.config_mock.is_windows.return_value = True - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_on_macos(self): - self.config_mock.is_macos.return_value = True - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_symbolize_return_true(self): - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertTrue(ret) - - def test_get_stacktrace_dir_returns_dir(self): - dbpath = "dbpath" - test = MagicMock(fixture=MagicMock(get_dbpath_prefix=MagicMock(return_value=dbpath))) - self.file_service_mock.check_path_exists.return_value = True - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, dbpath) - - def test_get_stacktrace_dir_if_dir_does_not_exist(self): - test = MagicMock() - self.file_service_mock.check_path_exists.return_value = False - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, None) - - def test_get_stacktrace_dir_if_fixture_is_not_available(self): - test = MagicMock(fixture=None) - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, None) - - -class TestFileService(unittest.TestCase): - def setUp(self) -> None: - self.file_service = under_test.FileService() - self.relative_dir_paths = [ - os.path.join("dir_1"), - os.path.join("dir_2", "dir_2_1"), - os.path.join("dir_2", "dir_2_2"), - os.path.join("dir_3"), - ] - self.relative_file_paths = [ - os.path.join("dir_1", "file_1.stacktrace"), - os.path.join("dir_2", "file_2.stacktrace"), - os.path.join("dir_2", "dir_2_1", "file_3.stacktrace"), - os.path.join("dir_2", "dir_2_2", "file_4.stacktrace"), - ] - - def test_find_all_children_recursively_returns_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(sorted(ret), sorted(abs_file_paths)) - - def test_find_all_children_recursively_no_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(ret, []) - - def test_find_all_children_recursively_no_dirs(self): - with TemporaryDirectory() as tmpdir: - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(ret, []) - - def test_filter_out_non_files_if_all_files_absent(self): - with TemporaryDirectory() as tmpdir: - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, []) - - def test_filter_out_non_files_if_some_files_present(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - Path(abs_file_paths[1]).touch() - Path(abs_file_paths[3]).touch() - - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, [abs_file_paths[1], abs_file_paths[3]]) - - def test_filter_out_non_files_if_all_files_present(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, abs_file_paths) - - def test_remove_empty_files_if_no_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - with open(file, "w") as fstream: - fstream.write("stacktrace") - - self.file_service.remove_empty(abs_file_paths) - for file in abs_file_paths: - self.assertTrue(os.path.exists(file)) - - def test_remove_empty_files_if_partly_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - with open(abs_file_paths[0], "w") as fstream: - fstream.write("stacktrace") - Path(abs_file_paths[1]).touch() - with open(abs_file_paths[2], "w") as fstream: - fstream.write("stacktrace") - Path(abs_file_paths[3]).touch() - - self.file_service.remove_empty(abs_file_paths) - - self.assertTrue(os.path.exists(abs_file_paths[0])) - self.assertFalse(os.path.exists(abs_file_paths[1])) - self.assertTrue(os.path.exists(abs_file_paths[2])) - self.assertFalse(os.path.exists(abs_file_paths[3])) - - def test_remove_empty_files_if_all_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - self.file_service.remove_empty(abs_file_paths) - for file in abs_file_paths: - self.assertFalse(os.path.exists(file)) - - def test_remove_all_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - with open(file, "w") as fstream: - fstream.write("stacktrace") - - self.file_service.remove_all(abs_file_paths) - for file in abs_file_paths: - self.assertFalse(os.path.exists(file)) diff --git a/buildscripts/tests/task_generation/test_suite_split.py b/buildscripts/tests/task_generation/test_suite_split.py index b2b197ed7c5..7c54ed489aa 100644 --- a/buildscripts/tests/task_generation/test_suite_split.py +++ b/buildscripts/tests/task_generation/test_suite_split.py @@ -8,7 +8,7 @@ import requests import buildscripts.task_generation.suite_split as under_test from buildscripts.task_generation.suite_split_strategies import greedy_division, \ round_robin_fallback -from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation +from buildscripts.util.teststats import TestRuntime # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access @@ -31,12 +31,7 @@ def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None): def tst_stat_mock(file, duration, pass_count): - return HistoricalTestInformation( - test_name=file, - num_pass=pass_count, - num_fail=0, - avg_duration_pass=duration, - ) + return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None): @@ -120,16 +115,15 @@ class TestGeneratedSuite(unittest.TestCase): class TestSplitSuite(unittest.TestCase): - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites(self, get_stats_from_s3_mock): + def test_calculate_suites(self): mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_name for stat in mock_test_stats + stat.test_file for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} @@ -143,15 +137,32 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(10, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock): + def test_calculate_suites_fallback_on_error(self): + n_tests = 100 + max_sub_suites = 4 + split_config = build_mock_split_config(max_sub_suites=max_sub_suites) + split_params = build_mock_split_params() + + suite_split_service = build_mock_service(split_config=split_config) + mock_evg_error(suite_split_service.evg_api) + suite_split_service.resmoke_proxy.list_tests.return_value = [ + f"test_{i}.js" for i in range(n_tests) + ] + + suite = suite_split_service.split_suite(split_params) + + self.assertEqual(max_sub_suites, len(suite)) + for sub_suite in suite.sub_suites: + self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) + + def test_calculate_suites_uses_fallback_on_no_results(self): n_tests = 100 max_sub_suites = 5 split_config = build_mock_split_config(max_sub_suites=max_sub_suites) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = [] + suite_split_service.evg_api.test_stats_by_project.return_value = [] suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -162,9 +173,7 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_uses_fallback_if_only_results_are_filtered( - self, get_stats_from_s3_mock): + def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self): n_tests = 100 max_sub_suites = 10 mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] @@ -173,7 +182,7 @@ class TestSplitSuite(unittest.TestCase): split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -189,17 +198,31 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock): + def test_calculate_suites_fail_on_unexpected_error(self): + n_tests = 100 + max_sub_suites = 4 + split_config = build_mock_split_config(max_sub_suites=max_sub_suites) + split_params = build_mock_split_params() + + suite_split_service = build_mock_service(split_config=split_config) + mock_evg_error(suite_split_service.evg_api, error_code=requests.codes.INTERNAL_SERVER_ERROR) + suite_split_service.resmoke_proxy.list_tests.return_value = [ + f"test_{i}.js" for i in range(n_tests) + ] + + with self.assertRaises(requests.HTTPError): + suite_split_service.split_suite(split_params) + + def test_calculate_suites_will_filter_specified_tests(self): mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params( test_filter=lambda t: t in {"test_1.js", "test_2.js"}) suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_name for stat in mock_test_stats + stat.test_file for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py new file mode 100644 index 00000000000..ec53a02d161 --- /dev/null +++ b/buildscripts/tests/test_burn_in_tags.py @@ -0,0 +1,261 @@ +"""Unit tests for the burn_in_tags.py script.""" +from collections import defaultdict +import json +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +from shrub.v2 import ShrubProject + +import buildscripts.ciconfig.evergreen as _evergreen +from buildscripts.burn_in_tests import TaskInfo +from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns +from buildscripts.ciconfig.evergreen import EvergreenProjectConfig + +import buildscripts.burn_in_tags as under_test + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access + +EMPTY_PROJECT = { + "buildvariants": [], + "tasks": [], +} +TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "test_burn_in_tags_evergreen.yml") + +NS = "buildscripts.burn_in_tags" + + +def ns(relative_name): # pylint: disable-invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + + +def get_expansions_data(): + return { + "branch_name": "fake_branch", + "build_variant": "enterprise-rhel-80-64-bit-suggested", + "check_evergreen": 2, + "distro_id": "rhel80-small", + "is_patch": "true", + "max_revisions": 25, + "repeat_tests_max": 1000, + "repeat_tests_min": 2, + "repeat_tests_secs": 600, + "revision": "fake_sha", + "project": "fake_project", + "task_id": "task id", + } # yapf: disable + + +def get_evergreen_config() -> EvergreenProjectConfig: + return _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) + + +class TestCreateEvgBuildVariantMap(unittest.TestCase): + def test_create_evg_buildvariant_map(self): + expansions_file_data = { + "build_variant": "variant1", "burn_in_tag_buildvariants": "variant2 variant3" + } + buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data) + + expected_buildvariant_map = { + "variant2": "variant2-required", "variant3": "variant3-required" + } + self.assertEqual(buildvariant_map, expected_buildvariant_map) + + +class TestGenerateEvgBuildVariants(unittest.TestCase): + def test_generate_evg_buildvariant_one_base_variant(self): + evg_conf_mock = get_evergreen_config() + base_variant = "enterprise-rhel-80-64-bit-inmem" + generated_variant = "enterprise-rhel-80-64-bit-inmem-required" + burn_in_tags_gen_variant = "enterprise-rhel-80-64-bit" + variant = evg_conf_mock.get_variant(base_variant) + + build_variant = under_test._generate_evg_build_variant(variant, generated_variant, + burn_in_tags_gen_variant) + + generated_build_variant = build_variant.as_dict() + self.assertEqual(generated_build_variant["name"], generated_variant) + self.assertEqual(generated_build_variant["modules"], variant.modules) + generated_expansions = generated_build_variant["expansions"] + burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass") + self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant) + self.assertEqual(generated_expansions, variant.expansions) + + +class TestGenerateEvgTasks(unittest.TestCase): + @patch(ns("create_tests_by_task")) + def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock): + evg_conf_mock = get_evergreen_config() + create_tests_by_task_mock.return_value = {} + expansions_file_data = get_expansions_data() + buildvariant_map = { + "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", + "enterprise-rhel-80-64-bit-majority-read-concern-off": + "enterprise-rhel-80-64-bit-majority-read-concern-off-required", + } # yapf: disable + shrub_config = ShrubProject() + evergreen_api = MagicMock() + repo = MagicMock(working_dir=os.getcwd()) + under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, + buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') + + self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT) + + @patch(ns("create_tests_by_task")) + def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock): + evg_conf_mock = get_evergreen_config() + create_tests_by_task_mock.return_value = { + "aggregation_mongos_passthrough": TaskInfo( + display_task_name="aggregation_mongos_passthrough", + suite="aggregation_mongos_passthrough", + resmoke_args="--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger", + tests=["jstests/aggregation/ifnull.js"], + require_multiversion_setup=False, + distro="", + build_variant="enterprise-rhel-80-64-bit-inmem" + ) + } # yapf: disable + expansions_file_data = get_expansions_data() + buildvariant_map = { + "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", + "enterprise-rhel-80-64-bit-majority-read-concern-off": + "enterprise-rhel-80-64-bit-majority-read-concern-off-required", + } # yapf: disable + shrub_config = ShrubProject.empty() + evergreen_api = MagicMock() + repo = MagicMock(working_dir=os.getcwd()) + evergreen_api.test_stats_by_project.return_value = [ + MagicMock(test_file="dir/test2.js", avg_duration_pass=10) + ] + under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, + buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') + + generated_config = shrub_config.as_dict() + self.assertEqual(len(generated_config["buildvariants"]), 2) + first_generated_build_variant = generated_config["buildvariants"][0] + self.assertIn(first_generated_build_variant["name"], buildvariant_map.values()) + self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests") + self.assertEqual( + first_generated_build_variant["display_tasks"][0]["execution_tasks"][0], + f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}") + + +EXPANSIONS_FILE_DATA = { + "build_variant": "enterprise-rhel-80-64-bit", + "revision": "badf00d000000000000000000000000000000000", "max_revisions": "1000", + "branch_name": "mongodb-mongo-master", "is_patch": "false", "distro_id": "rhel62-small", + "repeat_tests_min": "2", "repeat_tests_max": "1000", "repeat_tests_secs": "600", "project": + "mongodb-mongo-master", "task_id": "task id" +} + +CREATE_EVG_BUILD_VARIANT_MAP = { + 'enterprise-rhel-80-64-bit-majority-read-concern-off': + 'enterprise-rhel-80-64-bit-majority-read-concern-off-required', + 'enterprise-rhel-80-64-bit-inmem': + 'enterprise-rhel-80-64-bit-inmem-required' +} + +CREATE_TEST_MEMBERSHIP_MAP = { + "jstests/aggregation/accumulators/accumulator_js.js": [ + "aggregation", "aggregation_auth", "aggregation_disabled_optimization", "aggregation_ese", + "aggregation_ese_gcm", "aggregation_facet_unwind_passthrough", + "aggregation_mongos_passthrough", "aggregation_one_shard_sharded_collections", + "aggregation_read_concern_majority_passthrough", "aggregation_secondary_reads", + "aggregation_sharded_collections_passthrough" + ], "jstests/core/create_collection.js": [ + "core", "core_auth", "core_ese", "core_ese_gcm", "core_minimum_batch_size", "core_op_query", + "cwrwc_passthrough", "cwrwc_rc_majority_passthrough", "cwrwc_wc_majority_passthrough", + "logical_session_cache_replication_100ms_refresh_jscore_passthrough", + "logical_session_cache_replication_10sec_refresh_jscore_passthrough", + "logical_session_cache_replication_1sec_refresh_jscore_passthrough", + "logical_session_cache_replication_default_refresh_jscore_passthrough", + "logical_session_cache_standalone_100ms_refresh_jscore_passthrough", + "logical_session_cache_standalone_10sec_refresh_jscore_passthrough", + "logical_session_cache_standalone_1sec_refresh_jscore_passthrough", + "logical_session_cache_standalone_default_refresh_jscore_passthrough", + "read_concern_linearizable_passthrough", "read_concern_majority_passthrough", + "causally_consistent_read_concern_snapshot_passthrough", + "replica_sets_initsync_jscore_passthrough", "replica_sets_fcbis_jscore_passthrough", + "replica_sets_initsync_static_jscore_passthrough", "replica_sets_jscore_passthrough", + "replica_sets_kill_primary_jscore_passthrough", + "replica_sets_kill_secondaries_jscore_passthrough", + "replica_sets_reconfig_jscore_passthrough", + "replica_sets_terminate_primary_jscore_passthrough", "retryable_writes_jscore_passthrough", + "retryable_writes_jscore_stepdown_passthrough", "secondary_reads_passthrough", + "session_jscore_passthrough", "write_concern_majority_passthrough" + ] +} + + +class TestAcceptance(unittest.TestCase): + @patch(ns("write_file_to_dir")) + @patch(ns("_create_evg_build_variant_map")) + @patch(ns("EvergreenFileChangeDetector")) + def test_no_tests_run_if_none_changed(self, find_changed_tests_mock, + create_evg_build_variant_map_mock, write_to_file_mock): + """ + Given a git repository with no changes, + When burn_in_tags is run, + Then no tests are discovered to run. + """ + repos = [MagicMock(working_dir=os.getcwd())] + evg_conf_mock = MagicMock() + find_changed_tests_mock.return_value.find_changed_tests.return_value = {} + + create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP + + under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf_mock, MagicMock(), repos, + 'install_dir/bin') + + write_to_file_mock.assert_called_once() + shrub_config = write_to_file_mock.call_args[0][2] + self.assertEqual(EMPTY_PROJECT, json.loads(shrub_config)) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("write_file_to_dir")) + @patch(ns("_create_evg_build_variant_map")) + @patch(ns("EvergreenFileChangeDetector")) + @patch(burn_in_tests_ns("create_test_membership_map")) + def test_tests_generated_if_a_file_changed( + self, create_test_membership_map_mock, find_changed_tests_mock, + create_evg_build_variant_map_mock, write_to_file_mock): + """ + Given a git repository with changes, + When burn_in_tags is run, + Then some tags are discovered to run. + """ + create_test_membership_map_mock.return_value = defaultdict(list, CREATE_TEST_MEMBERSHIP_MAP) + + repos = [MagicMock(working_dir=os.getcwd())] + evg_conf = get_evergreen_config() + create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP + find_changed_tests_mock.return_value.find_changed_tests.return_value = { + 'jstests/slow1/large_role_chain.js', + 'jstests/aggregation/accumulators/accumulator_js.js' + } + + under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin') + + write_to_file_mock.assert_called_once() + written_config = write_to_file_mock.call_args[0][2] + written_config_map = json.loads(written_config) + + n_tasks = len(written_config_map["tasks"]) + # Ensure we are generating at least one task for the test. + self.assertGreaterEqual(n_tasks, 1) + + written_build_variants = written_config_map["buildvariants"] + written_build_variants_name = [variant['name'] for variant in written_build_variants] + self.assertEqual( + set(CREATE_EVG_BUILD_VARIANT_MAP.values()), set(written_build_variants_name)) + + tasks = written_config_map["tasks"] + self.assertGreaterEqual(len(tasks), len(CREATE_EVG_BUILD_VARIANT_MAP)) + + self.assertTrue( + all( + len(display_tasks) == 1 for display_tasks in + [build_variant["display_tasks"] for build_variant in written_build_variants])) diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml new file mode 100644 index 00000000000..d6a651fcde8 --- /dev/null +++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml @@ -0,0 +1,138 @@ +functions: + "fetch source": + - command: git.get_project + params: + directory: src + - command: shell.exec + params: + working_dir: src + script: | + echo "this is a 2nd command in the function!" + ls + +tasks: +- name: compile + depends_on: [] + commands: + - func: "fetch source" +- name: burn_in_tags_gen + depends_on: [] + commands: + - func: "fake command" +- name: compile_all_run_unittests_TG + depends_on: [] + commands: + - func: "fake command" +- name: clang_tidy_TG + depends_on: [] + commands: + - func: "fake command" +- name: stitch_support_lib_build_and_archive + depends_on: [] + commands: + - func: "fake command" +- name: lint_pylinters + depends_on: [] + commands: + - func: "fake command" +- name: lint_clang_format + depends_on: [] + commands: + - func: "fake command" +- name: burn_in_tests_gen + depends_on: [] + commands: + - func: "fake command" +- name: aggregation_multiversion_fuzzer_gen + depends_on: [] + commands: + - func: "generate resmoke tasks" +- name: aggregation_expression_multiversion_fuzzer_gen + depends_on: [] + commands: + - func: "generate resmoke tasks" +- name: aggregation + depends_on: + - name: compile + commands: + - func: run tests + vars: + resmoke_args: --suites=aggregation --storageEngine=wiredTiger + +buildvariants: +- name: enterprise-rhel-80-64-bit + display_name: "! Enterprise RHEL 8.0" + expansions: + multiversion_platform: rhel80 + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: lint_pylinters + - name: burn_in_tests_gen + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation + - name: burn_in_tags_gen +- name: buildvariant-without-burn-in-tag-buildvariants + display_name: "Buildvariant without burn in tag buildvariants expansion" + expansions: + multiversion_platform: rhel80 + tasks: + - name: burn_in_tags_gen +- name: enterprise-rhel-80-64-bit-majority-read-concern-off + display_name: "Enterprise RHEL 8.0 (majority read concern off)" + modules: ["enterprise"] + run_on: + - rhel80-small + expansions: &enterprise-rhel-80-64-bit-majority-read-concern-off-expansions + multiversion_edition: enterprise + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation +- name: enterprise-rhel-80-64-bit-inmem + display_name: Enterprise RHEL 8.0 (inMemory) + modules: ["enterprise"] + run_on: + - rhel80-small + expansions: &enterprise-rhel-80-64-bit-inmem-expansions + test_flags: >- + --majorityReadConcern=off + --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime + compile_flags: >- + -j$(grep -c ^processor /proc/cpuinfo) + --ssl + --release + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + MONGO_DISTMOD=rhel80 + multiversion_platform: rhel80 + multiversion_edition: enterprise + scons_cache_scope: shared + tooltags: "ssl sasl gssapi" + large_distro_name: rhel80-large + tasks: + - name: compile +- name: enterprise-rhel-80-64-bit-inmem + display_name: Enterprise RHEL 8.0 (inMemory) + expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + large_distro_name: rhel80-large + multiversion_edition: enterprise + multiversion_platform: rhel80 + scons_cache_scope: shared + test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling + modules: + - enterprise + run_on: + - rhel80-small + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 369131eff60..c51f8c60ecf 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,14 +4,12 @@ from __future__ import absolute_import import collections import datetime -from io import StringIO import os import sys import subprocess import unittest from mock import Mock, patch, MagicMock -import yaml import buildscripts.burn_in_tests as under_test from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask @@ -558,19 +556,3 @@ class TestLocalFileChangeDetector(unittest.TestCase): self.assertIn(file_list[2], found_tests) self.assertNotIn(file_list[1], found_tests) self.assertEqual(2, len(found_tests)) - - -class TestYamlBurnInExecutor(unittest.TestCase): - @patch('sys.stdout', new_callable=StringIO) - def test_found_tasks_should_be_reported_as_yaml(self, stdout): - n_tasks = 5 - n_tests = 3 - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - - yaml_executor = under_test.YamlBurnInExecutor() - yaml_executor.execute(tests_by_task) - - yaml_raw = stdout.getvalue() - results = yaml.safe_load(yaml_raw) - self.assertEqual(n_tasks, len(results["discovered_tasks"])) - self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) diff --git a/buildscripts/tests/test_debugsymb_mapper.py b/buildscripts/tests/test_debugsymb_mapper.py deleted file mode 100644 index b1b063f5331..00000000000 --- a/buildscripts/tests/test_debugsymb_mapper.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Unit tests for debugsymb_mapper.py.""" -# pylint: disable=missing-docstring -import unittest -from unittest.mock import MagicMock - -import buildscripts.debugsymb_mapper as under_test - - -def mock_cmd_client(): - cmd_client = MagicMock(spec_set=under_test.CmdClient) - return cmd_client - - -class TestCmdOutputExtractor(unittest.TestCase): - def setUp(self): - self.cmd_client_mock = mock_cmd_client() - self.cmd_output_extractor = under_test.CmdOutputExtractor(self.cmd_client_mock) - - -class TestGetBuildId(TestCmdOutputExtractor): - def test_get_build_id_returns_build_id(self): - readelf_output = ( - "Displaying notes found in: .note.gnu.build-id\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot\n" - " Owner Data size\tDescription\n" - " GA$<version>3h864 0x00000010\tOPEN\n" - " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)\n" - " GA$<version>3h864 0x00000010\tOPEN\n" - " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)") - self.cmd_client_mock.run.return_value = readelf_output - - build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") - self.assertEqual(build_id_output.build_id, "74c2322104428836f3d94af6cd7471ee7cb5c4ee") - self.assertEqual(build_id_output.cmd_output, readelf_output) - - def test_get_build_id_raises_error(self): - readelf_output = ( - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot") - self.cmd_client_mock.run.return_value = readelf_output - - self.assertRaises(ValueError, self.cmd_output_extractor.get_build_id, "path/to/bin") - - def test_get_build_id_returns_none(self): - readelf_output = ( - "Displaying notes found in: .note.gnu.build-id\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)") - self.cmd_client_mock.run.return_value = readelf_output - - build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") - self.assertIsNone(build_id_output.build_id) - self.assertEqual(build_id_output.cmd_output, readelf_output) - - -class TestGetBinVersion(TestCmdOutputExtractor): - def test_get_bin_version_returns_version(self): - # Newer versions command output - version_cmd_output = ('db version v4.4.14-25-gb0475e2\n' - 'Build Info: {\n' - ' "version": "4.4.14-25-gb0475e2",\n' - ' "gitVersion": "b0475e2657c3351b25499971d3340f054ea85b98",\n' - ' "openSSLVersion": "OpenSSL 1.1.1 11 Sep 2018",\n' - ' "modules": [\n' - ' "enterprise"\n' - ' ],\n' - ' "allocator": "tcmalloc",\n' - ' "environment": {\n' - ' "distmod": "ubuntu1804",\n' - ' "distarch": "x86_64",\n' - ' "target_arch": "x86_64"\n' - ' }\n' - '}') - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertEqual(bin_version_output.mongodb_version, "4.4.14-25-gb0475e2") - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) - - def test_get_bin_version_unsupported_output(self): - # Versions prior to 5.0 are not supported - version_cmd_output = ('db version v4.2.20-7-g5a81409\n' - 'git version: 5a81409faf16f30f1189af6367eb3ceee50a02b5\n' - 'OpenSSL version: OpenSSL 1.1.1 11 Sep 2018\n' - 'allocator: tcmalloc\n' - 'modules: enterprise \n' - 'build environment:\n' - ' distmod: ubuntu1804\n' - ' distarch: x86_64\n' - ' target_arch: x86_64') - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertIsNone(bin_version_output.mongodb_version) - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) - - def test_get_bin_version_returns_none(self): - version_cmd_output = "error: unrecognized arguments: --version" - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertIsNone(bin_version_output.mongodb_version) - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) diff --git a/buildscripts/tests/test_errorcodes.py b/buildscripts/tests/test_errorcodes.py index 17c102454b5..2a9c9ce1e3f 100644 --- a/buildscripts/tests/test_errorcodes.py +++ b/buildscripts/tests/test_errorcodes.py @@ -26,7 +26,7 @@ class TestErrorcodes(unittest.TestCase): captured_error_codes.append(code) errorcodes.parse_source_files(accumulate_files, TESTDATA_DIR + 'regex_matching/') - self.assertEqual(31, len(captured_error_codes)) + self.assertEqual(26, len(captured_error_codes)) def test_dup_checking(self): """Test dup checking.""" diff --git a/buildscripts/tests/test_evergreen_activate_gen_tasks.py b/buildscripts/tests/test_evergreen_activate_gen_tasks.py index 5470828c6c3..dbb64380838 100644 --- a/buildscripts/tests/test_evergreen_activate_gen_tasks.py +++ b/buildscripts/tests/test_evergreen_activate_gen_tasks.py @@ -1,126 +1,42 @@ """Unit tests for the generate_resmoke_suite script.""" -# pylint: disable=invalid-name import unittest -from mock import MagicMock, mock +from mock import MagicMock from buildscripts import evergreen_activate_gen_tasks as under_test -from evergreen import Build, EvergreenApi, Task, Version # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access # pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter -def build_mock_task(display_name, task_id): - mock_task = MagicMock(spec_set=Task, display_name=display_name, task_id=task_id) +def build_mock_task(name, task_id): + mock_task = MagicMock(display_name=name, task_id=task_id) return mock_task -def build_mock_task_list(num_tasks): - return [build_mock_task(f"task_{i}", f"id_{i}") for i in range(num_tasks)] - - -class MockVariantData(): - """An object to help create a mock evg api.""" - - def __init__(self, build_id, variant_name, task_list): - self.build_id = build_id - self.variant_name = variant_name - self.task_list = task_list - - -def build_mock_evg_api(variant_data_list): - class VersionPatchedSpec(Version): - """A patched `Version` with instance properties included for magic mock spec.""" - build_variants_map = MagicMock() - - mock_version = MagicMock(spec_set=VersionPatchedSpec) - mock_version.build_variants_map = { - variant_data.variant_name: variant_data.build_id - for variant_data in variant_data_list - } - - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_evg_api.version_by_id.return_value = mock_version - - build_id_mapping = { - variant_data.build_id: variant_data.task_list - for variant_data in variant_data_list - } - - def tasks_by_build_side_effect(build_id): - return build_id_mapping[build_id] - - mock_evg_api.tasks_by_build.side_effect = tasks_by_build_side_effect +def build_mock_evg_api(mock_task_list): + mock_build = MagicMock() + mock_build.get_tasks.return_value = mock_task_list + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value = mock_build return mock_evg_api class TestActivateTask(unittest.TestCase): def test_task_with_display_name_is_activated(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "task_3_gen", - }) - mock_task_list = build_mock_task_list(5) - mock_evg_api = build_mock_evg_api( - [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) - under_test.activate_task(expansions, mock_evg_api) + under_test.activate_task("build_id", "task_3", mock_evg_api) mock_evg_api.configure_task.assert_called_with("id_3", activated=True) def test_task_with_no_matching_name(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "not_an_existing_task", - }) - mock_task_list = build_mock_task_list(5) - mock_evg_api = build_mock_evg_api( - [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) - under_test.activate_task(expansions, mock_evg_api) + under_test.activate_task("build_id", "not_an_existing_task", mock_evg_api) mock_evg_api.configure_task.assert_not_called() - - def test_burn_in_tags_tasks_are_activated(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "burn_in_tags_gen", - }) - mock_task_list_2 = build_mock_task_list(5) - mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) - mock_task_list_3 = build_mock_task_list(5) - mock_task_list_3.append(build_mock_task("burn_in_tests", "burn_in_tests_id_3")) - mock_evg_api = build_mock_evg_api([ - MockVariantData("1", "variant1-generated-by-burn-in-tags", mock_task_list_2), - MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_3) - ]) - - under_test.activate_task(expansions, mock_evg_api) - - mock_evg_api.configure_task.assert_has_calls([ - mock.call("burn_in_tests_id_2", activated=True), - mock.call("burn_in_tests_id_3", activated=True) - ]) - - def test_burn_in_tags_task_skips_non_existing_build_variant(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "burn_in_tags_gen", - }) - mock_task_list_1 = build_mock_task_list(5) - mock_task_list_1.append(build_mock_task("burn_in_tags_gen", "burn_in_tags_gen_id_1")) - mock_task_list_2 = build_mock_task_list(5) - mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) - mock_evg_api = build_mock_evg_api([ - MockVariantData("1", "variant1-non-burn-in", mock_task_list_1), - MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_2) - ]) - - under_test.activate_task(expansions, mock_evg_api) - - mock_evg_api.configure_task.assert_called_once_with("burn_in_tests_id_2", activated=True) diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py new file mode 100644 index 00000000000..ee77ced2579 --- /dev/null +++ b/buildscripts/tests/test_evergreen_burn_in_tests.py @@ -0,0 +1,408 @@ +"""Unit tests for buildscripts/burn_in_tests.py.""" + +from __future__ import absolute_import + +import json +import os +import sys +import unittest +from datetime import datetime, timedelta +from math import ceil + +import requests +from mock import patch, MagicMock +from shrub.v2 import BuildVariant, ShrubProject +from evergreen.api import EvergreenApi + +import buildscripts.evergreen_burn_in_tests as under_test +from buildscripts.ciconfig.evergreen import parse_evergreen_file +import buildscripts.resmokelib.parser as _parser +import buildscripts.resmokelib.config as _config +import buildscripts.util.teststats as teststats_utils +_parser.set_run_options() + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access + +NS = "buildscripts.evergreen_burn_in_tests" + + +def ns(relative_name): # pylint: disable=invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + + +def mock_a_file(filename): + change = MagicMock(a_path=filename) + return change + + +def mock_git_diff(change_list): + diff = MagicMock() + diff.iter_change_type.return_value = change_list + return diff + + +def mock_changed_git_files(add_files): + repo = MagicMock() + repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) + repo.working_dir = "." + return repo + + +def get_evergreen_config(config_file_path): + evergreen_home = os.path.expanduser(os.path.join("~", "evergreen")) + if os.path.exists(evergreen_home): + return parse_evergreen_file(config_file_path, evergreen_home) + return parse_evergreen_file(config_file_path) + + +class TestAcceptance(unittest.TestCase): + def tearDown(self): + _parser.set_run_options() + + @patch(ns("write_file")) + def test_no_tests_run_if_none_changed(self, write_json_mock): + """ + Given a git repository with no changes, + When burn_in_tests is run, + Then no tests are discovered to run. + """ + variant = "build_variant" + repos = [mock_changed_git_files([])] + repeat_config = under_test.RepeatConfig() + gen_config = under_test.GenerateConfig( + variant, + "project", + ) # yapf: disable + mock_evg_conf = MagicMock() + mock_evg_conf.get_task_names_by_tag.return_value = set() + mock_evg_api = MagicMock() + + under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, + mock_evg_conf, repos, "testfile.json", "install-dir/bin") + + write_json_mock.assert_called_once() + written_config = json.loads(write_json_mock.call_args[0][1]) + display_task = written_config["buildvariants"][0]["display_tasks"][0] + self.assertEqual(1, len(display_task["execution_tasks"])) + self.assertEqual(under_test.BURN_IN_TESTS_GEN_TASK, display_task["execution_tasks"][0]) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("write_file")) + def test_tests_generated_if_a_file_changed(self, write_json_mock): + """ + Given a git repository with changes, + When burn_in_tests is run, + Then tests are discovered to run. + """ + # Note: this test is using actual tests and suites. So changes to those suites could + # introduce failures and require this test to be updated. + # You can see the test file it is using below. This test is used in the 'auth' and + # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. + variant = "enterprise-rhel-80-64-bit-inmem" + repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] + repeat_config = under_test.RepeatConfig() + gen_config = under_test.GenerateConfig( + variant, + "project", + ) # yapf: disable + mock_evg_conf = get_evergreen_config("etc/evergreen.yml") + mock_evg_api = MagicMock() + + under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, + mock_evg_conf, repos, "testfile.json", 'install-dir/bin') + + write_json_mock.assert_called_once() + written_config = json.loads(write_json_mock.call_args[0][1]) + n_tasks = len(written_config["tasks"]) + # Ensure we are generating at least one task for the test. + self.assertGreaterEqual(n_tasks, 1) + + written_build_variant = written_config["buildvariants"][0] + self.assertEqual(variant, written_build_variant["name"]) + self.assertEqual(n_tasks, len(written_build_variant["tasks"])) + + display_task = written_build_variant["display_tasks"][0] + # The display task should contain all the generated tasks as well as 1 extra task for + # the burn_in_test_gen task. + self.assertEqual(n_tasks + 1, len(display_task["execution_tasks"])) + + +class TestGenerateConfig(unittest.TestCase): + def test_run_build_variant_with_no_run_build_variant(self): + gen_config = under_test.GenerateConfig("build_variant", "project") + + self.assertEqual(gen_config.build_variant, gen_config.run_build_variant) + + def test_run_build_variant_with_run_build_variant(self): + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + + self.assertNotEqual(gen_config.build_variant, gen_config.run_build_variant) + self.assertEqual(gen_config.run_build_variant, "run_build_variant") + + def test_validate_non_existing_build_variant(self): + evg_conf_mock = MagicMock() + evg_conf_mock.get_variant.return_value = None + + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + + with self.assertRaises(ValueError): + gen_config.validate(evg_conf_mock) + + def test_validate_existing_build_variant(self): + evg_conf_mock = MagicMock() + + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + gen_config.validate(evg_conf_mock) + + def test_validate_non_existing_run_build_variant(self): + evg_conf_mock = MagicMock() + + gen_config = under_test.GenerateConfig("build_variant", "project") + gen_config.validate(evg_conf_mock) + + +class TestParseAvgTestRuntime(unittest.TestCase): + def test__parse_avg_test_runtime(self): + task_avg_test_runtime_stats = [ + teststats_utils.TestRuntime(test_name="dir/test1.js", runtime=30.2), + teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1) + ] + result = under_test._parse_avg_test_runtime("dir/test2.js", task_avg_test_runtime_stats) + self.assertEqual(result, 455.1) + + +class TestCalculateTimeout(unittest.TestCase): + def test__calculate_timeout(self): + avg_test_runtime = 455.1 + expected_result = ceil(avg_test_runtime * under_test.AVG_TEST_TIME_MULTIPLIER) + self.assertEqual(expected_result, under_test._calculate_timeout(avg_test_runtime)) + + def test__calculate_timeout_avg_is_less_than_min(self): + avg_test_runtime = 10 + self.assertEqual(under_test.MIN_AVG_TEST_TIME_SEC, + under_test._calculate_timeout(avg_test_runtime)) + + +class TestCalculateExecTimeout(unittest.TestCase): + def test__calculate_exec_timeout(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + avg_test_runtime = 455.1 + + exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) + + self.assertEqual(1771, exec_timeout) + + def test_average_timeout_greater_than_execution_time(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600, repeat_tests_min=2) + avg_test_runtime = 750 + + exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) + + # The timeout needs to be greater than the number of the test * the minimum number of runs. + minimum_expected_timeout = avg_test_runtime * repeat_config.repeat_tests_min + + self.assertGreater(exec_timeout, minimum_expected_timeout) + + +class TestGenerateTimeouts(unittest.TestCase): + def test__generate_timeouts(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)] + test_name = "dir/test2.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertEqual(timeout_info.exec_timeout, 1771) + self.assertEqual(timeout_info.timeout, 1366) + + def test__generate_timeouts_no_results(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [] + test_name = "dir/new_test.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertIsNone(timeout_info.cmd) + + def test__generate_timeouts_avg_runtime_is_zero(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [ + teststats_utils.TestRuntime(test_name="dir/test_with_zero_runtime.js", runtime=0) + ] + test_name = "dir/test_with_zero_runtime.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertIsNone(timeout_info.cmd) + + +class TestGetTaskRuntimeHistory(unittest.TestCase): + def test_get_task_runtime_history(self): + mock_evg_api = MagicMock() + mock_evg_api.test_stats_by_project.return_value = [ + MagicMock( + test_file="dir/test2.js", + task_name="task1", + variant="variant1", + distro="distro1", + date=datetime.utcnow().date(), + num_pass=1, + num_fail=0, + avg_duration_pass=10.1, + ) + ] + analysis_duration = under_test.AVG_TEST_RUNTIME_ANALYSIS_DAYS + end_date = datetime.utcnow().replace(microsecond=0) + start_date = end_date - timedelta(days=analysis_duration) + mock_gen_config = MagicMock(project="project1", build_variant="variant1") + + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api, + history_end_date=end_date) + result = executor.get_task_runtime_history("task1") + + self.assertEqual(result, [("dir/test2.js", 10.1)]) + mock_evg_api.test_stats_by_project.assert_called_with( + "project1", after_date=start_date, before_date=end_date, group_by="test", + group_num_days=14, tasks=["task1"], variants=["variant1"]) + + def test_get_task_runtime_history_evg_degraded_mode_error(self): + mock_response = MagicMock(status_code=requests.codes.SERVICE_UNAVAILABLE) + mock_evg_api = MagicMock() + mock_evg_api.test_stats_by_project.side_effect = requests.HTTPError(response=mock_response) + mock_gen_config = MagicMock(project="project1", build_variant="variant1") + + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api) + result = executor.get_task_runtime_history("task1") + + self.assertEqual(result, []) + + +TESTS_BY_TASK = { + "task1": { + "resmoke_args": "--suites=suite1", + "tests": ["jstests/test1.js", "jstests/test2.js"]}, + "task2": { + "resmoke_args": "--suites=suite1", + "tests": ["jstests/test1.js", "jstests/test3.js"]}, + "task3": { + "resmoke_args": "--suites=suite3", + "tests": ["jstests/test4.js", "jstests/test5.js"]}, + "task4": { + "resmoke_args": "--suites=suite4", "tests": []}, +} # yapf: disable + + +def create_tests_by_task_mock(n_tasks, n_tests): + return { + f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ + f"jstests/tests_{j}" for j in range(n_tests) + ], require_multiversion_setup=False, distro=f"distro_{i}", build_variant="variant", + suite=f"suite_{i}") + for i in range(n_tasks) + } + + +class TestCreateGenerateTasksConfig(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_no_tasks_given(self): + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant") + repeat_config = MagicMock() + mock_evg_api = MagicMock() + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant({}, build_variant) + + evg_config_dict = build_variant.as_dict() + self.assertEqual(0, len(evg_config_dict["tasks"])) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_one_task_one_test(self): + n_tasks = 1 + n_tests = 1 + resmoke_options = "options for resmoke" + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant", distro=None) + repeat_config = MagicMock() + repeat_config.generate_resmoke_options.return_value = resmoke_options + mock_evg_api = MagicMock() + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant(tests_by_task, build_variant) + + shrub_config = ShrubProject.empty().add_build_variant(build_variant) + evg_config_dict = shrub_config.as_dict() + tasks = evg_config_dict["tasks"] + self.assertEqual(n_tasks * n_tests, len(tasks)) + cmd = tasks[0]["commands"] + self.assertIn(resmoke_options, cmd[2]["vars"]["resmoke_args"]) + self.assertEqual("suite_0", cmd[2]["vars"]["suite"]) + self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"]) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_n_task_m_test(self): + n_tasks = 3 + n_tests = 5 + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant", distro=None) + repeat_config = MagicMock() + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + mock_evg_api = MagicMock() + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant(tests_by_task, build_variant) + + evg_config_dict = build_variant.as_dict() + self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"])) + + +class TestCreateGenerateTasksFile(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("sys.exit")) + @patch(ns("validate_task_generation_limit")) + def test_cap_on_task_generate(self, validate_mock, exit_mock): + gen_config = MagicMock(require_multiversion_setup=False) + repeat_config = MagicMock() + tests_by_task = MagicMock() + mock_evg_api = MagicMock() + + validate_mock.return_value = False + + exit_mock.side_effect = ValueError("exiting") + with self.assertRaises(ValueError): + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api, + "gen_file.json") + executor.execute(tests_by_task) + + exit_mock.assert_called_once() + + +class TestFindChangedTests(unittest.TestCase): + def test_manual_tests_should_be_specifiable_via_env_vars(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_repo = MagicMock() + mock_env = { + "BURN_IN_TESTS": "jstests/auth/auth1.js,jstests/core/where1.js", + } + change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) + + test_set = change_detector.find_changed_tests([mock_repo]) + + self.assertEqual(2, len(test_set)) + self.assertIn("jstests/auth/auth1.js", test_set) + self.assertIn("jstests/core/where1.js", test_set) + + def test_empty_env_should_not_add_extra_tests(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_repo = MagicMock() + mock_env = {} + change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) + + test_set = change_detector.find_changed_tests([mock_repo]) + + self.assertEqual(set(), test_set) diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py new file mode 100644 index 00000000000..7ece404ab36 --- /dev/null +++ b/buildscripts/tests/test_evergreen_gen_build_variant.py @@ -0,0 +1,382 @@ +"""Unit tests for the generate_resmoke_suite script.""" +import unittest + +from mock import MagicMock + +from buildscripts import evergreen_gen_build_variant as under_test +from buildscripts.ciconfig.evergreen import Variant, Task + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access +# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter + + +def build_mock_build_variant(expansions=None, task_list=None): + task_spec_list = [{"name": task.name} for task in task_list] if task_list else [] + config = { + "tasks": task_spec_list, + } + if expansions: + config["expansions"] = expansions + + if task_list is None: + task_list = [] + task_map = {task.name: task for task in task_list} + + return Variant(config, task_map, {}) + + +def build_mock_task(name, run_vars=None, depends_on=None): + config = { + "name": + name, "commands": [ + {"func": "do setup"}, + { + "func": "generate resmoke tasks", + "vars": run_vars if run_vars else {}, + }, + ] + } + + if depends_on is not None: + config["depends_on"] = depends_on + return Task(config) + + +def build_mock_project_config(variant=None, task_defs=None): + mock_project = MagicMock() + if variant: + mock_project.get_variant.return_value = variant + + if task_defs: + mock_project.get_task.side_effect = task_defs + + return mock_project + + +def build_mock_expansions(): + mock_expansions = MagicMock() + mock_expansions.config_location.return_value = "/path/to/config" + mock_expansions.get_max_sub_suites.return_value = 998 + mock_expansions.task_name = "generating_task" + return mock_expansions + + +def build_mock_evg_api(build_task_list): + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list + return mock_evg_api + + +def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None): + if build_expansions is None: + build_expansions = {} + if task_def_list is None: + task_def_list = [] + if build_task_list is None: + build_task_list = [] + + mock_build_variant = build_mock_build_variant(build_expansions, task_def_list) + mock_project = build_mock_project_config(mock_build_variant, task_def_list) + mock_evg_expansions = build_mock_expansions() + mock_evg_api = build_mock_evg_api(build_task_list) + + return under_test.GenerateBuildVariantOrchestrator( + gen_task_validation=MagicMock(), + gen_task_options=MagicMock(), + evg_project_config=mock_project, + evg_expansions=mock_evg_expansions, + evg_api=mock_evg_api, + ) + + +class TestEvgExpansions(unittest.TestCase): + def test_get_max_sub_suites_should_use_patch_value_in_patches(self): + evg_expansions = under_test.EvgExpansions( + is_patch=True, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), evg_expansions.max_sub_suites) + + def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self): + evg_expansions = under_test.EvgExpansions( + is_patch=False, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), + evg_expansions.mainline_max_sub_suites) + + def test_get_max_sub_suites_should_use_mainline_value_if_patch_status_unknown(self): + evg_expansions = under_test.EvgExpansions( + is_patch=None, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), + evg_expansions.mainline_max_sub_suites) + + +class TestTranslateRunVar(unittest.TestCase): + def test_normal_value_should_be_returned(self): + run_var = "some value" + mock_build_variant = build_mock_build_variant() + self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant(self): + run_var = "${my_expansion}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_none(self): + run_var = "${my_expansion}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_default(self): + run_var = "${my_expansion|default}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant_even_with_default(self): + run_var = "${my_expansion|default}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + +class TestTaskDefToSplitParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("my_task", split_param.suite_name) + self.assertEqual("my_task", split_param.filename) + + def test_params_should_allow_suite_to_be_overridden(self): + run_vars = { + "resmoke_args": "run tests", + "suite": "the suite", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("the suite", split_param.suite_name) + self.assertEqual("the suite", split_param.filename) + + +class TestDetermineTaskDependencies(unittest.TestCase): + def test_running_task_should_not_be_included_in_depends_on(self): + run_vars = { + "resmoke_args": "run tests", + } + depends_on = [ + {"name": "compile"}, + {"name": "build_variant_gen"}, + ] + mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def) + + self.assertIn("compile", dependencies) + self.assertNotIn("generating_task", dependencies) + + +class TestTaskDefToGenParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertFalse(gen_params.require_multiversion_setup) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertIsNone(gen_params.large_distro_name) + self.assertFalse(gen_params.use_large_distro) + + def test_params_should_be_overwritable(self): + run_vars = { + "resmoke_args": "run tests", + "use_large_distro": "true", + } + mock_task_def = build_mock_task("my_task", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertFalse(gen_params.require_multiversion_setup) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertEqual("my large distro", gen_params.large_distro_name) + self.assertTrue(gen_params.use_large_distro) + + +class TestTaskDefToFuzzerParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "num_files": "5", + "num_tasks": "3", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(5, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("jstestfuzz", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertIsNone(fuzzer_params.large_distro_name) + self.assertFalse(fuzzer_params.use_large_distro) + + def test_num_tasks_respects_max_sub_suites(self): + run_vars = { + "num_files": "5", + "num_tasks": "3", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + mock_orchestrator.evg_expansions.get_max_sub_suites.return_value = 1 + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual(5, fuzzer_params.num_files) + self.assertEqual(1, fuzzer_params.num_tasks) + + def test_params_should_be_overwritable(self): + run_vars = { + "num_files": "${file_count|8}", + "num_tasks": "3", + "use_large_distro": "true", + "npm_command": "aggfuzzer", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(8, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("aggfuzzer", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertEqual("my large distro", fuzzer_params.large_distro_name) + self.assertTrue(fuzzer_params.use_large_distro) + + +class TestGenerateBuildVariant(unittest.TestCase): + def test_a_whole_build_variant(self): + gen_run_vars = { + "resmoke_args": "run tests", + } + mv_gen_run_vars = { + "resmoke_args": "run tests", + "suite": "some suite", + } + fuzz_run_vars = { + "num_files": "5", + "num_tasks": "3", + "is_jstestfuzz": "true", + } + mv_fuzz_run_vars = { + "num_files": "5", + "num_tasks": "3", + "is_jstestfuzz": "true", + "suite": "aggfuzzer", + } + mock_task_defs = [ + build_mock_task("my_gen_task", gen_run_vars), + build_mock_task("my_fuzzer_task", fuzz_run_vars), + build_mock_task("my_mv_fuzzer_task", mv_fuzz_run_vars), + build_mock_task("my_mv_gen_task", mv_gen_run_vars), + ] + mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs) + builder = MagicMock() + + builder = mock_orchestrator.generate_build_variant(builder, "build variant") + + self.assertEqual(builder.generate_suite.call_count, 2) + self.assertEqual(builder.generate_fuzzer.call_count, 2) + + +class TestAdjustTaskPriority(unittest.TestCase): + def test_task_is_updates(self): + starting_priority = 42 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority + 1) + + def test_task_should_only_reach_99(self): + starting_priority = 99 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority) + + +class TestAdjustGenTasksPriority(unittest.TestCase): + def test_gen_tasks_in_task_list_are_adjusted(self): + gen_tasks = {"task_3", "task_8", "task_13"} + n_build_tasks = 25 + mock_task_list = [ + MagicMock(build_variant='dummy_variant', display_name=f"task_{i}", priority=0) + for i in range(n_build_tasks) + ] + mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list) + + n_tasks_adjusted = mock_orchestrator.adjust_gen_tasks_priority(gen_tasks) + + self.assertEqual(len(gen_tasks), n_tasks_adjusted) diff --git a/buildscripts/tests/test_evergreen_task_timeout.py b/buildscripts/tests/test_evergreen_task_timeout.py index 294c50d0a17..b3e4e201d96 100644 --- a/buildscripts/tests/test_evergreen_task_timeout.py +++ b/buildscripts/tests/test_evergreen_task_timeout.py @@ -110,7 +110,7 @@ class TestTimeoutOverrides(unittest.TestCase): class TestDetermineExecTimeout(unittest.TestCase): def _validate_exec_timeout(self, idle_timeout, exec_timeout, historic_timeout, evg_alias, - build_variant, display_name, timeout_override, expected_timeout): + build_variant, timeout_override, expected_timeout): task_name = "task_name" variant = build_variant overrides = {} @@ -121,9 +121,8 @@ class TestDetermineExecTimeout(unittest.TestCase): orchestrator = under_test.TaskTimeoutOrchestrator( timeout_service=MagicMock(spec_set=TimeoutService), - timeout_overrides=mock_timeout_overrides, evg_project_config=MagicMock( - spec_set=EvergreenProjectConfig, - get_variant=MagicMock(return_value=MagicMock(display_name=display_name)))) + timeout_overrides=mock_timeout_overrides, + evg_project_config=MagicMock(spec_set=EvergreenProjectConfig)) actual_timeout = orchestrator.determine_exec_timeout( task_name, variant, idle_timeout, exec_timeout, evg_alias, historic_timeout) @@ -133,83 +132,78 @@ class TestDetermineExecTimeout(unittest.TestCase): def test_timeout_used_if_specified(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=42), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, - expected_timeout=timedelta(seconds=42)) + timeout_override=None, expected_timeout=timedelta(seconds=42)) def test_default_is_returned_with_no_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, - evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, + evg_alias=None, build_variant="variant", timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_is_returned_with_timeout_at_zero(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, + timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_required_returned_on_required_variants(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - display_name="! required", timeout_override=None, + timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) def test_override_on_required_should_use_override(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - display_name="! required", timeout_override=3 * 60, + timeout_override=3 * 60, expected_timeout=timedelta(minutes=3 * 60)) def test_task_specific_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=60, - expected_timeout=timedelta(minutes=60)) + timeout_override=60, expected_timeout=timedelta(minutes=60)) def test_commit_queue_items_use_commit_queue_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=under_test.COMMIT_QUEUE_ALIAS, - build_variant="variant", display_name="not required", - timeout_override=None, + build_variant="variant", timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_use_idle_timeout_if_greater_than_exec_timeout(self): self._validate_exec_timeout( idle_timeout=timedelta(hours=2), exec_timeout=timedelta(minutes=10), - historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, expected_timeout=timedelta(hours=2)) + historic_timeout=None, evg_alias=None, build_variant="variant", timeout_override=None, + expected_timeout=timedelta(hours=2)) def test_historic_timeout_should_be_used_if_given(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", display_name="not required", - timeout_override=None, expected_timeout=timedelta(minutes=15)) + build_variant="variant", timeout_override=None, + expected_timeout=timedelta(minutes=15)) def test_commit_queue_should_override_historic_timeouts(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), - evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", - display_name="not required", timeout_override=None, + evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_override_should_override_historic_timeouts(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", display_name="not required", - timeout_override=33, expected_timeout=timedelta(minutes=33)) + build_variant="variant", timeout_override=33, + expected_timeout=timedelta(minutes=33)) def test_historic_timeout_should_not_be_overridden_by_required_bv(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant-required", display_name="! required", - timeout_override=None, expected_timeout=timedelta(minutes=15)) + build_variant="variant-required", timeout_override=None, + expected_timeout=timedelta(minutes=15)) def test_historic_timeout_should_not_be_increase_required_bv_timeout(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT + timedelta(minutes=30), - evg_alias=None, build_variant="variant-required", display_name="! required", - timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) + evg_alias=None, build_variant="variant-required", timeout_override=None, + expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) class TestDetermineIdleTimeout(unittest.TestCase): diff --git a/buildscripts/tests/test_jepsen_report.py b/buildscripts/tests/test_jepsen_report.py deleted file mode 100644 index cdfb095ba5a..00000000000 --- a/buildscripts/tests/test_jepsen_report.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Tests for jepsen report generator.""" -import unittest -import textwrap -import random -import os -from unittest.mock import patch, mock_open, MagicMock -from click.testing import CliRunner - -from buildscripts.jepsen_report import parse, ParserOutput, main - -_CORPUS = textwrap.dedent("""\ - "indeterminate: Command failed with error 251 (NoSuchTransaction): 'Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1' on server n9:27017. The full response is {\"writeConcernError\": {\"code\": 6, \"codeName\": \"HostUnreachable\", \"errmsg\": \"operation was interrupted\", \"errInfo\": {\"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"clientSupplied\"}}}, \"topologyVersion\": {\"processId\": {\"$oid\": \"625f0b0ef9d6a12d9b562ff9\"}, \"counter\": 21}, \"ok\": 0.0, \"errmsg\": \"Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1\", \"code\": 251, \"codeName\": \"NoSuchTransaction\", \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 16}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"operationTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 5}}, \"recoveryToken\": {\"recoveryShardId\": \"rs_shard1\"}}", - :index 1141}})}, - :workload {:valid? true}, - :valid? true} - - -Everything looks good! ヽ(‘ー`)ノ - - - -# Successful tests - -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T163539.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164724.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165317.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165910.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T170503.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171055.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171648.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172239.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172832.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T173634.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174227.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174820.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T175413.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180006.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180558.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181148.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181946.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T182539.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183720.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184313.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184906.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T185459.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190052.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190645.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191239.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191833.000Z - -# Crashed tests - -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition - -28 successes -0 unknown -2 crashed -0 failures -""").splitlines() - - -class TestParser(unittest.TestCase): - """TestParser.""" - - @classmethod - def _corpus_generator(cls): - n_pass = random.randint(1, 30) - n_fail = random.randint(1, 30) - n_unknown = random.randint(1, 30) - n_crash = random.randint(1, 30) - - corpus = textwrap.dedent("""\ - :index 2893}})}, - :workload {:valid? true}, - :valid? true} -Everything looks good! ヽ(‘ー`)ノ -""") - successful_tests = [] - if n_pass > 0: - corpus += "\n# Successful tests\n\n" - for i in range(0, n_pass): - corpus += f"test {i}\n" - successful_tests.append(f"test {i}") - - failed_tests = [] - if n_fail > 0: - corpus += "\n# Failed tests\n\n" - for i in range(0, n_fail): - corpus += f"test {i}\n" - failed_tests.append(f"test {i}") - - indeterminate_tests = [] - if n_unknown > 0: - corpus += "\n# Indeterminate tests\n\n" - for i in range(0, n_unknown): - corpus += f"test {i}\n" - indeterminate_tests.append(f"test {i}") - - crashed_tests = [] - if n_crash > 0: - corpus += "\n# Crashed tests\n\n" - for i in range(0, n_crash): - corpus += f"test {i}\n" - crashed_tests.append(f"test {i}") - # note leading newline for this block is required to match the actual - # logs - corpus += textwrap.dedent(f""" -{n_pass} successes -{n_unknown} unknown -{n_crash} crashed -{n_fail} failures -""") - - return { - 'expected': - ParserOutput({ - 'success': successful_tests, - 'unknown': indeterminate_tests, - 'crashed': crashed_tests, - 'failed': failed_tests, - }), 'corpus': - corpus - } - - def test_parser(self): - """Test with embedded corpus.""" - out = parse(_CORPUS) - self.assertEqual(len(out['success']), 28) - self.assertEqual(len(out['unknown']), 0) - self.assertEqual(len(out['crashed']), 2) - self.assertEqual(len(out['failed']), 0) - - def test_parser2(self): - """Test with jepsen.log file.""" - with open(os.path.join(os.path.dirname(__file__), - "test_jepsen_report_corpus.log.txt")) as fh: - corpus = fh.read().splitlines() - out = parse(corpus) - self.assertEqual(len(out['success']), 29) - self.assertEqual(len(out['unknown']), 0) - self.assertEqual(len(out['crashed']), 1) - self.assertEqual(len(out['failed']), 0) - - def test_generated_corpus(self): - """Generate 100 corpuses and test them.""" - for _ in range(0, 100): - self._test_generated_corpus() - - def _test_generated_corpus(self): - gen = self._corpus_generator() - corpus = gen['corpus'].splitlines() - out = parse(corpus) - self.assertDictEqual(out, gen['expected']) - - @patch('buildscripts.jepsen_report._try_find_log_file') - @patch('buildscripts.jepsen_report._get_log_lines') - @patch('buildscripts.jepsen_report._put_report') - def test_main(self, mock_put_report, mock_get_log_lines, mock_try_find_log_file): - """Test main function.""" - gen = self._corpus_generator() - corpus = gen['corpus'].splitlines() - mock_get_log_lines.return_value = corpus - - def _try_find_log_file(_store, _test): - if _try_find_log_file.counter == 0: - _try_find_log_file.counter += 1 - with open( - os.path.join( - os.path.dirname(__file__), "test_jepsen_report_corpus.log.txt")) as fh: - return fh.read() - return "" - - _try_find_log_file.counter = 0 - mock_try_find_log_file.side_effect = _try_find_log_file - - runner = CliRunner() - result = runner.invoke(main, - ["--start_time=0", "--end_time=10", "--elapsed=10", "test.log"]) - num_tests = len(gen['expected']['success']) + len(gen['expected']['unknown']) + len( - gen['expected']['crashed']) + len(gen['expected']['failed']) - num_fails = num_tests - len(gen['expected']['success']) - - callee_dict = mock_put_report.call_args[0][0] - self.assertEqual(callee_dict['failures'], num_fails) - self.assertEqual(len(callee_dict['results']), num_tests) - mock_get_log_lines.assert_called_once_with('test.log') - if gen['expected']['crashed']: - self.assertEqual(result.exit_code, 2) - elif gen['expected']['unknown'] or gen['expected']['failure']: - self.assertEqual(result.exit_code, 1) - else: - self.assertEqual(result.exit_code, 0) diff --git a/buildscripts/tests/test_jepsen_report_corpus.log.txt b/buildscripts/tests/test_jepsen_report_corpus.log.txt deleted file mode 100644 index 89d4e91892b..00000000000 --- a/buildscripts/tests/test_jepsen_report_corpus.log.txt +++ /dev/null @@ -1,67 +0,0 @@ -INFO [2022-04-20 02:46:34,505] jepsen test runner - jepsen.core {:perf - {:latency-graph {:valid? true}, - :rate-graph {:valid? true}, - :valid? true}, - :clock {:valid? true}, - :stats - {:valid? true, - :count 5768, - :ok-count 4930, - :fail-count 0, - :info-count 838, - :by-f - {:txn - {:valid? true, - :count 5768, - :ok-count 4930, - :fail-count 0, - :info-count 838}}}, - :exceptions {:valid? true}, - :workload {:valid? true}, - :valid? true} - - -Everything looks good! ヽ(‘ー`)ノ - - - -# Successful tests - -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235350.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235940.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T000533.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001724.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002316.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002909.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T003502.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004057.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004651.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005244.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005837.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T010429.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011021.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011617.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012210.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012802.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T013356.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014154.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014746.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015338.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015931.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T020525.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021116.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021708.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022301.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022854.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T023446.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T024039.000Z - -# Crashed tests - -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition - -29 successes -0 unknown -1 crashed -0 failures diff --git a/buildscripts/tests/test_mongosymb.py b/buildscripts/tests/test_mongosymb.py deleted file mode 100644 index 188ea385c5f..00000000000 --- a/buildscripts/tests/test_mongosymb.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Unit tests for buildscripts/mongosymb.py.""" -# pylint: disable=missing-docstring -import unittest - -from buildscripts import mongosymb as under_test - - -class TestGetVersion(unittest.TestCase): - def test_get_version_with_patch(self): - trace_doc = { - "processInfo": { - "mongodbVersion": "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169" - } - } - version = under_test.get_version(trace_doc) - self.assertEqual(version, "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169") - - def test_get_version_without_patch(self): - trace_doc = {"processInfo": {"mongodbVersion": "6.1.0-alpha-504-g0c8a142"}} - version = under_test.get_version(trace_doc) - self.assertEqual(version, "6.1.0-alpha-504-g0c8a142") - - def test_get_version_no_mongodb_version(self): - trace_doc = {"processInfo": {}} - version = under_test.get_version(trace_doc) - self.assertEqual(version, None) - - def test_get_version_no_process_info(self): - trace_doc = {} - version = under_test.get_version(trace_doc) - self.assertEqual(version, None) - - -class TestHasHighNotFoundPathsRatio(unittest.TestCase): - def test_not_found_paths_ratio_is_more_than_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": "some/path"}, - {"path": "some/path"}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, False) - - def test_not_found_paths_ratio_is_equal_to_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": "some/path"}, - {"path": None}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, True) - - def test_not_found_paths_ratio_is_less_than_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": None}, - {"path": None}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, True) - - def test_no_frames(self): - frames = [] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, False) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 85725e60858..70689345b69 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -24,7 +24,7 @@ from buildscripts.task_generation.task_types.gentask_options import GenTaskOptio from buildscripts.tests.test_burn_in_tests import get_evergreen_config, mock_changed_git_files from buildscripts import selected_tests as under_test -# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter,too-many-locals +# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter NS = "buildscripts.selected_tests" @@ -101,9 +101,7 @@ class TestAcceptance(unittest.TestCase): self.assertEqual(generated_config.file_list[0].file_name, "selected_tests_config.json") @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_when_test_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] + def test_when_test_mappings_are_found_for_changed_files(self): mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( @@ -135,7 +133,7 @@ class TestAcceptance(unittest.TestCase): # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file - self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate) + self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list @@ -147,13 +145,11 @@ class TestAcceptance(unittest.TestCase): # jstests/auth/auth1.js belongs to two suites, auth and auth_audit, rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_when_task_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] + def test_when_task_mappings_are_found_for_changed_files(self): mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( @@ -168,7 +164,7 @@ class TestAcceptance(unittest.TestCase): mock_task_mapping = TaskMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, - tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-8", flip_count=5)]) + tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse( task_mappings=[mock_task_mapping]) @@ -194,7 +190,7 @@ class TestAcceptance(unittest.TestCase): build_variants_with_generated_tasks = generated_evg_config["buildvariants"] rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5) @@ -404,6 +400,13 @@ class TestGetTaskConfigsForTaskMappings(unittest.TestCase): class TestRemoveRepoPathPrefix(unittest.TestCase): - def test_path_handling(self): - filepath = under_test._remove_repo_path_prefix("sample_directory/src/file1.cpp") - self.assertEqual(filepath, "sample_directory/src/file1.cpp") + def test_file_is_in_enterprise_modules(self): + filepath = under_test._remove_repo_path_prefix( + "src/mongo/db/modules/enterprise/src/file1.cpp") + + self.assertEqual(filepath, "src/file1.cpp") + + def test_file_is_not_in_enterprise_modules(self): + filepath = under_test._remove_repo_path_prefix("other_directory/src/file1.cpp") + + self.assertEqual(filepath, "other_directory/src/file1.cpp") diff --git a/buildscripts/tests/test_validate_commit_message.py b/buildscripts/tests/test_validate_commit_message.py index 1bf0039188d..db341a5add1 100644 --- a/buildscripts/tests/test_validate_commit_message.py +++ b/buildscripts/tests/test_validate_commit_message.py @@ -1,31 +1,130 @@ """Unit tests for the evergreen_task_timeout script.""" +import itertools import unittest +from typing import List, Optional +from unittest.mock import MagicMock -from buildscripts.validate_commit_message import main, STATUS_OK, STATUS_ERROR +import buildscripts.validate_commit_message as under_test +from buildscripts.client.jiraclient import JiraClient, SecurityLevel +from evergreen import EvergreenApi # pylint: disable=missing-docstring,no-self-use +INVALID_MESSAGES = [ + "", # You must provide a message + "RevertEVG-1", # revert and ticket must be formatted + "revert EVG-1", # revert must be capitalized + "This is not a valid message", # message must be valid + "Fix Lint", # Fix lint is strict in terms of caps +] + + +def create_mock_code_change(code_change_messages: List[str], branch_name: Optional[str] = None): + mock_code_change = MagicMock( + commit_messages=code_change_messages, + branch_name=branch_name if branch_name else "mongodb-mongo-master", + ) + return mock_code_change + + +def create_mock_patch(code_change_messages: List[str], branch_name: Optional[str] = None): + mock_code_change = create_mock_code_change(code_change_messages, branch_name) + mock_patch = MagicMock(module_code_changes=[mock_code_change]) + return mock_patch + + +def create_mock_evg_client(code_change_messages: List[str], + branch_name: Optional[str] = None) -> MagicMock: + mock_patch = create_mock_patch(code_change_messages, branch_name) + + mock_evg_client = MagicMock(spec_set=EvergreenApi) + mock_evg_client.patch_by_id.return_value = mock_patch + return mock_evg_client + + +def create_mock_jira_client(): + mock_jira = MagicMock(spec_set=JiraClient) + mock_jira.get_ticket_security_level.return_value = SecurityLevel.NONE + return mock_jira + + +def interleave_new_format(older): + """Create a new list containing a new and old format copy of each string.""" + newer = [ + f"Commit Queue Merge: '{old}' into 'mongodb/mongo:SERVER-45949-validate-message-format'" + for old in older + ] + return list(itertools.chain(*zip(older, newer))) + class ValidateCommitMessageTest(unittest.TestCase): - def test_valid(self): + def test_valid_commits(self): messages = [ + "Fix lint", + "EVG-1", # Test valid projects with various number lengths + "SERVER-20", + "WT-300", "SERVER-44338", - "Revert \"SERVER-60", + "Revert EVG-5", + "Revert SERVER-60", + "Revert WT-700", + "Revert 'SERVER-8000", + 'Revert "SERVER-90000', "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", + "Import tools: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", 'Revert "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4"', ] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_OK) - self.assertTrue(all(main([message]) == STATUS_OK for message in messages)) + def test_private(self): + messages = ["XYZ-1"] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - def test_invalid(self): + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_private_with_public(self): messages = [ - "SERVER-", # missing number - "Revert SERVER-60", # missing quote before SERVER - "", # empty value - "nonsense", # nonsense values + "Fix lint", + "EVG-1", # Test valid projects with various number lengths + "SERVER-20", + "XYZ-1", ] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_internal_ticket_to_public_repo_should_fail(self): + message = "SERVER-20" + mock_evg_api = create_mock_evg_client(interleave_new_format([message])) + mock_jira = create_mock_jira_client() + mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_internal_ticket_to_private_repo_should_succeed(self): + message = "SERVER-20" + mock_evg_api = create_mock_evg_client(interleave_new_format([message]), "private-repo") + mock_jira = create_mock_jira_client() + mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - self.assertTrue(all(main([message]) == STATUS_ERROR for message in messages)) + is_valid = orchestrator.validate_commit_messages("version_id") - def test_message_is_empty_list(self): - self.assertEqual(main([]), STATUS_ERROR) + self.assertEqual(is_valid, under_test.STATUS_OK) diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py index 23e9c1f331f..bb0dd8a0c3e 100644 --- a/buildscripts/tests/timeouts/test_timeout_service.py +++ b/buildscripts/tests/timeouts/test_timeout_service.py @@ -1,42 +1,41 @@ """Unit tests for timeout_service.py.""" import random import unittest -from unittest.mock import MagicMock, patch +from datetime import datetime, timedelta +from unittest.mock import MagicMock from requests.exceptions import HTTPError +from evergreen import EvergreenApi import buildscripts.timeouts.timeout_service as under_test from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.util.teststats import HistoricTaskData, HistoricTestInfo +from buildscripts.util.teststats import HistoricTaskData # pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access -NS = "buildscripts.timeouts.timeout_service" - -def ns(relative_name): # pylint: disable=invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def build_mock_service(resmoke_proxy=None): +def build_mock_service(evg_api=None, resmoke_proxy=None): + end_date = datetime.now() + start_date = end_date - timedelta(weeks=2) + timeout_settings = under_test.TimeoutSettings( + end_date=end_date, + start_date=start_date, + ) return under_test.TimeoutService( - resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService)) + evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi), + resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService), + timeout_settings=timeout_settings) def tst_stat_mock(file, duration, pass_count): - return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count, hooks=[]) - - -def tst_runtime_mock(file, duration, pass_count): - return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count) + return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) class TestGetTimeoutEstimate(unittest.TestCase): - @patch(ns("HistoricTaskData.from_s3")) - def test_no_stats_should_return_default_timeout(self, from_s3_mock: MagicMock): - timeout_service = build_mock_service() - from_s3_mock.return_value = [] + def test_no_stats_should_return_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.return_value = [] + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -49,19 +48,13 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_too_many_tests_missing_history_should_cause_a_default_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(23) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_a_test_with_missing_history_should_cause_a_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - test_names = [ts.test_name for ts in test_stats] - test_names.extend([f"test_with_no_stats_{i}.js" for i in range(7)]) - mock_resmoke_proxy.list_tests.return_value = test_names - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -74,21 +67,14 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_too_many_tests_with_zero_runtime_history_should_cause_a_default_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(23) - ] - test_stats.extend([ - HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[]) - for i in range(7) - ]) - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] + test_stats.append(tst_stat_mock("zero.js", 0.0, 1)) + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -101,73 +87,15 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_enough_history_but_some_tests_missing_history_should_cause_custom_task_and_default_test_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(25) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) - mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - test_names = [ts.test_name for ts in test_stats] - test_names.extend([f"test_with_no_stats_{i}.js" for i in range(5)]) - mock_resmoke_proxy.list_tests.return_value = test_names - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) - timeout_params = under_test.TimeoutParams( - evg_project="my project", - build_variant="bv", - task_name="my task", - suite_name="my suite", - is_asan=False, - ) - - timeout = timeout_service.get_timeout_estimate(timeout_params) - - self.assertTrue(timeout.is_specified()) - self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) - - @patch(ns("HistoricTaskData.from_s3")) - def test_enough_history_but_some_tests_with_zero_runtime_should_cause_custom_task_and_default_test_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(25) - ] - test_stats.extend([ - HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[]) - for i in range(5) - ]) - from_s3_mock.return_value = HistoricTaskData(test_stats) - mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) - timeout_params = under_test.TimeoutParams( - evg_project="my project", - build_variant="bv", - task_name="my task", - suite_name="my suite", - is_asan=False, - ) - - timeout = timeout_service.get_timeout_estimate(timeout_params) - - self.assertTrue(timeout.is_specified()) - self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) - - @patch(ns("HistoricTaskData.from_s3")) - def test_all_tests_with_runtime_history_should_use_custom_timeout(self, - from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(30) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_all_tests_with_runtime_history_should_use_custom_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + n_tests = 30 + test_runtime = 600 + test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)] + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -180,7 +108,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(1860, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) + self.assertEqual(54180, timeout.calculate_task_timeout(1)) class TestGetTaskHookOverhead(unittest.TestCase): @@ -221,10 +149,10 @@ class TestGetTaskHookOverhead(unittest.TestCase): class TestLookupHistoricStats(unittest.TestCase): - @patch(ns("HistoricTaskData.from_s3")) - def test_no_stats_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): - from_s3_mock.return_value = None - timeout_service = build_mock_service() + def test_no_stats_from_evergreen_should_return_none(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.return_value = [] + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -237,10 +165,10 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - @patch(ns("HistoricTaskData.from_s3")) - def test_errors_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): - from_s3_mock.side_effect = HTTPError("failed to connect") - timeout_service = build_mock_service() + def test_errors_from_evergreen_should_return_none(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect") + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -253,11 +181,11 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - @patch(ns("HistoricTaskData.from_s3")) - def test_stats_from_evergreen_should_return_the_stats(self, from_s3_mock: MagicMock): + def test_stats_from_evergreen_should_return_the_stats(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] - from_s3_mock.return_value = HistoricTaskData(test_stats) - timeout_service = build_mock_service() + mock_evg_api.test_stats_by_project.return_value = test_stats + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -328,34 +256,3 @@ class TestGetCleanEveryNCadence(unittest.TestCase): cadence = timeout_service._get_clean_every_n_cadence("suite", False) self.assertEqual(1, cadence) - - -class TestHaveEnoughHistoricStats(unittest.TestCase): - def test_should_return_true_when_number_of_tests_equals_zero(self): - timeout_service = build_mock_service() - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=0, num_tests_missing_data=0)) - - def test_should_return_true_when_number_of_tests_with_historic_data_more_than_threshold(self): - timeout_service = build_mock_service() - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=19)) - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=0)) - - def test_should_return_false_when_number_of_tests_with_historic_data_less_or_equal_to_threshold( - self): - timeout_service = build_mock_service() - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=20)) - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=21)) - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=100)) - - def test_exception_raised_when_number_of_tests_less_than_zero(self): - timeout_service = build_mock_service() - with self.assertRaises(ValueError): - timeout_service._have_enough_historic_stats(num_tests=-1, num_tests_missing_data=0) - with self.assertRaises(ValueError): - timeout_service._have_enough_historic_stats(num_tests=-100, num_tests_missing_data=0) diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py index 637ee59321e..7f3296ca1aa 100644 --- a/buildscripts/tests/util/test_taskname.py +++ b/buildscripts/tests/util/test_taskname.py @@ -28,7 +28,7 @@ class TestRemoveGenSuffix(unittest.TestCase): class TestDetermineTaskBaseName(unittest.TestCase): def test_task_name_with_build_variant_should_strip_bv_and_sub_task_index(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = f"auth_23_{bv}" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -36,7 +36,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_should_strip_sub_task_index(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = "auth_314" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -44,7 +44,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_or_subtask_index_should_self(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = "auth" base_task_name = under_test.determine_task_base_name(task_name, bv) diff --git a/buildscripts/tests/util/test_teststats.py b/buildscripts/tests/util/test_teststats.py index 9e72d87a737..ebba930d032 100644 --- a/buildscripts/tests/util/test_teststats.py +++ b/buildscripts/tests/util/test_teststats.py @@ -2,12 +2,8 @@ import datetime import unittest -from json import JSONDecodeError -from unittest.mock import patch from mock import Mock -from mock.mock import MagicMock -from requests import Session import buildscripts.util.teststats as under_test @@ -25,60 +21,6 @@ class NormalizeTestNameTest(unittest.TestCase): under_test.normalize_test_name("\\home\\user\\test.js")) -class TestHistoricTestInfo(unittest.TestCase): - def test_total_test_runtime_not_passing_test_no_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=0, - avg_duration=0.0, - hooks=[], - ) - - self.assertEqual(0.0, test_info.total_test_runtime()) - - def test_total_test_runtime_not_passing_test_with_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=0, - avg_duration=0.0, - hooks=[ - under_test.HistoricHookInfo( - hook_id='test:hook', - num_pass=10, - avg_duration=5.0, - ), - ], - ) - - self.assertEqual(0.0, test_info.total_test_runtime()) - - def test_total_test_runtime_passing_test_no_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=10, - avg_duration=23.0, - hooks=[], - ) - - self.assertEqual(23.0, test_info.total_test_runtime()) - - def test_total_test_runtime_passing_test_with_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=10, - avg_duration=23.0, - hooks=[ - under_test.HistoricHookInfo( - hook_id='test:hook', - num_pass=10, - avg_duration=5.0, - ), - ], - ) - - self.assertEqual(28.0, test_info.total_test_runtime()) - - class TestHistoricTaskData(unittest.TestCase): def test_no_hooks(self): evg_results = [ @@ -138,7 +80,7 @@ class TestHistoricTaskData(unittest.TestCase): @staticmethod def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0): return Mock( - test_name=test_file, + test_file=test_file, task_name="task1", variant="variant1", distro="distro1", @@ -147,53 +89,3 @@ class TestHistoricTaskData(unittest.TestCase): num_fail=0, avg_duration_pass=duration, ) - - @patch.object(Session, 'get') - def test_get_stats_from_s3_returns_data(self, mock_get): - mock_response = MagicMock() - mock_response.json.return_value = [ - { - "test_name": "jstests/noPassthroughWithMongod/geo_near_random1.js", - "num_pass": 74, - "num_fail": 0, - "avg_duration_pass": 23.16216216216216, - "max_duration_pass": 27.123, - }, - { - "test_name": "shell_advance_cluster_time:ValidateCollections", - "num_pass": 74, - "num_fail": 0, - "avg_duration_pass": 1.662162162162162, - "max_duration_pass": 100.0987, - }, - ] - mock_get.return_value = mock_response - - result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") - - self.assertEqual(result, [ - under_test.HistoricalTestInformation( - test_name="jstests/noPassthroughWithMongod/geo_near_random1.js", - num_pass=74, - num_fail=0, - avg_duration_pass=23.16216216216216, - max_duration_pass=27.123, - ), - under_test.HistoricalTestInformation( - test_name="shell_advance_cluster_time:ValidateCollections", - num_pass=74, - num_fail=0, - avg_duration_pass=1.662162162162162, - max_duration_pass=100.0987, - ), - ]) - - @patch.object(Session, 'get') - def test_get_stats_from_s3_json_decode_error(self, mock_get): - mock_response = MagicMock() - mock_response.json.side_effect = JSONDecodeError("msg", "doc", 0) - mock_get.return_value = mock_response - - result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") - - self.assertEqual(result, []) diff --git a/buildscripts/timeouts/timeout.py b/buildscripts/timeouts/timeout.py index 36f08db58df..3e3440f9c5b 100644 --- a/buildscripts/timeouts/timeout.py +++ b/buildscripts/timeouts/timeout.py @@ -9,7 +9,7 @@ from buildscripts.patch_builds.task_generation import TimeoutInfo LOGGER = structlog.getLogger(__name__) -AVG_TASK_SETUP_TIME = int(timedelta(minutes=5).total_seconds()) +AVG_TASK_SETUP_TIME = int(timedelta(minutes=2).total_seconds()) MIN_TIMEOUT_SECONDS = int(timedelta(minutes=5).total_seconds()) MAX_EXPECTED_TIMEOUT = int(timedelta(hours=48).total_seconds()) DEFAULT_SCALING_FACTOR = 3.0 @@ -45,11 +45,6 @@ class TimeoutEstimate(NamedTuple): """Create an instance with no estimation data.""" return cls(max_test_runtime=None, expected_task_runtime=None) - @classmethod - def only_task_timeout(cls, expected_task_runtime) -> "TimeoutEstimate": - """Create an instance with only task timeout estimation.""" - return cls(max_test_runtime=None, expected_task_runtime=expected_task_runtime) - def is_specified(self) -> bool: """Determine if any specific timeout value has been specified.""" return self.max_test_runtime is not None or self.expected_task_runtime is not None diff --git a/buildscripts/timeouts/timeout_service.py b/buildscripts/timeouts/timeout_service.py index b7326791873..8c0d5ad58cd 100644 --- a/buildscripts/timeouts/timeout_service.py +++ b/buildscripts/timeouts/timeout_service.py @@ -1,15 +1,16 @@ """Service for determining task timeouts.""" +from datetime import datetime from typing import Any, Dict, NamedTuple, Optional import inject import structlog from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService from buildscripts.timeouts.timeout import TimeoutEstimate -from buildscripts.util.teststats import HistoricTaskData, normalize_test_name +from buildscripts.util.teststats import HistoricTaskData +from evergreen import EvergreenApi LOGGER = structlog.get_logger(__name__) CLEAN_EVERY_N_HOOK = "CleanEveryN" -REQUIRED_STATS_THRESHOLD = 0.8 class TimeoutParams(NamedTuple): @@ -30,17 +31,29 @@ class TimeoutParams(NamedTuple): is_asan: bool +class TimeoutSettings(NamedTuple): + """Settings for determining timeouts.""" + + start_date: datetime + end_date: datetime + + class TimeoutService: """A service for determining task timeouts.""" @inject.autoparams() - def __init__(self, resmoke_proxy: ResmokeProxyService) -> None: + def __init__(self, evg_api: EvergreenApi, resmoke_proxy: ResmokeProxyService, + timeout_settings: TimeoutSettings) -> None: """ Initialize the service. + :param evg_api: Evergreen API client. :param resmoke_proxy: Proxy to query resmoke. + :param timeout_settings: Settings for how timeouts are calculated. """ + self.evg_api = evg_api self.resmoke_proxy = resmoke_proxy + self.timeout_settings = timeout_settings def get_timeout_estimate(self, timeout_params: TimeoutParams) -> TimeoutEstimate: """ @@ -51,22 +64,21 @@ class TimeoutService: """ historic_stats = self.lookup_historic_stats(timeout_params) if not historic_stats: - LOGGER.warning("Missing historic runtime information, using default timeout") return TimeoutEstimate.no_timeouts() - test_set = { - normalize_test_name(test) - for test in self.resmoke_proxy.list_tests(timeout_params.suite_name) - } + test_set = set(self.resmoke_proxy.list_tests(timeout_params.suite_name)) test_runtimes = [ stat for stat in historic_stats.get_tests_runtimes() if stat.test_name in test_set ] test_runtime_set = {test.test_name for test in test_runtimes} - num_tests_missing_historic_data = 0 for test in test_set: if test not in test_runtime_set: - LOGGER.warning("Could not find historic runtime information for test", test=test) - num_tests_missing_historic_data += 1 + # If we don't have historic runtime information for all the tests, we cannot + # reliable determine a timeout, so fallback to a default timeout. + LOGGER.warning( + "Could not find historic runtime information for test, using default timeout", + test=test) + return TimeoutEstimate.no_timeouts() total_runtime = 0.0 max_runtime = 0.0 @@ -76,31 +88,16 @@ class TimeoutService: total_runtime += runtime.runtime max_runtime = max(max_runtime, runtime.runtime) else: - LOGGER.warning("Found a test with 0 runtime", test=runtime.test_name) - num_tests_missing_historic_data += 1 - - total_num_tests = len(test_set) - if not self._have_enough_historic_stats(total_num_tests, num_tests_missing_historic_data): - LOGGER.warning( - "Not enough historic runtime information, using default timeout", - total_num_tests=total_num_tests, - num_tests_missing_historic_data=num_tests_missing_historic_data, - required_stats_threshold=REQUIRED_STATS_THRESHOLD, - ) - return TimeoutEstimate.no_timeouts() + LOGGER.warning("Found a test with 0 runtime, using default timeouts", + test=runtime.test_name) + # We found a test with a runtime of 0, which indicates that it does not have a + # proper runtime history, so fall back to a default timeout. + return TimeoutEstimate.no_timeouts() hook_overhead = self.get_task_hook_overhead( - timeout_params.suite_name, timeout_params.is_asan, total_num_tests, historic_stats) + timeout_params.suite_name, timeout_params.is_asan, len(test_set), historic_stats) total_runtime += hook_overhead - if num_tests_missing_historic_data > 0: - total_runtime += num_tests_missing_historic_data * max_runtime - LOGGER.warning( - "At least one test misses historic runtime information, using default idle timeout", - num_tests_missing_historic_data=num_tests_missing_historic_data, - ) - return TimeoutEstimate.only_task_timeout(expected_task_runtime=total_runtime) - return TimeoutEstimate(max_test_runtime=max_runtime, expected_task_runtime=total_runtime) def get_task_hook_overhead(self, suite_name: str, is_asan: bool, test_count: int, @@ -132,8 +129,7 @@ class TimeoutService: return n_expected_runs * avg_clean_every_n_runtime return 0.0 - @staticmethod - def lookup_historic_stats(timeout_params: TimeoutParams) -> Optional[HistoricTaskData]: + def lookup_historic_stats(self, timeout_params: TimeoutParams) -> Optional[HistoricTaskData]: """ Lookup historic test results stats for the given task. @@ -141,16 +137,13 @@ class TimeoutService: :return: Historic test results if they exist. """ try: - LOGGER.info( - "Getting historic runtime information", evg_project=timeout_params.evg_project, - build_variant=timeout_params.build_variant, task_name=timeout_params.task_name) - evg_stats = HistoricTaskData.from_s3( - timeout_params.evg_project, timeout_params.task_name, timeout_params.build_variant) + evg_stats = HistoricTaskData.from_evg( + self.evg_api, timeout_params.evg_project, self.timeout_settings.start_date, + self.timeout_settings.end_date, timeout_params.task_name, + timeout_params.build_variant) if not evg_stats: LOGGER.warning("No historic runtime information available") return None - LOGGER.info("Found historic runtime information", - evg_stats=evg_stats.historic_test_results) return evg_stats except Exception: # pylint: disable=broad-except # If we have any trouble getting the historic runtime information, log the issue, but @@ -159,21 +152,6 @@ class TimeoutService: exc_info=True) return None - @staticmethod - def _have_enough_historic_stats(num_tests: int, num_tests_missing_data: int) -> bool: - """ - Check whether the required number of stats threshold is met. - - :param num_tests: Number of tests to run. - :param num_tests_missing_data: Number of test that misses historic runtime data. - :return: Whether the required number of stats threshold is met. - """ - if num_tests < 0: - raise ValueError("Number of tests cannot be less than 0") - if num_tests == 0: - return True - return (num_tests - num_tests_missing_data) / num_tests > REQUIRED_STATS_THRESHOLD - def _get_clean_every_n_cadence(self, suite_name: str, is_asan: bool) -> int: """ Get the N value for the CleanEveryN hook. diff --git a/buildscripts/util/teststats.py b/buildscripts/util/teststats.py index 74ef9c5ab3e..a52fa3c79a4 100644 --- a/buildscripts/util/teststats.py +++ b/buildscripts/util/teststats.py @@ -1,35 +1,15 @@ """Utility to support parsing a TestStat.""" from collections import defaultdict from dataclasses import dataclass +from datetime import datetime from itertools import chain -from json import JSONDecodeError - from typing import NamedTuple, List, Callable, Optional -import requests -from requests.adapters import HTTPAdapter, Retry + +from evergreen import EvergreenApi, TestStats from buildscripts.util.testname import split_test_hook_name, is_resmoke_hook, get_short_name_from_test_file TASK_LEVEL_HOOKS = {"CleanEveryN"} -TESTS_STATS_S3_LOCATION = "https://mongo-test-stats.s3.amazonaws.com" - - -class HistoricalTestInformation(NamedTuple): - """ - Container for information about the historical runtime of a test. - - test_name: Name of test. - avg_duration_pass: Average of runtime of test that passed. - num_pass: Number of times the test has passed. - num_fail: Number of times the test has failed. - max_duration_pass: Maximum runtime of the test when it passed. - """ - - test_name: str - num_pass: int - num_fail: int - avg_duration_pass: float - max_duration_pass: Optional[float] = None class TestRuntime(NamedTuple): @@ -94,9 +74,9 @@ class HistoricHookInfo(NamedTuple): avg_duration: float @classmethod - def from_test_stats(cls, test_stats: HistoricalTestInformation) -> "HistoricHookInfo": + def from_test_stats(cls, test_stats: TestStats) -> "HistoricHookInfo": """Create an instance from a test_stats object.""" - return cls(hook_id=test_stats.test_name, num_pass=test_stats.num_pass, + return cls(hook_id=test_stats.test_file, num_pass=test_stats.num_pass, avg_duration=test_stats.avg_duration_pass) def test_name(self) -> str: @@ -121,10 +101,10 @@ class HistoricTestInfo(NamedTuple): hooks: List[HistoricHookInfo] @classmethod - def from_test_stats(cls, test_stats: HistoricalTestInformation, + def from_test_stats(cls, test_stats: TestStats, hooks: List[HistoricHookInfo]) -> "HistoricTestInfo": """Create an instance from a test_stats object.""" - return cls(test_name=test_stats.test_name, num_pass=test_stats.num_pass, + return cls(test_name=test_stats.test_file, num_pass=test_stats.num_pass, avg_duration=test_stats.avg_duration_pass, hooks=hooks) def normalized_test_name(self) -> str: @@ -143,9 +123,7 @@ class HistoricTestInfo(NamedTuple): def total_test_runtime(self) -> float: """Get the average runtime of this test and it's non-task level hooks.""" - if self.num_pass > 0: - return self.avg_duration + self.total_hook_runtime(lambda h: not h.is_task_level_hook()) - return 0.0 + return self.avg_duration + self.total_hook_runtime(lambda h: not h.is_task_level_hook()) def get_hook_overhead(self) -> float: """Get the average runtime of this test and it's non-task level hooks.""" @@ -159,59 +137,46 @@ class HistoricTaskData(object): """Initialize the TestStats with raw results from the Evergreen API.""" self.historic_test_results = historic_test_results - @staticmethod - def get_stats_from_s3(project: str, task: str, variant: str) -> List[HistoricalTestInformation]: - """ - Retrieve test stats from s3 for a given task. - - :param project: Project to query. - :param task: Task to query. - :param variant: Build variant to query. - :return: A list of the Test stats for the specified task. - """ - session = requests.Session() - retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504]) - session.mount('https://', HTTPAdapter(max_retries=retries)) - - response = session.get(f"{TESTS_STATS_S3_LOCATION}/{project}/{variant}/{task}") - - try: - data = response.json() - return [HistoricalTestInformation(**item) for item in data] - except JSONDecodeError: - return [] - + # pylint: disable=too-many-arguments @classmethod - def from_s3(cls, project: str, task: str, variant: str) -> "HistoricTaskData": + def from_evg(cls, evg_api: EvergreenApi, project: str, start_date: datetime, end_date: datetime, + task: str, variant: str) -> "HistoricTaskData": """ - Retrieve test stats from s3 for a given task. + Retrieve test stats from evergreen for a given task. + :param evg_api: Evergreen API client. :param project: Project to query. + :param start_date: Start date to query. + :param end_date: End date to query. :param task: Task to query. :param variant: Build variant to query. :return: Test stats for the specified task. """ - historical_test_data = cls.get_stats_from_s3(project, task, variant) - return cls.from_stats_list(historical_test_data) + days = (end_date - start_date).days + historic_stats = evg_api.test_stats_by_project( + project, after_date=start_date, before_date=end_date, tasks=[task], variants=[variant], + group_by="test", group_num_days=days) + + return cls.from_stats_list(historic_stats) @classmethod - def from_stats_list( - cls, historical_test_data: List[HistoricalTestInformation]) -> "HistoricTaskData": + def from_stats_list(cls, historic_stats: List[TestStats]) -> "HistoricTaskData": """ Build historic task data from a list of historic stats. - :param historical_test_data: A list of information about the runtime of a test. + :param historic_stats: List of historic stats to build from. :return: Historic task data from the list of stats. """ + hooks = defaultdict(list) - for hook in [stat for stat in historical_test_data if is_resmoke_hook(stat.test_name)]: + for hook in [stat for stat in historic_stats if is_resmoke_hook(stat.test_file)]: historical_hook = HistoricHookInfo.from_test_stats(hook) hooks[historical_hook.test_name()].append(historical_hook) return cls([ HistoricTestInfo.from_test_stats(stat, - hooks[get_short_name_from_test_file(stat.test_name)]) - for stat in historical_test_data if not is_resmoke_hook(stat.test_name) + hooks[get_short_name_from_test_file(stat.test_file)]) + for stat in historic_stats if not is_resmoke_hook(stat.test_file) ]) def get_tests_runtimes(self) -> List[TestRuntime]: diff --git a/buildscripts/validate_commit_message.py b/buildscripts/validate_commit_message.py index da01fa75a62..3aff60e89c1 100755 --- a/buildscripts/validate_commit_message.py +++ b/buildscripts/validate_commit_message.py @@ -28,46 +28,253 @@ # """Validate that the commit message is ok.""" import argparse +import logging +import os import re import sys -import logging +from typing import List, Optional + +from evergreen import EvergreenApi, RetryingEvergreenApi + +from buildscripts.client.jiraclient import JiraAuth, JiraClient, SecurityLevel + +JIRA_SERVER = "https://jira.mongodb.org" +EVG_CONFIG_FILE = "~/.evergreen.yml" +SERVER_TICKET_PREFIX = "SERVER-" +PUBLIC_PROJECT_PREFIX = "mongodb-mongo-" LOGGER = logging.getLogger(__name__) +ERROR_MSG = """ +################################################################################ +Encountered an invalid commit message. Please correct to the commit message to +continue. + +Commit message should start with a Public Jira ticket, an "Import" for wiredtiger +or tools, or a "Revert" message. + +{error_msg} on '{branch}': +'{commit_message}' +################################################################################ +""" + +COMMON_PUBLIC_PATTERN = r""" + ((?P<revert>Revert)\s+[\"\']?)? # Revert (optional) + ((?P<ticket>(?:EVG|SERVER|WT)-[0-9]+)[\"\']?\s*) # ticket identifier + (?P<body>(?:(?!\(cherry\spicked\sfrom).)*)? # To also capture the body + (?P<backport>\(cherry\spicked\sfrom.*)? # back port (optional) + """ +"""Common Public pattern format.""" + +COMMON_LINT_PATTERN = r"(?P<lint>Fix\slint)" +"""Common Lint pattern format.""" + +COMMON_IMPORT_PATTERN = r"(?P<imported>Import\s(wiredtiger|tools):\s.*)" +"""Common Import pattern format.""" + +COMMON_REVERT_IMPORT_PATTERN = (r"Revert\s+[\"\']?(?P<imported>Import\s(wiredtiger|tools):\s.*)") +"""Common revert Import pattern format.""" + +COMMON_PRIVATE_PATTERN = r""" + ((?P<revert>Revert)\s+[\"\']?)? # Revert (optional) + ((?P<ticket>[A-Z]+-[0-9]+)[\"\']?\s*) # ticket identifier + (?P<body>(?:(?!('\s(into\s'(([^/]+))/(([^:]+)):(([^']+))'))).)*)? # To also capture the body +""" +"""Common Private pattern format.""" + STATUS_OK = 0 STATUS_ERROR = 1 -def main(argv=None): +def new_patch_description(pattern: str) -> str: + """ + Wrap the pattern to conform to the new commit queue patch description format. + + Add the commit queue prefix and suffix to the pattern. The format looks like: + + Commit Queue Merge: '<commit message>' into '<owner>/<repo>:<branch>' + + :param pattern: The pattern to wrap. + :return: A pattern to match the new format for the patch description. + """ + return (r"""^((?P<commitqueue>Commit\sQueue\sMerge:)\s')""" + f"{pattern}" + # r"""('\s(?P<into>into\s'((?P<owner>[^/]+))/((?P<repo>[^:]+)):((?P<branch>[^']+))'))""" + ) + + +def old_patch_description(pattern: str) -> str: + """ + Wrap the pattern to conform to the new commit queue patch description format. + + Just add a start anchor. The format looks like: + + <commit message> + + :param pattern: The pattern to wrap. + :return: A pattern to match the old format for the patch description. + """ + return r"^" f"{pattern}" + + +# NOTE: re.VERBOSE is for visibility / debugging. As such significant white space must be +# escaped (e.g ' ' to \s). +VALID_PATTERNS = [ + re.compile( + new_patch_description(COMMON_PUBLIC_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + old_patch_description(COMMON_PUBLIC_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + new_patch_description(COMMON_LINT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + old_patch_description(COMMON_LINT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + new_patch_description(COMMON_IMPORT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + old_patch_description(COMMON_IMPORT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + new_patch_description(COMMON_REVERT_IMPORT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + old_patch_description(COMMON_REVERT_IMPORT_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), +] +"""valid public patterns.""" + +PRIVATE_PATTERNS = [ + re.compile( + new_patch_description(COMMON_PRIVATE_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), + re.compile( + old_patch_description(COMMON_PRIVATE_PATTERN), + re.MULTILINE | re.DOTALL | re.VERBOSE, + ), +] +"""private patterns.""" + + +class CommitMessageValidationOrchestrator: + """An orchestrator to validate that commit messages are valid.""" + + def __init__(self, evg_api: EvergreenApi, jira_client: JiraClient) -> None: + """ + Initialize the orchestrator. + + :param evg_api: Evergreen API client. + :param jira_client: Client to Jira API. + """ + self.evg_api = evg_api + self.jira_client = jira_client + + def validate_ticket(self, ticket: str, project: str) -> bool: + """ + Check that the given Jira ticket has a proper security level. + + Commits targeting a public project should not have a defined security level (these are + public by default). + + :param ticket: Ticket to check. + :param project: Project commit is targeting. + :return: True if ticket is valid. + """ + if ticket.startswith(SERVER_TICKET_PREFIX) and project.startswith(PUBLIC_PROJECT_PREFIX): + security_level = self.jira_client.get_ticket_security_level(ticket) + return security_level == SecurityLevel.NONE + return True + + def validate_msg(self, message: str, project: str) -> bool: + """ + Check that the given message is valid. + + :param message: Commit message to validate. + :param project: Project commit is targeting. + :return: True if the message is valid. + """ + valid_matches = [valid_pattern.match(message) for valid_pattern in VALID_PATTERNS] + if any(valid_matches): + ticket_matches = [pattern.match(message) for pattern in VALID_PATTERNS[0:2]] + for match in [ticket_match for ticket_match in ticket_matches if ticket_match]: + if not self.validate_ticket(match.group("ticket"), project): + print( + ERROR_MSG.format( + error_msg="Reference to a internal Jira Ticket", + branch=project, + commit_message=message, + )) + return False + return True + elif any(private_pattern.match(message) for private_pattern in PRIVATE_PATTERNS): + print( + ERROR_MSG.format( + error_msg="Reference to a private project", + branch=project, + commit_message=message, + )) + return False + else: + print( + ERROR_MSG.format( + error_msg="Commit without a ticket", + branch=project, + commit_message=message, + )) + return False + + def validate_commit_messages(self, version_id: str) -> int: + """ + Validate the commit messages for the given build. + + :param version_id: ID of version to validate. + :param evg_api: Evergreen API client. + :return: True if all commit messages were valid. + """ + found_error = False + code_changes = self.evg_api.patch_by_id(version_id).module_code_changes + for change in code_changes: + for message in change.commit_messages: + is_valid = self.validate_msg(message, change.branch_name) + found_error = found_error or not is_valid + + return STATUS_ERROR if found_error else STATUS_OK + + +def main(argv: Optional[List[str]] = None) -> int: """Execute Main function to validate commit messages.""" parser = argparse.ArgumentParser( usage="Validate the commit message. " "It validates the latest message when no arguments are provided.") parser.add_argument( - "message", - metavar="commit message", - nargs="*", - help="The commit message to validate", + "version_id", + metavar="version id", + help="The id of the version to validate", + ) + parser.add_argument( + "--evg-config-file", + default=EVG_CONFIG_FILE, + help="Path to evergreen configuration file containing auth information.", ) args = parser.parse_args(argv) + evg_api = RetryingEvergreenApi.get_api(config_file=os.path.expanduser(args.evg_config_file)) + jira_auth = JiraAuth() + jira_client = JiraClient(JIRA_SERVER, jira_auth) + orchestrator = CommitMessageValidationOrchestrator(evg_api, jira_client) - if not args.message: - LOGGER.error("Must specify non-empty value for --message") - return STATUS_ERROR - message = " ".join(args.message) - - # Valid values look like: - # 1. SERVER-\d+ - # 2. Revert "SERVER-\d+ - # 3. Import wiredtiger - # 4. Revert "Import wiredtiger - valid_pattern = re.compile(r'(Revert ")?(SERVER-[0-9]+|Import wiredtiger)') - - if valid_pattern.match(message): - return STATUS_OK - else: - LOGGER.error(f"Found a commit without a ticket\n{message}") # pylint: disable=logging-fstring-interpolation - return STATUS_ERROR + return orchestrator.validate_commit_messages(args.version_id) if __name__ == "__main__": |