diff options
Diffstat (limited to 'buildscripts/resmokelib/run')
| -rw-r--r-- | buildscripts/resmokelib/run/__init__.py | 77 | ||||
| -rwxr-xr-x | buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py | 49 |
2 files changed, 62 insertions, 64 deletions
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index e18099de679..20ff72c2c79 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -175,8 +175,7 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes def generate_multiversion_exclude_tags(self): """Generate multiversion exclude tags file.""" generate_multiversion_exclude_tags.generate_exclude_yaml( - config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, config.EXPANSIONS_FILE, - self._resmoke_logger) + config.MULTIVERSION_BIN_VERSION, config.EXCLUDE_TAGS_FILE_PATH, self._resmoke_logger) @staticmethod def _find_suites_by_test(suites): @@ -386,12 +385,25 @@ class TestRunnerEvg(TestRunner): additional options for running unreliable tests in Evergreen. """ + UNRELIABLE_TAG = _TagInfo( + tag_name="unreliable", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + report_failure_status="silentfail")) + RESOURCE_INTENSIVE_TAG = _TagInfo( tag_name="resource_intensive", evergreen_aware=False, suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore num_jobs=1)) + RETRY_ON_FAILURE_TAG = _TagInfo( + tag_name="retry_on_failure", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, + report_failure_status="silentfail")) + @staticmethod def _make_evergreen_aware_tags(tag_name): """Return a list of resmoke.py tags. @@ -426,8 +438,29 @@ class TestRunnerEvg(TestRunner): combinations = [] - combinations.append(("resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, True)])) - combinations.append(("not resource intensive", [(cls.RESOURCE_INTENSIVE_TAG, False)])) + if config.EVERGREEN_PATCH_BUILD: + combinations.append(("unreliable and resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("unreliable and not resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) + combinations.append(("reliable and resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("reliable and not resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + else: + combinations.append(("retry on failure and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("retry on failure and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + combinations.append(("run once and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("run once and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) return combinations @@ -707,19 +740,15 @@ class RunPlugin(PluginInterface): ) parser.add_argument( - "--runNoFeatureFlagTests", dest="run_no_feature_flag_tests", action="store_true", - help=("Do not run any tests tagged with enabled feature flags." - " This argument has precedence over --runAllFeatureFlagTests" - "; used for multiversion suites")) + "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests", + action="store_true", help= + "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites" + ) parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags", action="append", metavar="featureFlag1, featureFlag2, ...", help="Additional feature flags") - parser.add_argument("--additionalFeatureFlagsFile", dest="additional_feature_flags_file", - action="store", metavar="FILE", - help="The path to a file with feature flags, delimited by newlines.") - parser.add_argument("--maxTestQueueSize", type=int, dest="max_test_queue_size", help=argparse.SUPPRESS) @@ -761,11 +790,6 @@ class RunPlugin(PluginInterface): metavar="ON|OFF", help=("Enable or disable majority read concern support." " Defaults to %(default)s.")) - mongodb_server_options.add_argument( - "--enableEnterpriseTests", action="store", dest="enable_enterprise_tests", default="on", - choices=("on", "off"), metavar="ON|OFF", - help=("Enable or disable enterprise tests. Defaults to 'on'.")) - mongodb_server_options.add_argument("--flowControl", action="store", dest="flow_control", choices=("on", "off"), metavar="ON|OFF", help=("Enable or disable flow control.")) @@ -855,6 +879,13 @@ class RunPlugin(PluginInterface): help="Writes a JSON file with performance test results.") internal_options.add_argument( + "--reportFailureStatus", action="store", dest="report_failure_status", + choices=("fail", "silentfail"), metavar="STATUS", + help="Controls if the test failure status should be reported as failed" + " or be silently ignored (STATUS=silentfail). Dynamic test failures will" + " never be silently ignored. Defaults to STATUS=%(default)s.") + + internal_options.add_argument( "--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") @@ -952,9 +983,6 @@ class RunPlugin(PluginInterface): evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") - evergreen_options.add_argument("--taskWorkDir", dest="work_dir", metavar="TASK_WORK_DIR", - help="Sets the working directory of the task.") - benchmark_options = parser.add_argument_group( title=_BENCHMARK_ARGUMENT_TITLE, description="Options for running Benchmark/Benchrun tests") @@ -1068,6 +1096,15 @@ def to_local_args(input_args=None): # pylint: disable=too-many-branches,too-man if origin_suite is not None: setattr(parsed_args, "suite_files", origin_suite) + # Replace --runAllFeatureFlagTests with an explicit list of feature flags. The former relies on + # all_feature_flags.txt which may not exist in the local dev environment. + run_all_feature_flag_tests = getattr(parsed_args, "run_all_feature_flag_tests", None) + if run_all_feature_flag_tests is not None: + setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS) + del parsed_args.run_all_feature_flag_tests + + del parsed_args.run_all_feature_flags_no_tests + # The top-level parser has one subparser that contains all subcommand parsers. command_subparser = [ action for action in parser._actions # pylint: disable=protected-access diff --git a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py index d5c904bb2c3..98be3db0fa8 100755 --- a/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py +++ b/buildscripts/resmokelib/run/generate_multiversion_exclude_tags.py @@ -5,8 +5,6 @@ import re import tempfile from collections import defaultdict from subprocess import check_output -from typing import Optional -from github import GithubIntegration import requests @@ -15,35 +13,13 @@ from buildscripts.resmokelib.config import MultiversionOptions from buildscripts.resmokelib.core.programs import get_path_env_var from buildscripts.resmokelib.utils import is_windows from buildscripts.util.fileops import read_yaml_file -from buildscripts.util.read_config import read_config_file BACKPORT_REQUIRED_TAG = "backport_required_multiversion" # The directory in which BACKPORTS_REQUIRED_FILE resides. ETC_DIR = "etc" BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml" -BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/10gen/mongo" - - -def get_installation_access_token(app_id: int, private_key: str, - installation_id: int) -> Optional[str]: # noqa: D406,D407,D413 - """ - Obtain an installation access token using JWT. - - Args: - - app_id: The application ID for GitHub App. - - private_key: The private key associated with the GitHub App. - - installation_id: The installation ID of the GitHub App for a particular account. - - Returns: - - Optional[str]: The installation access token. Returns `None` if there's an error obtaining the token. - """ - integration = GithubIntegration(app_id, private_key) - auth = integration.get_access_token(installation_id) - if auth: - return auth.token - else: - raise Exception("Error obtaining installation token") +BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo" def get_backports_required_hash_for_shell_version(mongo_shell_path=None): @@ -75,24 +51,10 @@ def get_backports_required_hash_for_shell_version(mongo_shell_path=None): f"Could not find a valid commit hash from the {mongo_shell_path} mongo binary.") -def get_old_yaml(commit_hash, expansions_file): +def get_old_yaml(commit_hash): """Download BACKPORTS_REQUIRED_FILE from the old commit and return the yaml.""" - - if not os.path.exists(expansions_file): - raise FileNotFoundError(f"The specified file does not exist: {expansions_file}") - expansions = read_config_file(expansions_file) - - # Obtain installation access tokens using app credentials - access_token_10gen_mongo = get_installation_access_token( - expansions["app_id_10gen_mongo"], expansions["private_key_10gen_mongo"], - expansions["installation_id_10gen_mongo"]) - response = requests.get( - f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}', - headers={ - 'Authorization': f'token {access_token_10gen_mongo}', - }) - + f'{BACKPORTS_REQUIRED_BASE_URL}/{commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}') # If the response was successful, no exception will be raised. response.raise_for_status() @@ -106,8 +68,7 @@ def get_old_yaml(commit_hash, expansions_file): return backports_required_old -def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: str, - logger: logging.Logger) -> None: +def generate_exclude_yaml(old_bin_version: str, output: str, logger: logging.Logger) -> None: """ Create a tag file associating multiversion tests to tags for exclusion. @@ -138,7 +99,7 @@ def generate_exclude_yaml(old_bin_version: str, output: str, expansions_file: st # Get the yaml contents from the old commit. logger.info(f"Downloading file from commit hash of old branch {old_version_commit_hash}") - backports_required_old = get_old_yaml(old_version_commit_hash, expansions_file) + backports_required_old = get_old_yaml(old_version_commit_hash) def diff(list1, list2): return [elem for elem in (list1 or []) if elem not in (list2 or [])] |