diff options
Diffstat (limited to 'buildscripts/tests')
| -rw-r--r-- | buildscripts/tests/burn_in_tests_end2end/__init__.py | 1 | ||||
| -rw-r--r-- | buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py | 42 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/evergreen.yml | 17 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 303 | ||||
| -rw-r--r-- | buildscripts/tests/resmoke_validation/test_jstest_tags.py | 65 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tags.py | 271 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tags_evergreen.yml | 134 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 126 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_burn_in_tests.py | 399 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_gen_build_variant.py | 382 | ||||
| -rw-r--r-- | buildscripts/tests/test_selected_tests.py | 82 | ||||
| -rw-r--r-- | buildscripts/tests/test_validate_commit_message.py | 123 | ||||
| -rw-r--r-- | buildscripts/tests/timeouts/test_timeout_service.py | 6 | ||||
| -rw-r--r-- | buildscripts/tests/util/test_taskname.py | 6 |
14 files changed, 426 insertions, 1531 deletions
diff --git a/buildscripts/tests/burn_in_tests_end2end/__init__.py b/buildscripts/tests/burn_in_tests_end2end/__init__.py new file mode 100644 index 00000000000..4b7a2bb941b --- /dev/null +++ b/buildscripts/tests/burn_in_tests_end2end/__init__.py @@ -0,0 +1 @@ +"""Empty.""" diff --git a/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py b/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py new file mode 100644 index 00000000000..cea56d16e9d --- /dev/null +++ b/buildscripts/tests/burn_in_tests_end2end/test_burn_in_tests_end2end.py @@ -0,0 +1,42 @@ +"""E2E tests for the buildscripts/burn_in_tests.py.""" + +import os +import subprocess +import sys +import unittest + +import yaml + +import buildscripts.burn_in_tests as under_test + +# pylint: disable=missing-docstring,protected-access,invalid-name,subprocess-run-check,broad-except + + +class TestBurnInTestsEnd2End(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([ + sys.executable, + "buildscripts/burn_in_tests.py", + "generate-test-membership-map-file-for-ci", + ]) + + @classmethod + def tearDownClass(cls): + if os.path.exists(under_test.BURN_IN_TEST_MEMBERSHIP_FILE): + os.remove(under_test.BURN_IN_TEST_MEMBERSHIP_FILE) + + def test_valid_yaml_output(self): + process = subprocess.run([ + sys.executable, + "buildscripts/burn_in_tests.py", + "run", + "--yaml", + ], text=True, capture_output=True) + output = process.stdout + self.assertEqual(0, process.returncode) + + try: + yaml.safe_load(output) + except Exception: + self.fail(msg="burn_in_tests.py does not output valid yaml.") diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml index 60fd7628959..9a03764e5cd 100644 --- a/buildscripts/tests/ciconfig/evergreen.yml +++ b/buildscripts/tests/ciconfig/evergreen.yml @@ -141,6 +141,22 @@ tasks: vars: resmoke_args: "--storageEngine=wiredTiger" +- name: resmoke_multiversion_task_gen + depends_on: + - name: compile + commands: + - func: "initialize multiversion tasks" + vars: + multiversion_sanity_check_last_continuous_new_new_old: last_continuous + multiversion_sanity_check_last_continuous_new_old_new: last_continuous + multiversion_sanity_check_last_continuous_old_new_new: last_continuous + multiversion_sanity_check_last_lts_new_new_old: last_lts + multiversion_sanity_check_last_lts_new_old_new: last_lts + multiversion_sanity_check_last_lts_old_new_new: last_lts + - func: "generate resmoke tasks" + vars: + resmoke_args: "--storageEngine=wiredTiger" + modules: - name: render-module @@ -183,6 +199,7 @@ buildvariants: - debian-stretch tasks: - name: resmoke_task + - name: resmoke_multiversion_task_gen - name: amazon display_name: "! Amazon" run_on: diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index 32a8cd330f3..20a25d35801 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -6,7 +6,7 @@ import unittest import buildscripts.ciconfig.evergreen as _evergreen -# pylint: disable=missing-docstring,protected-access +# pylint: disable=missing-docstring,protected-access,invalid-name TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "evergreen.yml") @@ -24,14 +24,15 @@ class TestEvergreenProjectConfig(unittest.TestCase): _evergreen.parse_evergreen_file(invalid_path, evergreen_binary=None) def test_list_tasks(self): - self.assertEqual(6, len(self.conf.tasks)) - self.assertEqual(6, len(self.conf.task_names)) + self.assertEqual(7, len(self.conf.tasks)) + self.assertEqual(7, len(self.conf.task_names)) self.assertIn("compile", self.conf.task_names) self.assertIn("passing_test", self.conf.task_names) self.assertIn("failing_test", self.conf.task_names) self.assertIn("timeout_test", self.conf.task_names) self.assertIn("no_lifecycle_task", self.conf.task_names) self.assertIn("resmoke_task", self.conf.task_names) + self.assertIn("resmoke_multiversion_task_gen", self.conf.task_names) def test_list_task_groups(self): self.assertEqual(1, len(self.conf.task_groups)) @@ -83,55 +84,93 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertEqual([], task.depends_on) self.assertEqual(task_dict, task.raw) - def test_resmoke_args(self): + def test_suite_to_resmoke_args_map_for_non_gen_task(self): suite_and_task = "jstestfuzz" - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--arg=val"}}] + task_commands = [{ + "func": "run tests", + "vars": {"resmoke_args": "--arg=val"}, + }] task_dict = {"name": suite_and_task, "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual(f"--suites={suite_and_task} --arg=val", task.resmoke_args) + self.assertEqual({suite_and_task: f"--suites={suite_and_task} --arg=val"}, + task.suite_to_resmoke_args_map) - def test_is_run_tests_task(self): - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] - task_dict = {"name": "jsCore", "commands": task_commands} + def test_suite_to_resmoke_args_map_for_gen_task(self): + suite = "jsCore" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"resmoke_args": "--installDir=/bin"}, + }] + task_dict = {"name": f"{suite}_gen", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertTrue(task.is_run_tests_task) - self.assertFalse(task.is_generate_resmoke_task) + self.assertEqual({suite: f"--suites={suite} --installDir=/bin"}, + task.suite_to_resmoke_args_map) - def test_run_tests_command(self): - task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] + def test_suite_to_resmoke_args_map_for_gen_task_with_suite(self): + suite = "core" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"suite": suite, "resmoke_args": "--installDir=/bin"}, + }] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertDictEqual(task_commands[0], task.run_tests_command) - - def test_run_tests_multiversion(self): - require_multiversion_setup = True - task_commands = [{"func": "do multiversion setup"}, - {"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] - task_dict = {"name": "jsCore", "commands": task_commands, "tags": ["multiversion"]} + self.assertEqual({suite: f"--suites={suite} --installDir=/bin"}, + task.suite_to_resmoke_args_map) + + def test_suite_to_resmoke_args_map_for_initialize_multiversion_tasks_task(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + { + "func": "generate resmoke tasks", + "vars": {"resmoke_args": "--installDir=/bin"}, + }, + ] + task_dict = {"name": "multiversion_sanity_check_gen", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual(task.multiversion_setup_command, {"func": "do multiversion setup"}) - self.assertEqual(require_multiversion_setup, task.require_multiversion_setup()) + self.assertEqual({ + "multiversion_sanity_check_last_continuous_new_new_old": + "--suites=multiversion_sanity_check_last_continuous_new_new_old --installDir=/bin", + "multiversion_sanity_check_last_continuous_new_old_new": + "--suites=multiversion_sanity_check_last_continuous_new_old_new --installDir=/bin", + "multiversion_sanity_check_last_continuous_old_new_new": + "--suites=multiversion_sanity_check_last_continuous_old_new_new --installDir=/bin", + "multiversion_sanity_check_last_lts_new_new_old": + "--suites=multiversion_sanity_check_last_lts_new_new_old --installDir=/bin", + "multiversion_sanity_check_last_lts_new_old_new": + "--suites=multiversion_sanity_check_last_lts_new_old_new --installDir=/bin", + "multiversion_sanity_check_last_lts_old_new_new": + "--suites=multiversion_sanity_check_last_lts_old_new_new --installDir=/bin", + }, task.suite_to_resmoke_args_map) - def test_run_tests_no_multiversion(self): + def test_is_run_tests_task(self): task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertFalse(task.require_multiversion_setup()) - self.assertIsNone(task.multiversion_setup_command) + self.assertTrue(task.is_run_tests_task) + self.assertFalse(task.is_generate_resmoke_task) + self.assertFalse(task.is_initialize_multiversion_tasks_task) - def test_resmoke_args_gen(self): - task_commands = [{ - "func": "generate resmoke tasks", "vars": {"resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore_gen", "commands": task_commands} + def test_run_tests_command(self): + task_commands = [{"func": "run tests", "vars": {"resmoke_args": "--suites=core"}}] + task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual("--suites=jsCore --installDir=/bin", task.resmoke_args) + self.assertDictEqual(task_commands[0], task.run_tests_command) def test_is_generate_resmoke_task(self): task_name = "core" @@ -144,6 +183,7 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertTrue(task.is_generate_resmoke_task) self.assertFalse(task.is_run_tests_task) + self.assertFalse(task.is_initialize_multiversion_tasks_task) def test_generate_resmoke_tasks_command(self): task_commands = [{ @@ -155,17 +195,74 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual("jsCore", task.generated_task_name) - def test_resmoke_args_gen_with_suite(self): - task_name = "jsCore" - suite_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "suite": suite_name, "resmoke_args": "--installDir=/bin"} - }] + def test_is_initialize_multiversion_tasks_task(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ] + task = _evergreen.Task({ + "name": "multiversion_sanity_check_gen", + "commands": task_commands, + }) + + self.assertTrue(task.is_initialize_multiversion_tasks_task) + self.assertTrue(task.is_generate_resmoke_task) + self.assertFalse(task.is_run_tests_task) + + def test_initialize_multiversion_tasks_command(self): + task_commands = [ + { + "func": "initialize multiversion tasks", + "vars": { + "multiversion_sanity_check_last_continuous_new_new_old": "last_continuous", + "multiversion_sanity_check_last_continuous_new_old_new": "last_continuous", + "multiversion_sanity_check_last_continuous_old_new_new": "last_continuous", + "multiversion_sanity_check_last_lts_new_new_old": "last_lts", + "multiversion_sanity_check_last_lts_new_old_new": "last_lts", + "multiversion_sanity_check_last_lts_old_new_new": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ] + task = _evergreen.Task({ + "name": "multiversion_sanity_check_gen", + "commands": task_commands, + }) + + self.assertDictEqual(task_commands[0], task.initialize_multiversion_tasks_command) + self.assertEqual("multiversion_sanity_check", task.generated_task_name) + + def test_get_resmoke_command_vars_from_run_tests_command(self): + resmoke_command_vars = {"suite": "core"} + task_commands = [{"func": "run tests", "vars": resmoke_command_vars}] task_dict = {"name": "jsCore", "commands": task_commands} task = _evergreen.Task(task_dict) - self.assertEqual("--suites=core --installDir=/bin", task.resmoke_args) + self.assertEqual(resmoke_command_vars, task.get_resmoke_command_vars()) + + def test_get_resmoke_command_vars_from_generate_resmoke_tasks_command(self): + resmoke_command_vars = {"suite": "core"} + task_commands = [{"func": "generate resmoke tasks", "vars": resmoke_command_vars}] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertEqual(resmoke_command_vars, task.get_resmoke_command_vars()) + + def test_get_resmoke_command_vars_from_non_resmoke_task(self): + task_dict = {"name": "compile", "commands": []} + task = _evergreen.Task(task_dict) + + self.assertEqual({}, task.get_resmoke_command_vars()) def test_tags_with_no_tags(self): task_dict = { @@ -208,49 +305,61 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual("jsCore", task.generated_task_name) - def test_gen_resmoke_multiversion(self): - require_multiversion_setup = True - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore", "commands": task_commands, "tags": ["multiversion"]} - task = _evergreen.Task(task_dict) - - self.assertEqual(require_multiversion_setup, task.require_multiversion_setup()) + def test_get_suite_names_from_non_gen_task_name(self): + task = _evergreen.Task({ + "name": "task_name", + "commands": [{"func": "run tests"}], + }) - def test_gen_resmoke_no_multiversion(self): - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["task_name"], task.get_suite_names()) - self.assertFalse(task.require_multiversion_setup()) + def test_get_suite_names_from_non_gen_task_suite_var(self): + task = _evergreen.Task({ + "name": "task_name", + "commands": [{ + "func": "run tests", + "vars": {"suite": "suite_var"}, + }], + }) - def test_get_vars_suite_name_generate_resmoke_tasks(self): - task_name = "jsCore" - suite_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "suite": suite_name, "resmoke_args": "--installDir=/bin"} - }] - task_dict = {"name": task_name, "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["suite_var"], task.get_suite_names()) - self.assertEqual(suite_name, task.get_suite_name()) + def test_get_suite_names_from_gen_task_name(self): + task = _evergreen.Task({ + "name": "task_name_gen", + "commands": [{"func": "generate resmoke tasks"}], + }) - def test_get_suite_name_default_to_task_name(self): - task_name = "concurrency_gen" - no_gen_task_name = "concurrency" - task_commands = [{"func": "generate resmoke tasks"}] - task_dict = {"name": task_name, "commands": task_commands} - task = _evergreen.Task(task_dict) + self.assertEqual(["task_name"], task.get_suite_names()) - self.assertEqual(no_gen_task_name, task.get_suite_name()) + def test_get_suite_names_from_gen_task_suite_var(self): + task = _evergreen.Task({ + "name": "task_name_gen", + "commands": [{ + "func": "generate resmoke tasks", + "vars": {"suite": "suite_var"}, + }], + }) + + self.assertEqual(["suite_var"], task.get_suite_names()) + + def test_get_suite_names_from_init_multiversion_task(self): + task = _evergreen.Task({ + "name": + "task_name_multiversion_gen", + "commands": [ + { + "func": "initialize multiversion tasks", + "vars": { + "suite_last_continuous": "last_continuous", + "suite_last_lts": "last_lts", + }, + }, + {"func": "generate resmoke tasks"}, + ], + }) + + self.assertEqual(["suite_last_continuous", "suite_last_lts"], task.get_suite_names()) def test_generate_task_name_non_gen_tasks(self): task_name = "jsCore" @@ -359,10 +468,10 @@ class TestVariant(unittest.TestCase): def test_distro_names(self): variant_ubuntu = self.conf.get_variant("ubuntu") - self.assertEqual(set(["ubuntu1404-test", "pdp-11"]), variant_ubuntu.distro_names) + self.assertEqual({"ubuntu1404-test", "pdp-11"}, variant_ubuntu.distro_names) variant_osx = self.conf.get_variant("osx-108") - self.assertEqual(set(["localtestdistro"]), variant_osx.distro_names) + self.assertEqual({"localtestdistro"}, variant_osx.distro_names) def test_test_flags(self): variant_ubuntu = self.conf.get_variant("ubuntu") @@ -389,21 +498,41 @@ class TestVariant(unittest.TestCase): self.assertEqual(variant_ubuntu, task.variant) self.assertIn(task_name, variant_ubuntu.task_names) - # Check combined_resmoke_args when test_flags is set on the variant. + # Check combined_suite_to_resmoke_args_map when test_flags is set on the variant. resmoke_task = variant_ubuntu.get_task("resmoke_task") - self.assertEqual("--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu", - resmoke_task.combined_resmoke_args) + self.assertEqual({ + "resmoke_task": + "--suites=resmoke_task --storageEngine=wiredTiger --param=value --ubuntu" + }, resmoke_task.combined_suite_to_resmoke_args_map) - # Check combined_resmoke_args when the task doesn't have resmoke_args. + # Check combined_suite_to_resmoke_args_map when the task doesn't have resmoke_args. passing_task = variant_ubuntu.get_task("passing_test") - self.assertEqual("--suites=passing_test --param=value --ubuntu", - passing_task.combined_resmoke_args) + self.assertEqual({"passing_test": "--suites=passing_test --param=value --ubuntu"}, + passing_task.combined_suite_to_resmoke_args_map) - # Check combined_resmoke_args when test_flags is not set on the variant. + # Check combined_suite_to_resmoke_args_map when test_flags is not set on the variant. variant_debian = self.conf.get_variant("debian") resmoke_task = variant_debian.get_task("resmoke_task") - self.assertEqual("--suites=resmoke_task --storageEngine=wiredTiger", - resmoke_task.combined_resmoke_args) + self.assertEqual({"resmoke_task": "--suites=resmoke_task --storageEngine=wiredTiger"}, + resmoke_task.combined_suite_to_resmoke_args_map) + + # Check combined_suite_to_resmoke_args_map for "initialize multiversion tasks" task. + variant_debian = self.conf.get_variant("debian") + resmoke_task = variant_debian.get_task("resmoke_multiversion_task_gen") + self.assertEqual({ + "multiversion_sanity_check_last_continuous_new_new_old": + "--suites=multiversion_sanity_check_last_continuous_new_new_old --storageEngine=wiredTiger", + "multiversion_sanity_check_last_continuous_new_old_new": + "--suites=multiversion_sanity_check_last_continuous_new_old_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_continuous_old_new_new": + "--suites=multiversion_sanity_check_last_continuous_old_new_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_new_new_old": + "--suites=multiversion_sanity_check_last_lts_new_new_old --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_new_old_new": + "--suites=multiversion_sanity_check_last_lts_new_old_new --storageEngine=wiredTiger", + "multiversion_sanity_check_last_lts_old_new_new": + "--suites=multiversion_sanity_check_last_lts_old_new_new --storageEngine=wiredTiger", + }, resmoke_task.combined_suite_to_resmoke_args_map) # Check for tasks included in task_groups variant_amazon = self.conf.get_variant("amazon") diff --git a/buildscripts/tests/resmoke_validation/test_jstest_tags.py b/buildscripts/tests/resmoke_validation/test_jstest_tags.py new file mode 100644 index 00000000000..515fe7038c4 --- /dev/null +++ b/buildscripts/tests/resmoke_validation/test_jstest_tags.py @@ -0,0 +1,65 @@ +# pylint: disable=missing-docstring +import glob +import json +import unittest +from collections import defaultdict +from typing import Optional + +from buildscripts.resmokelib.multiversionconstants import ( + REQUIRES_FCV_TAG_LATEST, + REQUIRES_FCV_TAGS_LESS_THAN_LATEST, +) +from buildscripts.resmokelib.utils import jscomment + + +class JstestTagRule: + def __init__(self, failure_message): + self.failure_message = failure_message + self.failures = defaultdict(list) + + def check(self, file: str, tag: str) -> None: + if self._tag_failed(file, tag): + self.failures[file].append(tag) + + def _tag_failed(self, file: str, tag: str) -> bool: + raise NotImplementedError() + + def make_failure_message(self) -> Optional[str]: + if self.failures: + pretty_failures = json.dumps(self.failures, indent=4) + return f"{self.failure_message}:\n{pretty_failures}" + return None + + +class RequiresFcvTagRule(JstestTagRule): + def __init__(self): + super().__init__( + failure_message="The following tags reference FCV version that is not available") + self.allowed_tags = [*REQUIRES_FCV_TAGS_LESS_THAN_LATEST, REQUIRES_FCV_TAG_LATEST] + + def _tag_failed(self, file: str, tag: str) -> bool: + return tag.startswith("requires_fcv_") and tag not in self.allowed_tags + + +class TestJstestTags(unittest.TestCase): + def test_jstest_tags(self): + globs = ["src/mongo/db/modules/enterprise/jstests/**/*.js", "jstests/**/*.js"] + + tag_rules = [ + RequiresFcvTagRule(), + ] + + for pattern in globs: + for file in glob.glob(pattern, recursive=True): + for tag in jscomment.get_tags(file): + for tag_rule in tag_rules: + tag_rule.check(file, tag) + + full_failure_message = "" + for tag_rule in tag_rules: + failure_message = tag_rule.make_failure_message() + if failure_message: + full_failure_message = f"{full_failure_message}\n{failure_message}" + + if full_failure_message.strip(): + self.fail(full_failure_message.strip()) diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py deleted file mode 100644 index 0cb8554e1ad..00000000000 --- a/buildscripts/tests/test_burn_in_tags.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Unit tests for the burn_in_tags.py script.""" -from collections import defaultdict -import json -import os -import sys -import unittest -from unittest.mock import MagicMock, patch - -from shrub.v2 import ShrubProject - -import buildscripts.ciconfig.evergreen as _evergreen -from buildscripts.burn_in_tests import TaskInfo -from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns -from buildscripts.ciconfig.evergreen import EvergreenProjectConfig - -import buildscripts.burn_in_tags as under_test -from buildscripts.util.teststats import HistoricalTestInformation - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access,too-many-arguments - -EMPTY_PROJECT = { - "buildvariants": [], - "tasks": [], -} -TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "test_burn_in_tags_evergreen.yml") - -NS = "buildscripts.burn_in_tags" - - -def ns(relative_name): # pylint: disable-invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def get_expansions_data(): - return { - "branch_name": "fake_branch", - "build_variant": "enterprise-rhel-80-64-bit-suggested", - "check_evergreen": 2, - "distro_id": "rhel80-small", - "is_patch": "true", - "max_revisions": 25, - "repeat_tests_max": 1000, - "repeat_tests_min": 2, - "repeat_tests_secs": 600, - "revision": "fake_sha", - "project": "fake_project", - "task_id": "task id", - } # yapf: disable - - -def get_evergreen_config() -> EvergreenProjectConfig: - return _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) - - -class TestCreateEvgBuildVariantMap(unittest.TestCase): - def test_create_evg_buildvariant_map(self): - expansions_file_data = { - "build_variant": "variant1", "burn_in_tag_include_build_variants": "variant2 variant3" - } - buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data) - - expected_buildvariant_map = { - "variant2": "variant2-required", "variant3": "variant3-required" - } - self.assertEqual(buildvariant_map, expected_buildvariant_map) - - -class TestGenerateEvgBuildVariants(unittest.TestCase): - def test_generate_evg_buildvariant_one_base_variant(self): - evg_conf_mock = get_evergreen_config() - base_variant = "enterprise-rhel-80-64-bit-inmem" - generated_variant = "enterprise-rhel-80-64-bit-inmem-required" - burn_in_tags_gen_variant = "enterprise-rhel-80-64-bit" - variant = evg_conf_mock.get_variant(base_variant) - - build_variant = under_test._generate_evg_build_variant(variant, generated_variant, - burn_in_tags_gen_variant) - - generated_build_variant = build_variant.as_dict() - self.assertEqual(generated_build_variant["name"], generated_variant) - self.assertNotIn('modules', generated_build_variant) - generated_expansions = generated_build_variant["expansions"] - burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass") - self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant) - self.assertEqual(generated_expansions, variant.expansions) - - -class TestGenerateEvgTasks(unittest.TestCase): - @patch(ns("create_tests_by_task")) - def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock): - evg_conf_mock = get_evergreen_config() - create_tests_by_task_mock.return_value = {} - expansions_file_data = get_expansions_data() - buildvariant_map = { - "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", - "enterprise-rhel-80-64-bit-majority-read-concern-off": - "enterprise-rhel-80-64-bit-majority-read-concern-off-required", - } # yapf: disable - shrub_config = ShrubProject() - evergreen_api = MagicMock() - repo = MagicMock(working_dir=os.getcwd()) - under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, - buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') - - self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT) - - @patch(ns("create_tests_by_task")) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_generate_evg_tasks_one_test_changed(self, get_stats_from_s3_mock, - create_tests_by_task_mock): - evg_conf_mock = get_evergreen_config() - create_tests_by_task_mock.return_value = { - "aggregation_mongos_passthrough": TaskInfo( - display_task_name="aggregation_mongos_passthrough", - suite="aggregation_mongos_passthrough", - resmoke_args="--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger", - tests=["jstests/aggregation/ifnull.js"], - require_multiversion_setup=False, - distro="", - build_variant="enterprise-rhel-80-64-bit-inmem" - ) - } # yapf: disable - expansions_file_data = get_expansions_data() - buildvariant_map = { - "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", - "enterprise-rhel-80-64-bit-majority-read-concern-off": - "enterprise-rhel-80-64-bit-majority-read-concern-off-required", - } # yapf: disable - shrub_config = ShrubProject.empty() - evergreen_api = MagicMock() - repo = MagicMock(working_dir=os.getcwd()) - get_stats_from_s3_mock.return_value = [ - HistoricalTestInformation( - test_name="dir/test2.js", - num_pass=1, - num_fail=0, - avg_duration_pass=10, - ) - ] - under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, - buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') - - generated_config = shrub_config.as_dict() - self.assertEqual(len(generated_config["buildvariants"]), 2) - first_generated_build_variant = generated_config["buildvariants"][0] - self.assertIn(first_generated_build_variant["name"], buildvariant_map.values()) - self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests") - self.assertEqual( - first_generated_build_variant["display_tasks"][0]["execution_tasks"][0], - f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}") - - -EXPANSIONS_FILE_DATA = { - "build_variant": "enterprise-rhel-80-64-bit", - "revision": "badf00d000000000000000000000000000000000", "max_revisions": "1000", - "branch_name": "mongodb-mongo-master", "is_patch": "false", "distro_id": "rhel62-small", - "repeat_tests_min": "2", "repeat_tests_max": "1000", "repeat_tests_secs": "600", "project": - "mongodb-mongo-master", "task_id": "task id" -} - -CREATE_EVG_BUILD_VARIANT_MAP = { - 'enterprise-rhel-80-64-bit-majority-read-concern-off': - 'enterprise-rhel-80-64-bit-majority-read-concern-off-required', - 'enterprise-rhel-80-64-bit-inmem': - 'enterprise-rhel-80-64-bit-inmem-required' -} - -CREATE_TEST_MEMBERSHIP_MAP = { - "jstests/aggregation/accumulators/accumulator_js.js": [ - "aggregation", "aggregation_auth", "aggregation_disabled_optimization", "aggregation_ese", - "aggregation_ese_gcm", "aggregation_facet_unwind_passthrough", - "aggregation_mongos_passthrough", "aggregation_one_shard_sharded_collections", - "aggregation_read_concern_majority_passthrough", "aggregation_secondary_reads", - "aggregation_sharded_collections_passthrough" - ], "jstests/core/create_collection.js": [ - "core", "core_auth", "core_ese", "core_ese_gcm", "core_minimum_batch_size", "core_op_query", - "cwrwc_passthrough", "cwrwc_rc_majority_passthrough", "cwrwc_wc_majority_passthrough", - "logical_session_cache_replication_100ms_refresh_jscore_passthrough", - "logical_session_cache_replication_10sec_refresh_jscore_passthrough", - "logical_session_cache_replication_1sec_refresh_jscore_passthrough", - "logical_session_cache_replication_default_refresh_jscore_passthrough", - "logical_session_cache_standalone_100ms_refresh_jscore_passthrough", - "logical_session_cache_standalone_10sec_refresh_jscore_passthrough", - "logical_session_cache_standalone_1sec_refresh_jscore_passthrough", - "logical_session_cache_standalone_default_refresh_jscore_passthrough", - "read_concern_linearizable_passthrough", "read_concern_majority_passthrough", - "causally_consistent_read_concern_snapshot_passthrough", - "replica_sets_initsync_jscore_passthrough", "replica_sets_fcbis_jscore_passthrough", - "replica_sets_initsync_static_jscore_passthrough", "replica_sets_jscore_passthrough", - "replica_sets_kill_primary_jscore_passthrough", - "replica_sets_kill_secondaries_jscore_passthrough", - "replica_sets_reconfig_jscore_passthrough", - "replica_sets_terminate_primary_jscore_passthrough", "retryable_writes_jscore_passthrough", - "retryable_writes_jscore_stepdown_passthrough", "secondary_reads_passthrough", - "session_jscore_passthrough", "write_concern_majority_passthrough" - ] -} - - -class TestAcceptance(unittest.TestCase): - @patch(ns("write_file_to_dir")) - @patch(ns("_create_evg_build_variant_map")) - @patch(ns("EvergreenFileChangeDetector")) - def test_no_tests_run_if_none_changed(self, find_changed_tests_mock, - create_evg_build_variant_map_mock, write_to_file_mock): - """ - Given a git repository with no changes, - When burn_in_tags is run, - Then no tests are discovered to run. - """ - repos = [MagicMock(working_dir=os.getcwd())] - evg_conf_mock = MagicMock() - find_changed_tests_mock.return_value.find_changed_tests.return_value = {} - - create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP - - under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf_mock, MagicMock(), repos, - 'install_dir/bin') - - write_to_file_mock.assert_called_once() - shrub_config = write_to_file_mock.call_args[0][2] - self.assertEqual(EMPTY_PROJECT, json.loads(shrub_config)) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("write_file_to_dir")) - @patch(ns("_create_evg_build_variant_map")) - @patch(ns("EvergreenFileChangeDetector")) - @patch(burn_in_tests_ns("create_test_membership_map")) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_tests_generated_if_a_file_changed( - self, get_stats_from_s3_mock, create_test_membership_map_mock, find_changed_tests_mock, - create_evg_build_variant_map_mock, write_to_file_mock): - """ - Given a git repository with changes, - When burn_in_tags is run, - Then some tags are discovered to run. - """ - create_test_membership_map_mock.return_value = defaultdict(list, CREATE_TEST_MEMBERSHIP_MAP) - - repos = [MagicMock(working_dir=os.getcwd())] - evg_conf = get_evergreen_config() - create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP - find_changed_tests_mock.return_value.find_changed_tests.return_value = { - 'jstests/slow1/large_role_chain.js', - 'jstests/aggregation/accumulators/accumulator_js.js' - } - get_stats_from_s3_mock.return_value = [] - - under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin') - - write_to_file_mock.assert_called_once() - written_config = write_to_file_mock.call_args[0][2] - written_config_map = json.loads(written_config) - - n_tasks = len(written_config_map["tasks"]) - # Ensure we are generating at least one task for the test. - self.assertGreaterEqual(n_tasks, 1) - - written_build_variants = written_config_map["buildvariants"] - written_build_variants_name = [variant['name'] for variant in written_build_variants] - self.assertEqual( - set(CREATE_EVG_BUILD_VARIANT_MAP.values()), set(written_build_variants_name)) - - tasks = written_config_map["tasks"] - self.assertGreaterEqual(len(tasks), len(CREATE_EVG_BUILD_VARIANT_MAP)) - - self.assertTrue( - all( - len(display_tasks) == 1 for display_tasks in - [build_variant["display_tasks"] for build_variant in written_build_variants])) diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml deleted file mode 100644 index 685f79e9e0e..00000000000 --- a/buildscripts/tests/test_burn_in_tags_evergreen.yml +++ /dev/null @@ -1,134 +0,0 @@ -functions: - "fetch source": - - command: git.get_project - params: - directory: src - - command: shell.exec - params: - working_dir: src - script: | - echo "this is a 2nd command in the function!" - ls - -tasks: -- name: compile - depends_on: [] - commands: - - func: "fetch source" -- name: burn_in_tags_gen - depends_on: [] - commands: - - func: "fake command" -- name: compile_all_run_unittests_TG - depends_on: [] - commands: - - func: "fake command" -- name: clang_tidy_TG - depends_on: [] - commands: - - func: "fake command" -- name: stitch_support_lib_build_and_archive - depends_on: [] - commands: - - func: "fake command" -- name: lint_pylinters - depends_on: [] - commands: - - func: "fake command" -- name: lint_clang_format - depends_on: [] - commands: - - func: "fake command" -- name: burn_in_tests_gen - depends_on: [] - commands: - - func: "fake command" -- name: aggregation_multiversion_fuzzer_gen - depends_on: [] - commands: - - func: "generate resmoke tasks" -- name: aggregation_expression_multiversion_fuzzer_gen - depends_on: [] - commands: - - func: "generate resmoke tasks" -- name: aggregation - depends_on: - - name: compile - commands: - - func: run tests - vars: - resmoke_args: --suites=aggregation --storageEngine=wiredTiger - -buildvariants: -- name: enterprise-rhel-80-64-bit - display_name: "! Enterprise RHEL 8.0" - expansions: - multiversion_platform: rhel80 - burn_in_tag_include_build_variants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: lint_pylinters - - name: burn_in_tests_gen - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation - - name: burn_in_tags_gen -- name: buildvariant-without-burn-in-tag-buildvariants - display_name: "Buildvariant without burn in tag buildvariants expansion" - expansions: - multiversion_platform: rhel80 - tasks: - - name: burn_in_tags_gen -- name: enterprise-rhel-80-64-bit-majority-read-concern-off - display_name: "Enterprise RHEL 8.0 (majority read concern off)" - run_on: - - rhel80-small - expansions: &enterprise-rhel-80-64-bit-majority-read-concern-off-expansions - multiversion_edition: enterprise - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation -- name: enterprise-rhel-80-64-bit-inmem - display_name: Enterprise RHEL 8.0 (inMemory) - run_on: - - rhel80-small - expansions: &enterprise-rhel-80-64-bit-inmem-expansions - test_flags: >- - --majorityReadConcern=off - --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime - compile_flags: >- - -j$(grep -c ^processor /proc/cpuinfo) - --ssl - --release - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - MONGO_DISTMOD=rhel80 - multiversion_platform: rhel80 - multiversion_edition: enterprise - scons_cache_scope: shared - tooltags: "ssl sasl gssapi" - large_distro_name: rhel80-large - tasks: - - name: compile -- name: enterprise-rhel-80-64-bit-inmem - display_name: Enterprise RHEL 8.0 (inMemory) - expansions: - additional_targets: archive-mongocryptd archive-mongocryptd-debug - compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) - --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars - large_distro_name: rhel80-large - multiversion_edition: enterprise - multiversion_platform: rhel80 - scons_cache_scope: shared - test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling - run_on: - - rhel80-small - tasks: - - name: compile_all_run_unittests_TG - distros: - - rhel80-large - - name: aggregation_multiversion_fuzzer_gen - - name: aggregation diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 369131eff60..a5b9ba625d3 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,18 +4,19 @@ from __future__ import absolute_import import collections import datetime -from io import StringIO import os -import sys import subprocess +import sys import unittest +from io import StringIO -from mock import Mock, patch, MagicMock import yaml +from mock import MagicMock, Mock, patch import buildscripts.burn_in_tests as under_test -from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask import buildscripts.resmokelib.parser as _parser +from buildscripts.ciconfig.evergreen import parse_evergreen_file + _parser.set_run_options() # pylint: disable=missing-docstring,protected-access,too-many-lines,no-self-use @@ -23,10 +24,16 @@ _parser.set_run_options() def create_tests_by_task_mock(n_tasks, n_tests): return { - f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ - f"jstests/tests_{j}" for j in range(n_tests) - ], require_multiversion_setup=False, distro=f"distro_{i}", suite=f"suite_{i}", - build_variant="dummy_variant") + f"task_{i}_gen": under_test.TaskToBurnInInfo( + display_task_name=f"task_{i}", + suites=[ + under_test.SuiteToBurnInInfo( + name=f"suite_{i}", + resmoke_args="", + tests=[f"jstests/tests_{j}" for j in range(n_tests)], + ), + ], + ) for i in range(n_tasks) } @@ -294,9 +301,8 @@ def create_variant_task_mock(task_name, suite_name, distro="distro"): variant_task = MagicMock() variant_task.name = task_name variant_task.generated_task_name = task_name - variant_task.get_suite_name.return_value = suite_name - variant_task.resmoke_args = f"--suites={suite_name}" - variant_task.require_multiversion_setup.return_value = False + variant_task.get_suite_names.return_value = [suite_name] + variant_task.combined_suite_to_resmoke_args_map = {suite_name: f"--suites={suite_name}"} variant_task.run_on = [distro] return variant_task @@ -304,89 +310,9 @@ def create_variant_task_mock(task_name, suite_name, distro="distro"): class TestTaskInfo(unittest.TestCase): def test_non_generated_task(self): suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" evg_conf_mock = MagicMock() evg_conf_mock.get_task.return_value.is_generate_resmoke_task = False - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_no_large_on_task(self): - suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {}} - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_no_large_on_build_variant(self): - suite_name = "suite_1" - distro_name = "distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {"use_large_distro": True}} - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) - test_list = [f"test{i}.js" for i in range(3)] - tests_by_suite = { - suite_name: test_list, - "suite 2": [f"test{i}.js" for i in range(1)], - "suite 3": [f"test{i}.js" for i in range(2)], - } - - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) - - self.assertIn(suite_name, task_info.resmoke_args) - for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(distro_name, task_info.distro) - - def test_generated_task_large_distro(self): - suite_name = "suite_1" - distro_name = "distro_1" - large_distro_name = "large_distro_1" - variant = "build_variant" - evg_conf_mock = MagicMock() - task_def_mock = evg_conf_mock.get_task.return_value - task_def_mock.is_generate_resmoke_task = True - task_def_mock.generate_resmoke_tasks_command = {"vars": {"use_large_distro": True}} - evg_conf_mock.get_variant.return_value.raw = { - "expansions": { - "large_distro_name": large_distro_name - } - } # yapf: disable - task_mock = create_variant_task_mock("task 1", suite_name, distro_name) + task_mock = create_variant_task_mock("task 1", suite_name) test_list = [f"test{i}.js" for i in range(3)] tests_by_suite = { suite_name: test_list, @@ -394,13 +320,11 @@ class TestTaskInfo(unittest.TestCase): "suite 3": [f"test{i}.js" for i in range(2)], } - task_info = under_test.TaskInfo.from_task(task_mock, tests_by_suite, evg_conf_mock, variant) + task_info = under_test.TaskToBurnInInfo.from_task(task_mock, tests_by_suite) - self.assertIn(suite_name, task_info.resmoke_args) + self.assertIn(suite_name, task_info.suites[0].name) for test in test_list: - self.assertIn(test, task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual(large_distro_name, task_info.distro) + self.assertIn(test, task_info.suites[0].tests) class TestCreateTaskList(unittest.TestCase): @@ -444,11 +368,9 @@ class TestCreateTaskList(unittest.TestCase): self.assertIn("task 1", task_list) task_info = task_list["task 1"] - self.assertIn("suite_1", task_info.resmoke_args) + self.assertIn("suite_1", task_info.suites[0].resmoke_args) for i in range(3): - self.assertIn(f"test{i}.js", task_info.tests) - self.assertFalse(task_info.require_multiversion_setup) - self.assertEqual("distro 1", task_info.distro) + self.assertIn(f"test{i}.js", task_info.suites[0].tests) def test_create_task_list_with_excludes(self): variant = "variant name" @@ -573,4 +495,4 @@ class TestYamlBurnInExecutor(unittest.TestCase): yaml_raw = stdout.getvalue() results = yaml.safe_load(yaml_raw) self.assertEqual(n_tasks, len(results["discovered_tasks"])) - self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) + self.assertEqual(n_tests, len(results["discovered_tasks"][0]["suites"][0]["test_list"])) diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py deleted file mode 100644 index 3ea78039930..00000000000 --- a/buildscripts/tests/test_evergreen_burn_in_tests.py +++ /dev/null @@ -1,399 +0,0 @@ -"""Unit tests for buildscripts/burn_in_tests.py.""" - -from __future__ import absolute_import - -import json -import os -import sys -import unittest -from datetime import datetime, timedelta -from math import ceil - -import requests -from mock import patch, MagicMock -from shrub.v2 import BuildVariant, ShrubProject -from evergreen.api import EvergreenApi - -import buildscripts.evergreen_burn_in_tests as under_test -from buildscripts.ciconfig.evergreen import parse_evergreen_file -import buildscripts.resmokelib.parser as _parser -import buildscripts.resmokelib.config as _config -import buildscripts.util.teststats as teststats_utils -_parser.set_run_options() - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access - -NS = "buildscripts.evergreen_burn_in_tests" - - -def ns(relative_name): # pylint: disable=invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def mock_a_file(filename): - change = MagicMock(a_path=filename) - return change - - -def mock_git_diff(change_list): - diff = MagicMock() - diff.iter_change_type.return_value = change_list - return diff - - -def mock_changed_git_files(add_files): - repo = MagicMock() - repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) - repo.working_dir = "." - return repo - - -def get_evergreen_config(config_file_path): - evergreen_home = os.path.expanduser(os.path.join("~", "evergreen")) - if os.path.exists(evergreen_home): - return parse_evergreen_file(config_file_path, evergreen_home) - return parse_evergreen_file(config_file_path) - - -class TestAcceptance(unittest.TestCase): - def tearDown(self): - _parser.set_run_options() - - @patch(ns("write_file")) - def test_no_tests_run_if_none_changed(self, write_json_mock): - """ - Given a git repository with no changes, - When burn_in_tests is run, - Then no tests are discovered to run. - """ - variant = "build_variant" - repos = [mock_changed_git_files([])] - repeat_config = under_test.RepeatConfig() - gen_config = under_test.GenerateConfig( - variant, - "project", - ) # yapf: disable - mock_evg_conf = MagicMock() - mock_evg_conf.get_task_names_by_tag.return_value = set() - mock_evg_api = MagicMock() - - under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, - mock_evg_conf, repos, "testfile.json", "install-dir/bin") - - write_json_mock.assert_called_once() - written_config = json.loads(write_json_mock.call_args[0][1]) - display_task = written_config["buildvariants"][0]["display_tasks"][0] - self.assertEqual(1, len(display_task["execution_tasks"])) - self.assertEqual(under_test.BURN_IN_TESTS_GEN_TASK, display_task["execution_tasks"][0]) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("write_file")) - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_tests_generated_if_a_file_changed(self, get_stats_from_s3_mock, write_json_mock): - """ - Given a git repository with changes, - When burn_in_tests is run, - Then tests are discovered to run. - """ - # Note: this test is using actual tests and suites. So changes to those suites could - # introduce failures and require this test to be updated. - # You can see the test file it is using below. This test is used in the 'auth' and - # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. - variant = "enterprise-rhel-80-64-bit-inmem" - repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] - repeat_config = under_test.RepeatConfig() - gen_config = under_test.GenerateConfig( - variant, - "project", - ) # yapf: disable - mock_evg_conf = get_evergreen_config("etc/evergreen.yml") - mock_evg_api = MagicMock() - get_stats_from_s3_mock.return_value = [] - - under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, - mock_evg_conf, repos, "testfile.json", 'install-dir/bin') - - write_json_mock.assert_called_once() - written_config = json.loads(write_json_mock.call_args[0][1]) - n_tasks = len(written_config["tasks"]) - # Ensure we are generating at least one task for the test. - self.assertGreaterEqual(n_tasks, 1) - - written_build_variant = written_config["buildvariants"][0] - self.assertEqual(variant, written_build_variant["name"]) - self.assertEqual(n_tasks, len(written_build_variant["tasks"])) - - display_task = written_build_variant["display_tasks"][0] - # The display task should contain all the generated tasks as well as 1 extra task for - # the burn_in_test_gen task. - self.assertEqual(n_tasks + 1, len(display_task["execution_tasks"])) - - -class TestGenerateConfig(unittest.TestCase): - def test_run_build_variant_with_no_run_build_variant(self): - gen_config = under_test.GenerateConfig("build_variant", "project") - - self.assertEqual(gen_config.build_variant, gen_config.run_build_variant) - - def test_run_build_variant_with_run_build_variant(self): - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - - self.assertNotEqual(gen_config.build_variant, gen_config.run_build_variant) - self.assertEqual(gen_config.run_build_variant, "run_build_variant") - - def test_validate_non_existing_build_variant(self): - evg_conf_mock = MagicMock() - evg_conf_mock.get_variant.return_value = None - - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - - with self.assertRaises(ValueError): - gen_config.validate(evg_conf_mock) - - def test_validate_existing_build_variant(self): - evg_conf_mock = MagicMock() - - gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") - gen_config.validate(evg_conf_mock) - - def test_validate_non_existing_run_build_variant(self): - evg_conf_mock = MagicMock() - - gen_config = under_test.GenerateConfig("build_variant", "project") - gen_config.validate(evg_conf_mock) - - -class TestParseAvgTestRuntime(unittest.TestCase): - def test__parse_avg_test_runtime(self): - task_avg_test_runtime_stats = [ - teststats_utils.TestRuntime(test_name="dir/test1.js", runtime=30.2), - teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1) - ] - result = under_test._parse_avg_test_runtime("dir/test2.js", task_avg_test_runtime_stats) - self.assertEqual(result, 455.1) - - -class TestCalculateTimeout(unittest.TestCase): - def test__calculate_timeout(self): - avg_test_runtime = 455.1 - expected_result = ceil(avg_test_runtime * under_test.AVG_TEST_TIME_MULTIPLIER) - self.assertEqual(expected_result, under_test._calculate_timeout(avg_test_runtime)) - - def test__calculate_timeout_avg_is_less_than_min(self): - avg_test_runtime = 10 - self.assertEqual(under_test.MIN_AVG_TEST_TIME_SEC, - under_test._calculate_timeout(avg_test_runtime)) - - -class TestCalculateExecTimeout(unittest.TestCase): - def test__calculate_exec_timeout(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - avg_test_runtime = 455.1 - - exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) - - self.assertEqual(1771, exec_timeout) - - def test_average_timeout_greater_than_execution_time(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600, repeat_tests_min=2) - avg_test_runtime = 750 - - exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) - - # The timeout needs to be greater than the number of the test * the minimum number of runs. - minimum_expected_timeout = avg_test_runtime * repeat_config.repeat_tests_min - - self.assertGreater(exec_timeout, minimum_expected_timeout) - - -class TestGenerateTimeouts(unittest.TestCase): - def test__generate_timeouts(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)] - test_name = "dir/test2.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertEqual(timeout_info.exec_timeout, 1771) - self.assertEqual(timeout_info.timeout, 1366) - - def test__generate_timeouts_no_results(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [] - test_name = "dir/new_test.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertIsNone(timeout_info.cmd) - - def test__generate_timeouts_avg_runtime_is_zero(self): - repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) - runtime_stats = [ - teststats_utils.TestRuntime(test_name="dir/test_with_zero_runtime.js", runtime=0) - ] - test_name = "dir/test_with_zero_runtime.js" - - task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) - timeout_info = task_generator.generate_timeouts(test_name) - - self.assertIsNone(timeout_info.cmd) - - -class TestGetTaskRuntimeHistory(unittest.TestCase): - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_get_task_runtime_history(self, get_stats_from_s3_mock): - test_stats = [ - teststats_utils.HistoricalTestInformation( - test_name="dir/test2.js", - num_pass=1, - num_fail=0, - avg_duration_pass=10.1, - ) - ] - get_stats_from_s3_mock.return_value = test_stats - mock_gen_config = MagicMock(project="project1", build_variant="variant1") - - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) - result = executor.get_task_runtime_history("task1") - - self.assertEqual(result, [("dir/test2.js", 10.1)]) - - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_get_task_runtime_history_when_s3_has_no_data(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] - mock_gen_config = MagicMock(project="project1", build_variant="variant1") - - executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock()) - result = executor.get_task_runtime_history("task1") - - self.assertEqual(result, []) - - -TESTS_BY_TASK = { - "task1": { - "resmoke_args": "--suites=suite1", - "tests": ["jstests/test1.js", "jstests/test2.js"]}, - "task2": { - "resmoke_args": "--suites=suite1", - "tests": ["jstests/test1.js", "jstests/test3.js"]}, - "task3": { - "resmoke_args": "--suites=suite3", - "tests": ["jstests/test4.js", "jstests/test5.js"]}, - "task4": { - "resmoke_args": "--suites=suite4", "tests": []}, -} # yapf: disable - - -def create_tests_by_task_mock(n_tasks, n_tests): - return { - f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ - f"jstests/tests_{j}" for j in range(n_tests) - ], require_multiversion_setup=False, distro=f"distro_{i}", build_variant="variant", - suite=f"suite_{i}") - for i in range(n_tasks) - } - - -class TestCreateGenerateTasksConfig(unittest.TestCase): - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - def test_no_tasks_given(self): - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant") - repeat_config = MagicMock() - mock_evg_api = MagicMock() - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) - executor.generate_tasks_for_variant({}, build_variant) - - evg_config_dict = build_variant.as_dict() - self.assertEqual(0, len(evg_config_dict["tasks"])) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_one_task_one_test(self, get_stats_from_s3_mock): - n_tasks = 1 - n_tests = 1 - resmoke_options = "options for resmoke" - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant", distro=None) - repeat_config = MagicMock() - repeat_config.generate_resmoke_options.return_value = resmoke_options - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - get_stats_from_s3_mock.return_value = [] - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) - executor.generate_tasks_for_variant(tests_by_task, build_variant) - - shrub_config = ShrubProject.empty().add_build_variant(build_variant) - evg_config_dict = shrub_config.as_dict() - tasks = evg_config_dict["tasks"] - self.assertEqual(n_tasks * n_tests, len(tasks)) - cmd = tasks[0]["commands"] - self.assertIn(resmoke_options, cmd[2]["vars"]["resmoke_args"]) - self.assertEqual("suite_0", cmd[2]["vars"]["suite"]) - self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"]) - - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("HistoricTaskData.get_stats_from_s3")) - def test_n_task_m_test(self, get_stats_from_s3_mock): - n_tasks = 3 - n_tests = 5 - build_variant = BuildVariant("build variant") - gen_config = MagicMock(run_build_variant="variant", distro=None) - repeat_config = MagicMock() - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - get_stats_from_s3_mock.return_value = [] - - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config) - executor.generate_tasks_for_variant(tests_by_task, build_variant) - - evg_config_dict = build_variant.as_dict() - self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"])) - - -class TestCreateGenerateTasksFile(unittest.TestCase): - @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch(ns("sys.exit")) - @patch(ns("validate_task_generation_limit")) - def test_cap_on_task_generate(self, validate_mock, exit_mock): - gen_config = MagicMock(require_multiversion_setup=False) - repeat_config = MagicMock() - tests_by_task = MagicMock() - - validate_mock.return_value = False - - exit_mock.side_effect = ValueError("exiting") - with self.assertRaises(ValueError): - executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, "gen_file.json") - executor.execute(tests_by_task) - - exit_mock.assert_called_once() - - -class TestFindChangedTests(unittest.TestCase): - def test_manual_tests_should_be_specifiable_via_env_vars(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_repo = MagicMock() - mock_env = { - "BURN_IN_TESTS": "jstests/auth/auth1.js,jstests/core/where1.js", - } - change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) - - test_set = change_detector.find_changed_tests([mock_repo]) - - self.assertEqual(2, len(test_set)) - self.assertIn("jstests/auth/auth1.js", test_set) - self.assertIn("jstests/core/where1.js", test_set) - - def test_empty_env_should_not_add_extra_tests(self): - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_repo = MagicMock() - mock_env = {} - change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) - - test_set = change_detector.find_changed_tests([mock_repo]) - - self.assertEqual(set(), test_set) diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py deleted file mode 100644 index 7ece404ab36..00000000000 --- a/buildscripts/tests/test_evergreen_gen_build_variant.py +++ /dev/null @@ -1,382 +0,0 @@ -"""Unit tests for the generate_resmoke_suite script.""" -import unittest - -from mock import MagicMock - -from buildscripts import evergreen_gen_build_variant as under_test -from buildscripts.ciconfig.evergreen import Variant, Task - -# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access -# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter - - -def build_mock_build_variant(expansions=None, task_list=None): - task_spec_list = [{"name": task.name} for task in task_list] if task_list else [] - config = { - "tasks": task_spec_list, - } - if expansions: - config["expansions"] = expansions - - if task_list is None: - task_list = [] - task_map = {task.name: task for task in task_list} - - return Variant(config, task_map, {}) - - -def build_mock_task(name, run_vars=None, depends_on=None): - config = { - "name": - name, "commands": [ - {"func": "do setup"}, - { - "func": "generate resmoke tasks", - "vars": run_vars if run_vars else {}, - }, - ] - } - - if depends_on is not None: - config["depends_on"] = depends_on - return Task(config) - - -def build_mock_project_config(variant=None, task_defs=None): - mock_project = MagicMock() - if variant: - mock_project.get_variant.return_value = variant - - if task_defs: - mock_project.get_task.side_effect = task_defs - - return mock_project - - -def build_mock_expansions(): - mock_expansions = MagicMock() - mock_expansions.config_location.return_value = "/path/to/config" - mock_expansions.get_max_sub_suites.return_value = 998 - mock_expansions.task_name = "generating_task" - return mock_expansions - - -def build_mock_evg_api(build_task_list): - mock_evg_api = MagicMock() - mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list - return mock_evg_api - - -def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None): - if build_expansions is None: - build_expansions = {} - if task_def_list is None: - task_def_list = [] - if build_task_list is None: - build_task_list = [] - - mock_build_variant = build_mock_build_variant(build_expansions, task_def_list) - mock_project = build_mock_project_config(mock_build_variant, task_def_list) - mock_evg_expansions = build_mock_expansions() - mock_evg_api = build_mock_evg_api(build_task_list) - - return under_test.GenerateBuildVariantOrchestrator( - gen_task_validation=MagicMock(), - gen_task_options=MagicMock(), - evg_project_config=mock_project, - evg_expansions=mock_evg_expansions, - evg_api=mock_evg_api, - ) - - -class TestEvgExpansions(unittest.TestCase): - def test_get_max_sub_suites_should_use_patch_value_in_patches(self): - evg_expansions = under_test.EvgExpansions( - is_patch=True, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), evg_expansions.max_sub_suites) - - def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self): - evg_expansions = under_test.EvgExpansions( - is_patch=False, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), - evg_expansions.mainline_max_sub_suites) - - def test_get_max_sub_suites_should_use_mainline_value_if_patch_status_unknown(self): - evg_expansions = under_test.EvgExpansions( - is_patch=None, - max_sub_suites=5, - mainline_max_sub_suites=1, - build_id="build_id", - build_variant="build_variant", - project="project", - revision="revision", - task_name="task_name", - task_id="task_id", - ) - - self.assertEqual(evg_expansions.get_max_sub_suites(), - evg_expansions.mainline_max_sub_suites) - - -class TestTranslateRunVar(unittest.TestCase): - def test_normal_value_should_be_returned(self): - run_var = "some value" - mock_build_variant = build_mock_build_variant() - self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_should_be_returned_from_build_variant(self): - run_var = "${my_expansion}" - value = "my value" - mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) - self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_not_found_should_return_none(self): - run_var = "${my_expansion}" - mock_build_variant = build_mock_build_variant(expansions={}) - self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_not_found_should_return_default(self): - run_var = "${my_expansion|default}" - mock_build_variant = build_mock_build_variant(expansions={}) - self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant)) - - def test_expansion_should_be_returned_from_build_variant_even_with_default(self): - run_var = "${my_expansion|default}" - value = "my value" - mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) - self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) - - -class TestTaskDefToSplitParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "resmoke_args": "run tests", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") - - self.assertEqual("build_variant", split_param.build_variant) - self.assertEqual("my_task", split_param.task_name) - self.assertEqual("my_task", split_param.suite_name) - self.assertEqual("my_task", split_param.filename) - - def test_params_should_allow_suite_to_be_overridden(self): - run_vars = { - "resmoke_args": "run tests", - "suite": "the suite", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") - - self.assertEqual("build_variant", split_param.build_variant) - self.assertEqual("my_task", split_param.task_name) - self.assertEqual("the suite", split_param.suite_name) - self.assertEqual("the suite", split_param.filename) - - -class TestDetermineTaskDependencies(unittest.TestCase): - def test_running_task_should_not_be_included_in_depends_on(self): - run_vars = { - "resmoke_args": "run tests", - } - depends_on = [ - {"name": "compile"}, - {"name": "build_variant_gen"}, - ] - mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def) - - self.assertIn("compile", dependencies) - self.assertNotIn("generating_task", dependencies) - - -class TestTaskDefToGenParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "resmoke_args": "run tests", - } - mock_task_def = build_mock_task("my_task", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - - gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") - - self.assertFalse(gen_params.require_multiversion_setup) - self.assertEqual("run tests", gen_params.resmoke_args) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - gen_params.config_location) - self.assertIsNone(gen_params.large_distro_name) - self.assertFalse(gen_params.use_large_distro) - - def test_params_should_be_overwritable(self): - run_vars = { - "resmoke_args": "run tests", - "use_large_distro": "true", - } - mock_task_def = build_mock_task("my_task", run_vars) - build_expansions = {"large_distro_name": "my large distro"} - mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, - task_def_list=[mock_task_def]) - gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") - - self.assertFalse(gen_params.require_multiversion_setup) - self.assertEqual("run tests", gen_params.resmoke_args) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - gen_params.config_location) - self.assertEqual("my large distro", gen_params.large_distro_name) - self.assertTrue(gen_params.use_large_distro) - - -class TestTaskDefToFuzzerParams(unittest.TestCase): - def test_params_should_be_generated(self): - run_vars = { - "num_files": "5", - "num_tasks": "3", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual("my_fuzzer", fuzzer_params.task_name) - self.assertEqual(5, fuzzer_params.num_files) - self.assertEqual(3, fuzzer_params.num_tasks) - self.assertEqual("jstestfuzz", fuzzer_params.npm_command) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - fuzzer_params.config_location) - self.assertIsNone(fuzzer_params.large_distro_name) - self.assertFalse(fuzzer_params.use_large_distro) - - def test_num_tasks_respects_max_sub_suites(self): - run_vars = { - "num_files": "5", - "num_tasks": "3", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) - mock_orchestrator.evg_expansions.get_max_sub_suites.return_value = 1 - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual(5, fuzzer_params.num_files) - self.assertEqual(1, fuzzer_params.num_tasks) - - def test_params_should_be_overwritable(self): - run_vars = { - "num_files": "${file_count|8}", - "num_tasks": "3", - "use_large_distro": "true", - "npm_command": "aggfuzzer", - } - mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) - build_expansions = {"large_distro_name": "my large distro"} - mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, - task_def_list=[mock_task_def]) - - fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") - - self.assertEqual("my_fuzzer", fuzzer_params.task_name) - self.assertEqual(8, fuzzer_params.num_files) - self.assertEqual(3, fuzzer_params.num_tasks) - self.assertEqual("aggfuzzer", fuzzer_params.npm_command) - self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, - fuzzer_params.config_location) - self.assertEqual("my large distro", fuzzer_params.large_distro_name) - self.assertTrue(fuzzer_params.use_large_distro) - - -class TestGenerateBuildVariant(unittest.TestCase): - def test_a_whole_build_variant(self): - gen_run_vars = { - "resmoke_args": "run tests", - } - mv_gen_run_vars = { - "resmoke_args": "run tests", - "suite": "some suite", - } - fuzz_run_vars = { - "num_files": "5", - "num_tasks": "3", - "is_jstestfuzz": "true", - } - mv_fuzz_run_vars = { - "num_files": "5", - "num_tasks": "3", - "is_jstestfuzz": "true", - "suite": "aggfuzzer", - } - mock_task_defs = [ - build_mock_task("my_gen_task", gen_run_vars), - build_mock_task("my_fuzzer_task", fuzz_run_vars), - build_mock_task("my_mv_fuzzer_task", mv_fuzz_run_vars), - build_mock_task("my_mv_gen_task", mv_gen_run_vars), - ] - mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs) - builder = MagicMock() - - builder = mock_orchestrator.generate_build_variant(builder, "build variant") - - self.assertEqual(builder.generate_suite.call_count, 2) - self.assertEqual(builder.generate_fuzzer.call_count, 2) - - -class TestAdjustTaskPriority(unittest.TestCase): - def test_task_is_updates(self): - starting_priority = 42 - task_id = "task 314" - mock_task = MagicMock(task_id=task_id, priority=starting_priority) - mock_orchestrator = build_mock_orchestrator() - - mock_orchestrator.adjust_task_priority(mock_task) - - mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, - priority=starting_priority + 1) - - def test_task_should_only_reach_99(self): - starting_priority = 99 - task_id = "task 314" - mock_task = MagicMock(task_id=task_id, priority=starting_priority) - mock_orchestrator = build_mock_orchestrator() - - mock_orchestrator.adjust_task_priority(mock_task) - - mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, - priority=starting_priority) - - -class TestAdjustGenTasksPriority(unittest.TestCase): - def test_gen_tasks_in_task_list_are_adjusted(self): - gen_tasks = {"task_3", "task_8", "task_13"} - n_build_tasks = 25 - mock_task_list = [ - MagicMock(build_variant='dummy_variant', display_name=f"task_{i}", priority=0) - for i in range(n_build_tasks) - ] - mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list) - - n_tasks_adjusted = mock_orchestrator.adjust_gen_tasks_priority(gen_tasks) - - self.assertEqual(len(gen_tasks), n_tasks_adjusted) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 819d963ac48..c59952a8312 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -11,7 +11,7 @@ from evergreen import EvergreenApi # pylint: disable=wrong-import-position import buildscripts.ciconfig.evergreen as _evergreen -from buildscripts.burn_in_tests import TaskInfo +from buildscripts.burn_in_tests import TaskToBurnInInfo, SuiteToBurnInInfo from buildscripts.patch_builds.selected_tests.selected_tests_client import SelectedTestsClient, \ TestMappingsResponse, TestMapping, TestFileInstance, TaskMappingsResponse, TaskMapInstance, \ TaskMapping @@ -135,7 +135,7 @@ class TestAcceptance(unittest.TestCase): # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file - self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate) + self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list @@ -147,7 +147,7 @@ class TestAcceptance(unittest.TestCase): # jstests/auth/auth1.js belongs to two suites, auth and auth_audit, rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @@ -168,7 +168,7 @@ class TestAcceptance(unittest.TestCase): mock_task_mapping = TaskMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, - tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)]) + tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-8", flip_count=5)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse( task_mappings=[mock_task_mapping]) @@ -194,7 +194,7 @@ class TestAcceptance(unittest.TestCase): build_variants_with_generated_tasks = generated_evg_config["buildvariants"] rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5) @@ -274,27 +274,29 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): exclude_task_mock.return_value = False tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core_auth", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core_auth", - build_variant="dummy_variant", ), "auth_gen": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 2", - tests=["jstests/auth/auth3.js"], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="auth", - build_variant="dummy_variant", + suites=[ + SuiteToBurnInInfo( + name="auth", + resmoke_args="", + tests=["jstests/auth/auth3.js"], + ), + ], ), } @@ -319,17 +321,18 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): exclude_task_mock.return_value = True tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core", - build_variant="dummy_variant", ), } @@ -344,17 +347,18 @@ class TestGetTaskConfigsForTestMappings(unittest.TestCase): find_task_mock.return_value = None tests_by_task = { "jsCore_auth": - TaskInfo( + TaskToBurnInInfo( display_task_name="task 1", - tests=[ - "jstests/core/currentop_waiting_for_latch.js", - "jstests/core/latch_analyzer.js", + suites=[ + SuiteToBurnInInfo( + name="core", + resmoke_args="", + tests=[ + "jstests/core/currentop_waiting_for_latch.js", + "jstests/core/latch_analyzer.js", + ], + ), ], - resmoke_args="", - require_multiversion_setup=False, - distro="", - suite="core", - build_variant="dummy_variant", ), } diff --git a/buildscripts/tests/test_validate_commit_message.py b/buildscripts/tests/test_validate_commit_message.py index db341a5add1..1bf0039188d 100644 --- a/buildscripts/tests/test_validate_commit_message.py +++ b/buildscripts/tests/test_validate_commit_message.py @@ -1,130 +1,31 @@ """Unit tests for the evergreen_task_timeout script.""" -import itertools import unittest -from typing import List, Optional -from unittest.mock import MagicMock -import buildscripts.validate_commit_message as under_test -from buildscripts.client.jiraclient import JiraClient, SecurityLevel -from evergreen import EvergreenApi +from buildscripts.validate_commit_message import main, STATUS_OK, STATUS_ERROR # pylint: disable=missing-docstring,no-self-use -INVALID_MESSAGES = [ - "", # You must provide a message - "RevertEVG-1", # revert and ticket must be formatted - "revert EVG-1", # revert must be capitalized - "This is not a valid message", # message must be valid - "Fix Lint", # Fix lint is strict in terms of caps -] - - -def create_mock_code_change(code_change_messages: List[str], branch_name: Optional[str] = None): - mock_code_change = MagicMock( - commit_messages=code_change_messages, - branch_name=branch_name if branch_name else "mongodb-mongo-master", - ) - return mock_code_change - - -def create_mock_patch(code_change_messages: List[str], branch_name: Optional[str] = None): - mock_code_change = create_mock_code_change(code_change_messages, branch_name) - mock_patch = MagicMock(module_code_changes=[mock_code_change]) - return mock_patch - - -def create_mock_evg_client(code_change_messages: List[str], - branch_name: Optional[str] = None) -> MagicMock: - mock_patch = create_mock_patch(code_change_messages, branch_name) - - mock_evg_client = MagicMock(spec_set=EvergreenApi) - mock_evg_client.patch_by_id.return_value = mock_patch - return mock_evg_client - - -def create_mock_jira_client(): - mock_jira = MagicMock(spec_set=JiraClient) - mock_jira.get_ticket_security_level.return_value = SecurityLevel.NONE - return mock_jira - - -def interleave_new_format(older): - """Create a new list containing a new and old format copy of each string.""" - newer = [ - f"Commit Queue Merge: '{old}' into 'mongodb/mongo:SERVER-45949-validate-message-format'" - for old in older - ] - return list(itertools.chain(*zip(older, newer))) - class ValidateCommitMessageTest(unittest.TestCase): - def test_valid_commits(self): + def test_valid(self): messages = [ - "Fix lint", - "EVG-1", # Test valid projects with various number lengths - "SERVER-20", - "WT-300", "SERVER-44338", - "Revert EVG-5", - "Revert SERVER-60", - "Revert WT-700", - "Revert 'SERVER-8000", - 'Revert "SERVER-90000', + "Revert \"SERVER-60", "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", - "Import tools: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", 'Revert "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4"', ] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_OK) - def test_private(self): - messages = ["XYZ-1"] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + self.assertTrue(all(main([message]) == STATUS_OK for message in messages)) - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_private_with_public(self): + def test_invalid(self): messages = [ - "Fix lint", - "EVG-1", # Test valid projects with various number lengths - "SERVER-20", - "XYZ-1", + "SERVER-", # missing number + "Revert SERVER-60", # missing quote before SERVER + "", # empty value + "nonsense", # nonsense values ] - mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) - mock_jira = create_mock_jira_client() - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_internal_ticket_to_public_repo_should_fail(self): - message = "SERVER-20" - mock_evg_api = create_mock_evg_client(interleave_new_format([message])) - mock_jira = create_mock_jira_client() - mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - - is_valid = orchestrator.validate_commit_messages("version_id") - - self.assertEqual(is_valid, under_test.STATUS_ERROR) - - def test_internal_ticket_to_private_repo_should_succeed(self): - message = "SERVER-20" - mock_evg_api = create_mock_evg_client(interleave_new_format([message]), "private-repo") - mock_jira = create_mock_jira_client() - mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL - orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - is_valid = orchestrator.validate_commit_messages("version_id") + self.assertTrue(all(main([message]) == STATUS_ERROR for message in messages)) - self.assertEqual(is_valid, under_test.STATUS_OK) + def test_message_is_empty_list(self): + self.assertEqual(main([]), STATUS_ERROR) diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py index 4bd8dff252a..23e9c1f331f 100644 --- a/buildscripts/tests/timeouts/test_timeout_service.py +++ b/buildscripts/tests/timeouts/test_timeout_service.py @@ -126,7 +126,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) @patch(ns("HistoricTaskData.from_s3")) def test_enough_history_but_some_tests_with_zero_runtime_should_cause_custom_task_and_default_test_timeout( @@ -155,7 +155,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) @patch(ns("HistoricTaskData.from_s3")) def test_all_tests_with_runtime_history_should_use_custom_timeout(self, @@ -180,7 +180,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(1860, timeout.calculate_test_timeout(1)) - self.assertEqual(54180, timeout.calculate_task_timeout(1)) + self.assertEqual(54360, timeout.calculate_task_timeout(1)) class TestGetTaskHookOverhead(unittest.TestCase): diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py index 7f3296ca1aa..637ee59321e 100644 --- a/buildscripts/tests/util/test_taskname.py +++ b/buildscripts/tests/util/test_taskname.py @@ -28,7 +28,7 @@ class TestRemoveGenSuffix(unittest.TestCase): class TestDetermineTaskBaseName(unittest.TestCase): def test_task_name_with_build_variant_should_strip_bv_and_sub_task_index(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = f"auth_23_{bv}" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -36,7 +36,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_should_strip_sub_task_index(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = "auth_314" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -44,7 +44,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_or_subtask_index_should_self(self): - bv = "enterprise-rhel-80-64-bit-dynamic-required" + bv = "enterprise-rhel-8-64-bit-dynamic-required" task_name = "auth" base_task_name = under_test.determine_task_base_name(task_name, bv) |