diff options
Diffstat (limited to 'buildscripts/tests')
24 files changed, 1476 insertions, 1207 deletions
diff --git a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp index 24b0b42fed3..834d2052a98 100644 --- a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp +++ b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp @@ -27,9 +27,4 @@ LOGV2_ERROR(25, "more words"); LOGV2_ERROR(26, "words", - "comma, more words words words words words words words words words words words words "); -iassert(27, "words"); -iasserted(28, "words"); -iassertNoTrace(29, "words"); -iassertedNoTrace(30, "words"); -MONGO_UNREACHABLE_TASSERT(31); + "comma, more words words words words words words words words words words words words " diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py index 0cba8c8d82f..82434ee25c4 100644 --- a/buildscripts/tests/resmoke_end2end/test_resmoke.py +++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py @@ -4,7 +4,6 @@ import logging import json import os import os.path -import subprocess import sys import time import unittest @@ -172,7 +171,7 @@ class TestTimeout(_ResmokeSelftest): ] self.execute_resmoke(resmoke_args) - archival_dirs_to_expect = 8 # 2 tests * 2 nodes + archival_dirs_to_expect = 4 # 2 tests * 2 nodes self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) @@ -190,11 +189,9 @@ class TestTimeout(_ResmokeSelftest): self.execute_resmoke(resmoke_args, sleep_secs=25) - archival_dirs_to_expect = 4 + archival_dirs_to_expect = 2 # 2 tests * 2 nodes / 2 data_file directories self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect) - archival_inner_dirs_to_expect = 2 - self.assert_dir_file_count(self.test_dir_inner, self.archival_file, - archival_inner_dirs_to_expect) + self.assert_dir_file_count(self.test_dir_inner, self.archival_file, archival_dirs_to_expect) analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo) self.assert_dir_file_count(self.test_dir, self.analysis_file, analysis_pids_to_expect) @@ -428,21 +425,3 @@ class TestSetParameters(_ResmokeSelftest): """--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}""", """--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}""" ]).wait()) - - -class TestMultiversionConfig(unittest.TestCase): - def test_valid_yaml(self): - file_name = "multiversion-config.yml" - subprocess.run([ - sys.executable, "buildscripts/resmoke.py", "multiversion-config", - "--config-file-output", file_name - ], check=True) - with open(file_name, "r") as file: - file_contents = file.read() - - try: - yaml.safe_load(file_contents) - except Exception: # pylint: disable=broad-except - self.fail(msg="`resmoke.py multiversion-config` does not output valid yaml.") - - os.remove(file_name) diff --git a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py index 274b7e5ffbe..2f685f36c97 100644 --- a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py +++ b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py @@ -38,8 +38,6 @@ class TestCalculateFcvConstants(TestCase): "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0"], - "eolVersions": - ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -47,15 +45,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() - self.assertEqual(version_constants.latest, Version("6.0")) - self.assertEqual(version_constants.last_continuous, Version("5.3")) - self.assertEqual(version_constants.last_lts, Version("5.0")) - self.assertEqual(version_constants.requires_fcv_tag_list, + self.assertEqual(fcv_constants.latest, Version("6.0")) + self.assertEqual(fcv_constants.last_continuous, Version("5.3")) + self.assertEqual(fcv_constants.last_lts, Version("5.0")) + self.assertEqual(fcv_constants.requires_fcv_tag_list, [Version(v) for v in ["5.1", "5.2", "5.3", "6.0"]]) - self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) - self.assertEqual(version_constants.fcvs_less_than_latest, [ + self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("6.0")]) + self.assertEqual(fcv_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3"] ]) @@ -69,8 +67,6 @@ class TestCalculateFcvConstants(TestCase): "6.1", "100.0" ], "longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0", "6.0"], - "eolVersions": - ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"], }) multiversion_service = under_test.MultiversionService( @@ -78,15 +74,15 @@ class TestCalculateFcvConstants(TestCase): mongo_releases=mongo_releases, ) - version_constants = multiversion_service.calculate_version_constants() + fcv_constants = multiversion_service.calculate_fcv_constants() - self.assertEqual(version_constants.latest, Version("100.0")) - self.assertEqual(version_constants.last_continuous, Version("6.1")) - self.assertEqual(version_constants.last_lts, Version("6.0")) - self.assertEqual(version_constants.requires_fcv_tag_list, + self.assertEqual(fcv_constants.latest, Version("100.0")) + self.assertEqual(fcv_constants.last_continuous, Version("6.1")) + self.assertEqual(fcv_constants.last_lts, Version("6.0")) + self.assertEqual(fcv_constants.requires_fcv_tag_list, [Version(v) for v in ["6.1", "100.0"]]) - self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) - self.assertEqual(version_constants.fcvs_less_than_latest, [ + self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("100.0")]) + self.assertEqual(fcv_constants.fcvs_less_than_latest, [ Version(v) for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1"] ]) diff --git a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py index ec98f233840..decc81f4433 100644 --- a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py +++ b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py @@ -11,7 +11,6 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags as un from buildscripts.util.fileops import read_yaml_file EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml" -EXPANSIONS_FILE = "expansions.yml" class TestGenerateExcludeYaml(unittest.TestCase): @@ -43,7 +42,6 @@ class TestGenerateExcludeYaml(unittest.TestCase): output = os.path.join(self._tmpdir.name, EXCLUDE_TAGS_FILE) under_test.generate_exclude_yaml(old_bin_version=old_bin_version, output=output, - expansions_file=EXPANSIONS_FILE, logger=MagicMock()) mock_read_yaml.assert_called_once() diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py index c1ba97c6056..b9fc74f088b 100644 --- a/buildscripts/tests/resmokelib/test_parser.py +++ b/buildscripts/tests/resmokelib/test_parser.py @@ -248,6 +248,7 @@ class TestLocalCommandLine(unittest.TestCase): cmdline = to_local_args([ "run", "--suites=my_suite", + "--reportFailureStatus=fail", "--reportFile=report.json", "--perfReportFile=perf.json", "--storageEngine=my_storage_engine", diff --git a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py b/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py deleted file mode 100644 index caeb81e1e4b..00000000000 --- a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Unit tests for buildscripts/resmokelib/testing/symbolizer_service.py.""" -# pylint: disable=missing-docstring -import os -import unittest -from pathlib import Path -from tempfile import TemporaryDirectory -from unittest.mock import MagicMock - -from buildscripts.resmokelib.testing import symbolizer_service as under_test - - -def mock_resmoke_symbolizer_config(): - config_mock: under_test.ResmokeSymbolizerConfig = MagicMock( - spec_set=under_test.ResmokeSymbolizerConfig) - config_mock.evg_task_id = "evg_task_id" - config_mock.client_id = "client_id" - config_mock.client_secret = "client_secret" - config_mock.is_windows.return_value = False - config_mock.is_macos.return_value = False - return config_mock - - -class TestResmokeSymbolizer(unittest.TestCase): - def setUp(self) -> None: - self.config_mock = mock_resmoke_symbolizer_config() - self.symbolizer_service_mock: under_test.SymbolizerService = MagicMock( - spec_set=under_test.SymbolizerService) - self.file_service_mock: under_test.FileService = MagicMock(spec_set=under_test.FileService) - self.resmoke_symbolizer = under_test.ResmokeSymbolizer( - self.config_mock, self.symbolizer_service_mock, self.file_service_mock) - - def test_symbolize_test_logs_process_all_files(self): - stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] - self.file_service_mock.filter_out_non_files.return_value = stacktrace_files - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - - self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 5) - for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): - self.assertEqual(call.args[0], f"file{i}.stacktrace") - self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) - - def test_symbolize_test_logs_hit_timeout(self): - stacktrace_files = [f"file{i}.stacktrace" for i in range(5)] - self.file_service_mock.filter_out_non_files.return_value = stacktrace_files - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock(), 0) - - self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 1) - for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list): - self.assertEqual(call.args[0], f"file{i}.stacktrace") - self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files) - - def test_symbolize_test_logs_should_not_symbolize(self): - self.config_mock.is_windows.return_value = True - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_symbolize_test_logs_could_not_get_dbpath(self): - self.file_service_mock.check_path_exists.return_value = False - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_symbolize_test_logs_did_not_find_files(self): - self.file_service_mock.find_all_children_recursively.return_value = [] - - self.resmoke_symbolizer.symbolize_test_logs(MagicMock()) - self.symbolizer_service_mock.run_symbolizer_script.assert_not_called() - - def test_should_not_symbolize_if_not_in_evergreen(self): - self.config_mock.evg_task_id = None - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_secrets_are_absent(self): - self.config_mock.client_id = None - self.config_mock.client_secret = None - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_on_windows(self): - self.config_mock.is_windows.return_value = True - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_not_symbolize_if_on_macos(self): - self.config_mock.is_macos.return_value = True - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertFalse(ret) - - def test_should_symbolize_return_true(self): - ret = self.resmoke_symbolizer.should_symbolize(MagicMock()) - self.assertTrue(ret) - - def test_get_stacktrace_dir_returns_dir(self): - dbpath = "dbpath" - test = MagicMock(fixture=MagicMock(get_dbpath_prefix=MagicMock(return_value=dbpath))) - self.file_service_mock.check_path_exists.return_value = True - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, dbpath) - - def test_get_stacktrace_dir_if_dir_does_not_exist(self): - test = MagicMock() - self.file_service_mock.check_path_exists.return_value = False - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, None) - - def test_get_stacktrace_dir_if_fixture_is_not_available(self): - test = MagicMock(fixture=None) - - ret = self.resmoke_symbolizer.get_stacktrace_dir(test) - self.assertEqual(ret, None) - - -class TestFileService(unittest.TestCase): - def setUp(self) -> None: - self.file_service = under_test.FileService() - self.relative_dir_paths = [ - os.path.join("dir_1"), - os.path.join("dir_2", "dir_2_1"), - os.path.join("dir_2", "dir_2_2"), - os.path.join("dir_3"), - ] - self.relative_file_paths = [ - os.path.join("dir_1", "file_1.stacktrace"), - os.path.join("dir_2", "file_2.stacktrace"), - os.path.join("dir_2", "dir_2_1", "file_3.stacktrace"), - os.path.join("dir_2", "dir_2_2", "file_4.stacktrace"), - ] - - def test_find_all_children_recursively_returns_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(sorted(ret), sorted(abs_file_paths)) - - def test_find_all_children_recursively_no_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(ret, []) - - def test_find_all_children_recursively_no_dirs(self): - with TemporaryDirectory() as tmpdir: - ret = self.file_service.find_all_children_recursively(tmpdir) - self.assertListEqual(ret, []) - - def test_filter_out_non_files_if_all_files_absent(self): - with TemporaryDirectory() as tmpdir: - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, []) - - def test_filter_out_non_files_if_some_files_present(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - Path(abs_file_paths[1]).touch() - Path(abs_file_paths[3]).touch() - - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, [abs_file_paths[1], abs_file_paths[3]]) - - def test_filter_out_non_files_if_all_files_present(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - ret = self.file_service.filter_out_non_files(abs_file_paths) - self.assertListEqual(ret, abs_file_paths) - - def test_remove_empty_files_if_no_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - with open(file, "w") as fstream: - fstream.write("stacktrace") - - self.file_service.remove_empty(abs_file_paths) - for file in abs_file_paths: - self.assertTrue(os.path.exists(file)) - - def test_remove_empty_files_if_partly_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - with open(abs_file_paths[0], "w") as fstream: - fstream.write("stacktrace") - Path(abs_file_paths[1]).touch() - with open(abs_file_paths[2], "w") as fstream: - fstream.write("stacktrace") - Path(abs_file_paths[3]).touch() - - self.file_service.remove_empty(abs_file_paths) - - self.assertTrue(os.path.exists(abs_file_paths[0])) - self.assertFalse(os.path.exists(abs_file_paths[1])) - self.assertTrue(os.path.exists(abs_file_paths[2])) - self.assertFalse(os.path.exists(abs_file_paths[3])) - - def test_remove_empty_files_if_all_empty(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - Path(file).touch() - - self.file_service.remove_empty(abs_file_paths) - for file in abs_file_paths: - self.assertFalse(os.path.exists(file)) - - def test_remove_all_files(self): - with TemporaryDirectory() as tmpdir: - abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths] - abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths] - for dir_ in abs_dir_paths: - Path(dir_).mkdir(parents=True) - for file in abs_file_paths: - with open(file, "w") as fstream: - fstream.write("stacktrace") - - self.file_service.remove_all(abs_file_paths) - for file in abs_file_paths: - self.assertFalse(os.path.exists(file)) diff --git a/buildscripts/tests/task_generation/test_suite_split.py b/buildscripts/tests/task_generation/test_suite_split.py index b2b197ed7c5..7c54ed489aa 100644 --- a/buildscripts/tests/task_generation/test_suite_split.py +++ b/buildscripts/tests/task_generation/test_suite_split.py @@ -8,7 +8,7 @@ import requests import buildscripts.task_generation.suite_split as under_test from buildscripts.task_generation.suite_split_strategies import greedy_division, \ round_robin_fallback -from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation +from buildscripts.util.teststats import TestRuntime # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access @@ -31,12 +31,7 @@ def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None): def tst_stat_mock(file, duration, pass_count): - return HistoricalTestInformation( - test_name=file, - num_pass=pass_count, - num_fail=0, - avg_duration_pass=duration, - ) + return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None): @@ -120,16 +115,15 @@ class TestGeneratedSuite(unittest.TestCase): class TestSplitSuite(unittest.TestCase): - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites(self, get_stats_from_s3_mock): + def test_calculate_suites(self): mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_name for stat in mock_test_stats + stat.test_file for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} @@ -143,15 +137,32 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(10, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock): + def test_calculate_suites_fallback_on_error(self): + n_tests = 100 + max_sub_suites = 4 + split_config = build_mock_split_config(max_sub_suites=max_sub_suites) + split_params = build_mock_split_params() + + suite_split_service = build_mock_service(split_config=split_config) + mock_evg_error(suite_split_service.evg_api) + suite_split_service.resmoke_proxy.list_tests.return_value = [ + f"test_{i}.js" for i in range(n_tests) + ] + + suite = suite_split_service.split_suite(split_params) + + self.assertEqual(max_sub_suites, len(suite)) + for sub_suite in suite.sub_suites: + self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) + + def test_calculate_suites_uses_fallback_on_no_results(self): n_tests = 100 max_sub_suites = 5 split_config = build_mock_split_config(max_sub_suites=max_sub_suites) split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = [] + suite_split_service.evg_api.test_stats_by_project.return_value = [] suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -162,9 +173,7 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_uses_fallback_if_only_results_are_filtered( - self, get_stats_from_s3_mock): + def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self): n_tests = 100 max_sub_suites = 10 mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)] @@ -173,7 +182,7 @@ class TestSplitSuite(unittest.TestCase): split_params = build_mock_split_params() suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ f"test_{i}.js" for i in range(n_tests) ] @@ -189,17 +198,31 @@ class TestSplitSuite(unittest.TestCase): for sub_suite in suite.sub_suites: self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list)) - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock): + def test_calculate_suites_fail_on_unexpected_error(self): + n_tests = 100 + max_sub_suites = 4 + split_config = build_mock_split_config(max_sub_suites=max_sub_suites) + split_params = build_mock_split_params() + + suite_split_service = build_mock_service(split_config=split_config) + mock_evg_error(suite_split_service.evg_api, error_code=requests.codes.INTERNAL_SERVER_ERROR) + suite_split_service.resmoke_proxy.list_tests.return_value = [ + f"test_{i}.js" for i in range(n_tests) + ] + + with self.assertRaises(requests.HTTPError): + suite_split_service.split_suite(split_params) + + def test_calculate_suites_will_filter_specified_tests(self): mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] split_config = build_mock_split_config(target_resmoke_time=10) split_params = build_mock_split_params( test_filter=lambda t: t in {"test_1.js", "test_2.js"}) suite_split_service = build_mock_service(split_config=split_config) - get_stats_from_s3_mock.return_value = mock_test_stats + suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats suite_split_service.resmoke_proxy.list_tests.return_value = [ - stat.test_name for stat in mock_test_stats + stat.test_file for stat in mock_test_stats ] suite_split_service.resmoke_proxy.read_suite_config.return_value = {} diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py new file mode 100644 index 00000000000..ec53a02d161 --- /dev/null +++ b/buildscripts/tests/test_burn_in_tags.py @@ -0,0 +1,261 @@ +"""Unit tests for the burn_in_tags.py script.""" +from collections import defaultdict +import json +import os +import sys +import unittest +from unittest.mock import MagicMock, patch + +from shrub.v2 import ShrubProject + +import buildscripts.ciconfig.evergreen as _evergreen +from buildscripts.burn_in_tests import TaskInfo +from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns +from buildscripts.ciconfig.evergreen import EvergreenProjectConfig + +import buildscripts.burn_in_tags as under_test + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access + +EMPTY_PROJECT = { + "buildvariants": [], + "tasks": [], +} +TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "test_burn_in_tags_evergreen.yml") + +NS = "buildscripts.burn_in_tags" + + +def ns(relative_name): # pylint: disable-invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + + +def get_expansions_data(): + return { + "branch_name": "fake_branch", + "build_variant": "enterprise-rhel-80-64-bit-suggested", + "check_evergreen": 2, + "distro_id": "rhel80-small", + "is_patch": "true", + "max_revisions": 25, + "repeat_tests_max": 1000, + "repeat_tests_min": 2, + "repeat_tests_secs": 600, + "revision": "fake_sha", + "project": "fake_project", + "task_id": "task id", + } # yapf: disable + + +def get_evergreen_config() -> EvergreenProjectConfig: + return _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None) + + +class TestCreateEvgBuildVariantMap(unittest.TestCase): + def test_create_evg_buildvariant_map(self): + expansions_file_data = { + "build_variant": "variant1", "burn_in_tag_buildvariants": "variant2 variant3" + } + buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data) + + expected_buildvariant_map = { + "variant2": "variant2-required", "variant3": "variant3-required" + } + self.assertEqual(buildvariant_map, expected_buildvariant_map) + + +class TestGenerateEvgBuildVariants(unittest.TestCase): + def test_generate_evg_buildvariant_one_base_variant(self): + evg_conf_mock = get_evergreen_config() + base_variant = "enterprise-rhel-80-64-bit-inmem" + generated_variant = "enterprise-rhel-80-64-bit-inmem-required" + burn_in_tags_gen_variant = "enterprise-rhel-80-64-bit" + variant = evg_conf_mock.get_variant(base_variant) + + build_variant = under_test._generate_evg_build_variant(variant, generated_variant, + burn_in_tags_gen_variant) + + generated_build_variant = build_variant.as_dict() + self.assertEqual(generated_build_variant["name"], generated_variant) + self.assertEqual(generated_build_variant["modules"], variant.modules) + generated_expansions = generated_build_variant["expansions"] + burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass") + self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant) + self.assertEqual(generated_expansions, variant.expansions) + + +class TestGenerateEvgTasks(unittest.TestCase): + @patch(ns("create_tests_by_task")) + def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock): + evg_conf_mock = get_evergreen_config() + create_tests_by_task_mock.return_value = {} + expansions_file_data = get_expansions_data() + buildvariant_map = { + "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", + "enterprise-rhel-80-64-bit-majority-read-concern-off": + "enterprise-rhel-80-64-bit-majority-read-concern-off-required", + } # yapf: disable + shrub_config = ShrubProject() + evergreen_api = MagicMock() + repo = MagicMock(working_dir=os.getcwd()) + under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, + buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') + + self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT) + + @patch(ns("create_tests_by_task")) + def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock): + evg_conf_mock = get_evergreen_config() + create_tests_by_task_mock.return_value = { + "aggregation_mongos_passthrough": TaskInfo( + display_task_name="aggregation_mongos_passthrough", + suite="aggregation_mongos_passthrough", + resmoke_args="--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger", + tests=["jstests/aggregation/ifnull.js"], + require_multiversion_setup=False, + distro="", + build_variant="enterprise-rhel-80-64-bit-inmem" + ) + } # yapf: disable + expansions_file_data = get_expansions_data() + buildvariant_map = { + "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required", + "enterprise-rhel-80-64-bit-majority-read-concern-off": + "enterprise-rhel-80-64-bit-majority-read-concern-off-required", + } # yapf: disable + shrub_config = ShrubProject.empty() + evergreen_api = MagicMock() + repo = MagicMock(working_dir=os.getcwd()) + evergreen_api.test_stats_by_project.return_value = [ + MagicMock(test_file="dir/test2.js", avg_duration_pass=10) + ] + under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data, + buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin') + + generated_config = shrub_config.as_dict() + self.assertEqual(len(generated_config["buildvariants"]), 2) + first_generated_build_variant = generated_config["buildvariants"][0] + self.assertIn(first_generated_build_variant["name"], buildvariant_map.values()) + self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests") + self.assertEqual( + first_generated_build_variant["display_tasks"][0]["execution_tasks"][0], + f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}") + + +EXPANSIONS_FILE_DATA = { + "build_variant": "enterprise-rhel-80-64-bit", + "revision": "badf00d000000000000000000000000000000000", "max_revisions": "1000", + "branch_name": "mongodb-mongo-master", "is_patch": "false", "distro_id": "rhel62-small", + "repeat_tests_min": "2", "repeat_tests_max": "1000", "repeat_tests_secs": "600", "project": + "mongodb-mongo-master", "task_id": "task id" +} + +CREATE_EVG_BUILD_VARIANT_MAP = { + 'enterprise-rhel-80-64-bit-majority-read-concern-off': + 'enterprise-rhel-80-64-bit-majority-read-concern-off-required', + 'enterprise-rhel-80-64-bit-inmem': + 'enterprise-rhel-80-64-bit-inmem-required' +} + +CREATE_TEST_MEMBERSHIP_MAP = { + "jstests/aggregation/accumulators/accumulator_js.js": [ + "aggregation", "aggregation_auth", "aggregation_disabled_optimization", "aggregation_ese", + "aggregation_ese_gcm", "aggregation_facet_unwind_passthrough", + "aggregation_mongos_passthrough", "aggregation_one_shard_sharded_collections", + "aggregation_read_concern_majority_passthrough", "aggregation_secondary_reads", + "aggregation_sharded_collections_passthrough" + ], "jstests/core/create_collection.js": [ + "core", "core_auth", "core_ese", "core_ese_gcm", "core_minimum_batch_size", "core_op_query", + "cwrwc_passthrough", "cwrwc_rc_majority_passthrough", "cwrwc_wc_majority_passthrough", + "logical_session_cache_replication_100ms_refresh_jscore_passthrough", + "logical_session_cache_replication_10sec_refresh_jscore_passthrough", + "logical_session_cache_replication_1sec_refresh_jscore_passthrough", + "logical_session_cache_replication_default_refresh_jscore_passthrough", + "logical_session_cache_standalone_100ms_refresh_jscore_passthrough", + "logical_session_cache_standalone_10sec_refresh_jscore_passthrough", + "logical_session_cache_standalone_1sec_refresh_jscore_passthrough", + "logical_session_cache_standalone_default_refresh_jscore_passthrough", + "read_concern_linearizable_passthrough", "read_concern_majority_passthrough", + "causally_consistent_read_concern_snapshot_passthrough", + "replica_sets_initsync_jscore_passthrough", "replica_sets_fcbis_jscore_passthrough", + "replica_sets_initsync_static_jscore_passthrough", "replica_sets_jscore_passthrough", + "replica_sets_kill_primary_jscore_passthrough", + "replica_sets_kill_secondaries_jscore_passthrough", + "replica_sets_reconfig_jscore_passthrough", + "replica_sets_terminate_primary_jscore_passthrough", "retryable_writes_jscore_passthrough", + "retryable_writes_jscore_stepdown_passthrough", "secondary_reads_passthrough", + "session_jscore_passthrough", "write_concern_majority_passthrough" + ] +} + + +class TestAcceptance(unittest.TestCase): + @patch(ns("write_file_to_dir")) + @patch(ns("_create_evg_build_variant_map")) + @patch(ns("EvergreenFileChangeDetector")) + def test_no_tests_run_if_none_changed(self, find_changed_tests_mock, + create_evg_build_variant_map_mock, write_to_file_mock): + """ + Given a git repository with no changes, + When burn_in_tags is run, + Then no tests are discovered to run. + """ + repos = [MagicMock(working_dir=os.getcwd())] + evg_conf_mock = MagicMock() + find_changed_tests_mock.return_value.find_changed_tests.return_value = {} + + create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP + + under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf_mock, MagicMock(), repos, + 'install_dir/bin') + + write_to_file_mock.assert_called_once() + shrub_config = write_to_file_mock.call_args[0][2] + self.assertEqual(EMPTY_PROJECT, json.loads(shrub_config)) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("write_file_to_dir")) + @patch(ns("_create_evg_build_variant_map")) + @patch(ns("EvergreenFileChangeDetector")) + @patch(burn_in_tests_ns("create_test_membership_map")) + def test_tests_generated_if_a_file_changed( + self, create_test_membership_map_mock, find_changed_tests_mock, + create_evg_build_variant_map_mock, write_to_file_mock): + """ + Given a git repository with changes, + When burn_in_tags is run, + Then some tags are discovered to run. + """ + create_test_membership_map_mock.return_value = defaultdict(list, CREATE_TEST_MEMBERSHIP_MAP) + + repos = [MagicMock(working_dir=os.getcwd())] + evg_conf = get_evergreen_config() + create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP + find_changed_tests_mock.return_value.find_changed_tests.return_value = { + 'jstests/slow1/large_role_chain.js', + 'jstests/aggregation/accumulators/accumulator_js.js' + } + + under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin') + + write_to_file_mock.assert_called_once() + written_config = write_to_file_mock.call_args[0][2] + written_config_map = json.loads(written_config) + + n_tasks = len(written_config_map["tasks"]) + # Ensure we are generating at least one task for the test. + self.assertGreaterEqual(n_tasks, 1) + + written_build_variants = written_config_map["buildvariants"] + written_build_variants_name = [variant['name'] for variant in written_build_variants] + self.assertEqual( + set(CREATE_EVG_BUILD_VARIANT_MAP.values()), set(written_build_variants_name)) + + tasks = written_config_map["tasks"] + self.assertGreaterEqual(len(tasks), len(CREATE_EVG_BUILD_VARIANT_MAP)) + + self.assertTrue( + all( + len(display_tasks) == 1 for display_tasks in + [build_variant["display_tasks"] for build_variant in written_build_variants])) diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml new file mode 100644 index 00000000000..d6a651fcde8 --- /dev/null +++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml @@ -0,0 +1,138 @@ +functions: + "fetch source": + - command: git.get_project + params: + directory: src + - command: shell.exec + params: + working_dir: src + script: | + echo "this is a 2nd command in the function!" + ls + +tasks: +- name: compile + depends_on: [] + commands: + - func: "fetch source" +- name: burn_in_tags_gen + depends_on: [] + commands: + - func: "fake command" +- name: compile_all_run_unittests_TG + depends_on: [] + commands: + - func: "fake command" +- name: clang_tidy_TG + depends_on: [] + commands: + - func: "fake command" +- name: stitch_support_lib_build_and_archive + depends_on: [] + commands: + - func: "fake command" +- name: lint_pylinters + depends_on: [] + commands: + - func: "fake command" +- name: lint_clang_format + depends_on: [] + commands: + - func: "fake command" +- name: burn_in_tests_gen + depends_on: [] + commands: + - func: "fake command" +- name: aggregation_multiversion_fuzzer_gen + depends_on: [] + commands: + - func: "generate resmoke tasks" +- name: aggregation_expression_multiversion_fuzzer_gen + depends_on: [] + commands: + - func: "generate resmoke tasks" +- name: aggregation + depends_on: + - name: compile + commands: + - func: run tests + vars: + resmoke_args: --suites=aggregation --storageEngine=wiredTiger + +buildvariants: +- name: enterprise-rhel-80-64-bit + display_name: "! Enterprise RHEL 8.0" + expansions: + multiversion_platform: rhel80 + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: lint_pylinters + - name: burn_in_tests_gen + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation + - name: burn_in_tags_gen +- name: buildvariant-without-burn-in-tag-buildvariants + display_name: "Buildvariant without burn in tag buildvariants expansion" + expansions: + multiversion_platform: rhel80 + tasks: + - name: burn_in_tags_gen +- name: enterprise-rhel-80-64-bit-majority-read-concern-off + display_name: "Enterprise RHEL 8.0 (majority read concern off)" + modules: ["enterprise"] + run_on: + - rhel80-small + expansions: &enterprise-rhel-80-64-bit-majority-read-concern-off-expansions + multiversion_edition: enterprise + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation +- name: enterprise-rhel-80-64-bit-inmem + display_name: Enterprise RHEL 8.0 (inMemory) + modules: ["enterprise"] + run_on: + - rhel80-small + expansions: &enterprise-rhel-80-64-bit-inmem-expansions + test_flags: >- + --majorityReadConcern=off + --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime + compile_flags: >- + -j$(grep -c ^processor /proc/cpuinfo) + --ssl + --release + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + MONGO_DISTMOD=rhel80 + multiversion_platform: rhel80 + multiversion_edition: enterprise + scons_cache_scope: shared + tooltags: "ssl sasl gssapi" + large_distro_name: rhel80-large + tasks: + - name: compile +- name: enterprise-rhel-80-64-bit-inmem + display_name: Enterprise RHEL 8.0 (inMemory) + expansions: + additional_targets: archive-mongocryptd archive-mongocryptd-debug + compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo) + --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars + large_distro_name: rhel80-large + multiversion_edition: enterprise + multiversion_platform: rhel80 + scons_cache_scope: shared + test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling + modules: + - enterprise + run_on: + - rhel80-small + tasks: + - name: compile_all_run_unittests_TG + distros: + - rhel80-large + - name: aggregation_multiversion_fuzzer_gen + - name: aggregation diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 369131eff60..c51f8c60ecf 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,14 +4,12 @@ from __future__ import absolute_import import collections import datetime -from io import StringIO import os import sys import subprocess import unittest from mock import Mock, patch, MagicMock -import yaml import buildscripts.burn_in_tests as under_test from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask @@ -558,19 +556,3 @@ class TestLocalFileChangeDetector(unittest.TestCase): self.assertIn(file_list[2], found_tests) self.assertNotIn(file_list[1], found_tests) self.assertEqual(2, len(found_tests)) - - -class TestYamlBurnInExecutor(unittest.TestCase): - @patch('sys.stdout', new_callable=StringIO) - def test_found_tasks_should_be_reported_as_yaml(self, stdout): - n_tasks = 5 - n_tests = 3 - tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) - - yaml_executor = under_test.YamlBurnInExecutor() - yaml_executor.execute(tests_by_task) - - yaml_raw = stdout.getvalue() - results = yaml.safe_load(yaml_raw) - self.assertEqual(n_tasks, len(results["discovered_tasks"])) - self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) diff --git a/buildscripts/tests/test_debugsymb_mapper.py b/buildscripts/tests/test_debugsymb_mapper.py deleted file mode 100644 index b1b063f5331..00000000000 --- a/buildscripts/tests/test_debugsymb_mapper.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Unit tests for debugsymb_mapper.py.""" -# pylint: disable=missing-docstring -import unittest -from unittest.mock import MagicMock - -import buildscripts.debugsymb_mapper as under_test - - -def mock_cmd_client(): - cmd_client = MagicMock(spec_set=under_test.CmdClient) - return cmd_client - - -class TestCmdOutputExtractor(unittest.TestCase): - def setUp(self): - self.cmd_client_mock = mock_cmd_client() - self.cmd_output_extractor = under_test.CmdOutputExtractor(self.cmd_client_mock) - - -class TestGetBuildId(TestCmdOutputExtractor): - def test_get_build_id_returns_build_id(self): - readelf_output = ( - "Displaying notes found in: .note.gnu.build-id\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot\n" - " Owner Data size\tDescription\n" - " GA$<version>3h864 0x00000010\tOPEN\n" - " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)\n" - " GA$<version>3h864 0x00000010\tOPEN\n" - " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)") - self.cmd_client_mock.run.return_value = readelf_output - - build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") - self.assertEqual(build_id_output.build_id, "74c2322104428836f3d94af6cd7471ee7cb5c4ee") - self.assertEqual(build_id_output.cmd_output, readelf_output) - - def test_get_build_id_raises_error(self): - readelf_output = ( - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n" - " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n" - "\n" - "Displaying notes found in: .gnu.build.attributes.hot") - self.cmd_client_mock.run.return_value = readelf_output - - self.assertRaises(ValueError, self.cmd_output_extractor.get_build_id, "path/to/bin") - - def test_get_build_id_returns_none(self): - readelf_output = ( - "Displaying notes found in: .note.gnu.build-id\n" - " Owner Data size\tDescription\n" - " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)") - self.cmd_client_mock.run.return_value = readelf_output - - build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin") - self.assertIsNone(build_id_output.build_id) - self.assertEqual(build_id_output.cmd_output, readelf_output) - - -class TestGetBinVersion(TestCmdOutputExtractor): - def test_get_bin_version_returns_version(self): - # Newer versions command output - version_cmd_output = ('db version v4.4.14-25-gb0475e2\n' - 'Build Info: {\n' - ' "version": "4.4.14-25-gb0475e2",\n' - ' "gitVersion": "b0475e2657c3351b25499971d3340f054ea85b98",\n' - ' "openSSLVersion": "OpenSSL 1.1.1 11 Sep 2018",\n' - ' "modules": [\n' - ' "enterprise"\n' - ' ],\n' - ' "allocator": "tcmalloc",\n' - ' "environment": {\n' - ' "distmod": "ubuntu1804",\n' - ' "distarch": "x86_64",\n' - ' "target_arch": "x86_64"\n' - ' }\n' - '}') - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertEqual(bin_version_output.mongodb_version, "4.4.14-25-gb0475e2") - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) - - def test_get_bin_version_unsupported_output(self): - # Versions prior to 5.0 are not supported - version_cmd_output = ('db version v4.2.20-7-g5a81409\n' - 'git version: 5a81409faf16f30f1189af6367eb3ceee50a02b5\n' - 'OpenSSL version: OpenSSL 1.1.1 11 Sep 2018\n' - 'allocator: tcmalloc\n' - 'modules: enterprise \n' - 'build environment:\n' - ' distmod: ubuntu1804\n' - ' distarch: x86_64\n' - ' target_arch: x86_64') - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertIsNone(bin_version_output.mongodb_version) - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) - - def test_get_bin_version_returns_none(self): - version_cmd_output = "error: unrecognized arguments: --version" - self.cmd_client_mock.run.return_value = version_cmd_output - - bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin") - self.assertIsNone(bin_version_output.mongodb_version) - self.assertEqual(bin_version_output.cmd_output, version_cmd_output) diff --git a/buildscripts/tests/test_errorcodes.py b/buildscripts/tests/test_errorcodes.py index 17c102454b5..2a9c9ce1e3f 100644 --- a/buildscripts/tests/test_errorcodes.py +++ b/buildscripts/tests/test_errorcodes.py @@ -26,7 +26,7 @@ class TestErrorcodes(unittest.TestCase): captured_error_codes.append(code) errorcodes.parse_source_files(accumulate_files, TESTDATA_DIR + 'regex_matching/') - self.assertEqual(31, len(captured_error_codes)) + self.assertEqual(26, len(captured_error_codes)) def test_dup_checking(self): """Test dup checking.""" diff --git a/buildscripts/tests/test_evergreen_activate_gen_tasks.py b/buildscripts/tests/test_evergreen_activate_gen_tasks.py index 5470828c6c3..dbb64380838 100644 --- a/buildscripts/tests/test_evergreen_activate_gen_tasks.py +++ b/buildscripts/tests/test_evergreen_activate_gen_tasks.py @@ -1,126 +1,42 @@ """Unit tests for the generate_resmoke_suite script.""" -# pylint: disable=invalid-name import unittest -from mock import MagicMock, mock +from mock import MagicMock from buildscripts import evergreen_activate_gen_tasks as under_test -from evergreen import Build, EvergreenApi, Task, Version # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access # pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter -def build_mock_task(display_name, task_id): - mock_task = MagicMock(spec_set=Task, display_name=display_name, task_id=task_id) +def build_mock_task(name, task_id): + mock_task = MagicMock(display_name=name, task_id=task_id) return mock_task -def build_mock_task_list(num_tasks): - return [build_mock_task(f"task_{i}", f"id_{i}") for i in range(num_tasks)] - - -class MockVariantData(): - """An object to help create a mock evg api.""" - - def __init__(self, build_id, variant_name, task_list): - self.build_id = build_id - self.variant_name = variant_name - self.task_list = task_list - - -def build_mock_evg_api(variant_data_list): - class VersionPatchedSpec(Version): - """A patched `Version` with instance properties included for magic mock spec.""" - build_variants_map = MagicMock() - - mock_version = MagicMock(spec_set=VersionPatchedSpec) - mock_version.build_variants_map = { - variant_data.variant_name: variant_data.build_id - for variant_data in variant_data_list - } - - mock_evg_api = MagicMock(spec_set=EvergreenApi) - mock_evg_api.version_by_id.return_value = mock_version - - build_id_mapping = { - variant_data.build_id: variant_data.task_list - for variant_data in variant_data_list - } - - def tasks_by_build_side_effect(build_id): - return build_id_mapping[build_id] - - mock_evg_api.tasks_by_build.side_effect = tasks_by_build_side_effect +def build_mock_evg_api(mock_task_list): + mock_build = MagicMock() + mock_build.get_tasks.return_value = mock_task_list + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value = mock_build return mock_evg_api class TestActivateTask(unittest.TestCase): def test_task_with_display_name_is_activated(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "task_3_gen", - }) - mock_task_list = build_mock_task_list(5) - mock_evg_api = build_mock_evg_api( - [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) - under_test.activate_task(expansions, mock_evg_api) + under_test.activate_task("build_id", "task_3", mock_evg_api) mock_evg_api.configure_task.assert_called_with("id_3", activated=True) def test_task_with_no_matching_name(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "not_an_existing_task", - }) - mock_task_list = build_mock_task_list(5) - mock_evg_api = build_mock_evg_api( - [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)]) + n_tasks = 5 + mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)] + mock_evg_api = build_mock_evg_api(mock_task_list) - under_test.activate_task(expansions, mock_evg_api) + under_test.activate_task("build_id", "not_an_existing_task", mock_evg_api) mock_evg_api.configure_task.assert_not_called() - - def test_burn_in_tags_tasks_are_activated(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "burn_in_tags_gen", - }) - mock_task_list_2 = build_mock_task_list(5) - mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) - mock_task_list_3 = build_mock_task_list(5) - mock_task_list_3.append(build_mock_task("burn_in_tests", "burn_in_tests_id_3")) - mock_evg_api = build_mock_evg_api([ - MockVariantData("1", "variant1-generated-by-burn-in-tags", mock_task_list_2), - MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_3) - ]) - - under_test.activate_task(expansions, mock_evg_api) - - mock_evg_api.configure_task.assert_has_calls([ - mock.call("burn_in_tests_id_2", activated=True), - mock.call("burn_in_tests_id_3", activated=True) - ]) - - def test_burn_in_tags_task_skips_non_existing_build_variant(self): - expansions = under_test.EvgExpansions(**{ - "build_id": "build_id", - "version_id": "version_id", - "task_name": "burn_in_tags_gen", - }) - mock_task_list_1 = build_mock_task_list(5) - mock_task_list_1.append(build_mock_task("burn_in_tags_gen", "burn_in_tags_gen_id_1")) - mock_task_list_2 = build_mock_task_list(5) - mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2")) - mock_evg_api = build_mock_evg_api([ - MockVariantData("1", "variant1-non-burn-in", mock_task_list_1), - MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_2) - ]) - - under_test.activate_task(expansions, mock_evg_api) - - mock_evg_api.configure_task.assert_called_once_with("burn_in_tests_id_2", activated=True) diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py new file mode 100644 index 00000000000..ee77ced2579 --- /dev/null +++ b/buildscripts/tests/test_evergreen_burn_in_tests.py @@ -0,0 +1,408 @@ +"""Unit tests for buildscripts/burn_in_tests.py.""" + +from __future__ import absolute_import + +import json +import os +import sys +import unittest +from datetime import datetime, timedelta +from math import ceil + +import requests +from mock import patch, MagicMock +from shrub.v2 import BuildVariant, ShrubProject +from evergreen.api import EvergreenApi + +import buildscripts.evergreen_burn_in_tests as under_test +from buildscripts.ciconfig.evergreen import parse_evergreen_file +import buildscripts.resmokelib.parser as _parser +import buildscripts.resmokelib.config as _config +import buildscripts.util.teststats as teststats_utils +_parser.set_run_options() + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access + +NS = "buildscripts.evergreen_burn_in_tests" + + +def ns(relative_name): # pylint: disable=invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + + +def mock_a_file(filename): + change = MagicMock(a_path=filename) + return change + + +def mock_git_diff(change_list): + diff = MagicMock() + diff.iter_change_type.return_value = change_list + return diff + + +def mock_changed_git_files(add_files): + repo = MagicMock() + repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) + repo.working_dir = "." + return repo + + +def get_evergreen_config(config_file_path): + evergreen_home = os.path.expanduser(os.path.join("~", "evergreen")) + if os.path.exists(evergreen_home): + return parse_evergreen_file(config_file_path, evergreen_home) + return parse_evergreen_file(config_file_path) + + +class TestAcceptance(unittest.TestCase): + def tearDown(self): + _parser.set_run_options() + + @patch(ns("write_file")) + def test_no_tests_run_if_none_changed(self, write_json_mock): + """ + Given a git repository with no changes, + When burn_in_tests is run, + Then no tests are discovered to run. + """ + variant = "build_variant" + repos = [mock_changed_git_files([])] + repeat_config = under_test.RepeatConfig() + gen_config = under_test.GenerateConfig( + variant, + "project", + ) # yapf: disable + mock_evg_conf = MagicMock() + mock_evg_conf.get_task_names_by_tag.return_value = set() + mock_evg_api = MagicMock() + + under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, + mock_evg_conf, repos, "testfile.json", "install-dir/bin") + + write_json_mock.assert_called_once() + written_config = json.loads(write_json_mock.call_args[0][1]) + display_task = written_config["buildvariants"][0]["display_tasks"][0] + self.assertEqual(1, len(display_task["execution_tasks"])) + self.assertEqual(under_test.BURN_IN_TESTS_GEN_TASK, display_task["execution_tasks"][0]) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("write_file")) + def test_tests_generated_if_a_file_changed(self, write_json_mock): + """ + Given a git repository with changes, + When burn_in_tests is run, + Then tests are discovered to run. + """ + # Note: this test is using actual tests and suites. So changes to those suites could + # introduce failures and require this test to be updated. + # You can see the test file it is using below. This test is used in the 'auth' and + # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. + variant = "enterprise-rhel-80-64-bit-inmem" + repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] + repeat_config = under_test.RepeatConfig() + gen_config = under_test.GenerateConfig( + variant, + "project", + ) # yapf: disable + mock_evg_conf = get_evergreen_config("etc/evergreen.yml") + mock_evg_api = MagicMock() + + under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api, + mock_evg_conf, repos, "testfile.json", 'install-dir/bin') + + write_json_mock.assert_called_once() + written_config = json.loads(write_json_mock.call_args[0][1]) + n_tasks = len(written_config["tasks"]) + # Ensure we are generating at least one task for the test. + self.assertGreaterEqual(n_tasks, 1) + + written_build_variant = written_config["buildvariants"][0] + self.assertEqual(variant, written_build_variant["name"]) + self.assertEqual(n_tasks, len(written_build_variant["tasks"])) + + display_task = written_build_variant["display_tasks"][0] + # The display task should contain all the generated tasks as well as 1 extra task for + # the burn_in_test_gen task. + self.assertEqual(n_tasks + 1, len(display_task["execution_tasks"])) + + +class TestGenerateConfig(unittest.TestCase): + def test_run_build_variant_with_no_run_build_variant(self): + gen_config = under_test.GenerateConfig("build_variant", "project") + + self.assertEqual(gen_config.build_variant, gen_config.run_build_variant) + + def test_run_build_variant_with_run_build_variant(self): + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + + self.assertNotEqual(gen_config.build_variant, gen_config.run_build_variant) + self.assertEqual(gen_config.run_build_variant, "run_build_variant") + + def test_validate_non_existing_build_variant(self): + evg_conf_mock = MagicMock() + evg_conf_mock.get_variant.return_value = None + + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + + with self.assertRaises(ValueError): + gen_config.validate(evg_conf_mock) + + def test_validate_existing_build_variant(self): + evg_conf_mock = MagicMock() + + gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant") + gen_config.validate(evg_conf_mock) + + def test_validate_non_existing_run_build_variant(self): + evg_conf_mock = MagicMock() + + gen_config = under_test.GenerateConfig("build_variant", "project") + gen_config.validate(evg_conf_mock) + + +class TestParseAvgTestRuntime(unittest.TestCase): + def test__parse_avg_test_runtime(self): + task_avg_test_runtime_stats = [ + teststats_utils.TestRuntime(test_name="dir/test1.js", runtime=30.2), + teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1) + ] + result = under_test._parse_avg_test_runtime("dir/test2.js", task_avg_test_runtime_stats) + self.assertEqual(result, 455.1) + + +class TestCalculateTimeout(unittest.TestCase): + def test__calculate_timeout(self): + avg_test_runtime = 455.1 + expected_result = ceil(avg_test_runtime * under_test.AVG_TEST_TIME_MULTIPLIER) + self.assertEqual(expected_result, under_test._calculate_timeout(avg_test_runtime)) + + def test__calculate_timeout_avg_is_less_than_min(self): + avg_test_runtime = 10 + self.assertEqual(under_test.MIN_AVG_TEST_TIME_SEC, + under_test._calculate_timeout(avg_test_runtime)) + + +class TestCalculateExecTimeout(unittest.TestCase): + def test__calculate_exec_timeout(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + avg_test_runtime = 455.1 + + exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) + + self.assertEqual(1771, exec_timeout) + + def test_average_timeout_greater_than_execution_time(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600, repeat_tests_min=2) + avg_test_runtime = 750 + + exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime) + + # The timeout needs to be greater than the number of the test * the minimum number of runs. + minimum_expected_timeout = avg_test_runtime * repeat_config.repeat_tests_min + + self.assertGreater(exec_timeout, minimum_expected_timeout) + + +class TestGenerateTimeouts(unittest.TestCase): + def test__generate_timeouts(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)] + test_name = "dir/test2.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertEqual(timeout_info.exec_timeout, 1771) + self.assertEqual(timeout_info.timeout, 1366) + + def test__generate_timeouts_no_results(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [] + test_name = "dir/new_test.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertIsNone(timeout_info.cmd) + + def test__generate_timeouts_avg_runtime_is_zero(self): + repeat_config = under_test.RepeatConfig(repeat_tests_secs=600) + runtime_stats = [ + teststats_utils.TestRuntime(test_name="dir/test_with_zero_runtime.js", runtime=0) + ] + test_name = "dir/test_with_zero_runtime.js" + + task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats) + timeout_info = task_generator.generate_timeouts(test_name) + + self.assertIsNone(timeout_info.cmd) + + +class TestGetTaskRuntimeHistory(unittest.TestCase): + def test_get_task_runtime_history(self): + mock_evg_api = MagicMock() + mock_evg_api.test_stats_by_project.return_value = [ + MagicMock( + test_file="dir/test2.js", + task_name="task1", + variant="variant1", + distro="distro1", + date=datetime.utcnow().date(), + num_pass=1, + num_fail=0, + avg_duration_pass=10.1, + ) + ] + analysis_duration = under_test.AVG_TEST_RUNTIME_ANALYSIS_DAYS + end_date = datetime.utcnow().replace(microsecond=0) + start_date = end_date - timedelta(days=analysis_duration) + mock_gen_config = MagicMock(project="project1", build_variant="variant1") + + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api, + history_end_date=end_date) + result = executor.get_task_runtime_history("task1") + + self.assertEqual(result, [("dir/test2.js", 10.1)]) + mock_evg_api.test_stats_by_project.assert_called_with( + "project1", after_date=start_date, before_date=end_date, group_by="test", + group_num_days=14, tasks=["task1"], variants=["variant1"]) + + def test_get_task_runtime_history_evg_degraded_mode_error(self): + mock_response = MagicMock(status_code=requests.codes.SERVICE_UNAVAILABLE) + mock_evg_api = MagicMock() + mock_evg_api.test_stats_by_project.side_effect = requests.HTTPError(response=mock_response) + mock_gen_config = MagicMock(project="project1", build_variant="variant1") + + executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api) + result = executor.get_task_runtime_history("task1") + + self.assertEqual(result, []) + + +TESTS_BY_TASK = { + "task1": { + "resmoke_args": "--suites=suite1", + "tests": ["jstests/test1.js", "jstests/test2.js"]}, + "task2": { + "resmoke_args": "--suites=suite1", + "tests": ["jstests/test1.js", "jstests/test3.js"]}, + "task3": { + "resmoke_args": "--suites=suite3", + "tests": ["jstests/test4.js", "jstests/test5.js"]}, + "task4": { + "resmoke_args": "--suites=suite4", "tests": []}, +} # yapf: disable + + +def create_tests_by_task_mock(n_tasks, n_tests): + return { + f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[ + f"jstests/tests_{j}" for j in range(n_tests) + ], require_multiversion_setup=False, distro=f"distro_{i}", build_variant="variant", + suite=f"suite_{i}") + for i in range(n_tasks) + } + + +class TestCreateGenerateTasksConfig(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_no_tasks_given(self): + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant") + repeat_config = MagicMock() + mock_evg_api = MagicMock() + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant({}, build_variant) + + evg_config_dict = build_variant.as_dict() + self.assertEqual(0, len(evg_config_dict["tasks"])) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_one_task_one_test(self): + n_tasks = 1 + n_tests = 1 + resmoke_options = "options for resmoke" + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant", distro=None) + repeat_config = MagicMock() + repeat_config.generate_resmoke_options.return_value = resmoke_options + mock_evg_api = MagicMock() + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant(tests_by_task, build_variant) + + shrub_config = ShrubProject.empty().add_build_variant(build_variant) + evg_config_dict = shrub_config.as_dict() + tasks = evg_config_dict["tasks"] + self.assertEqual(n_tasks * n_tests, len(tasks)) + cmd = tasks[0]["commands"] + self.assertIn(resmoke_options, cmd[2]["vars"]["resmoke_args"]) + self.assertEqual("suite_0", cmd[2]["vars"]["suite"]) + self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"]) + + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + def test_n_task_m_test(self): + n_tasks = 3 + n_tests = 5 + build_variant = BuildVariant("build variant") + gen_config = MagicMock(run_build_variant="variant", distro=None) + repeat_config = MagicMock() + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + mock_evg_api = MagicMock() + + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api) + executor.generate_tasks_for_variant(tests_by_task, build_variant) + + evg_config_dict = build_variant.as_dict() + self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"])) + + +class TestCreateGenerateTasksFile(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") + @patch(ns("sys.exit")) + @patch(ns("validate_task_generation_limit")) + def test_cap_on_task_generate(self, validate_mock, exit_mock): + gen_config = MagicMock(require_multiversion_setup=False) + repeat_config = MagicMock() + tests_by_task = MagicMock() + mock_evg_api = MagicMock() + + validate_mock.return_value = False + + exit_mock.side_effect = ValueError("exiting") + with self.assertRaises(ValueError): + executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api, + "gen_file.json") + executor.execute(tests_by_task) + + exit_mock.assert_called_once() + + +class TestFindChangedTests(unittest.TestCase): + def test_manual_tests_should_be_specifiable_via_env_vars(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_repo = MagicMock() + mock_env = { + "BURN_IN_TESTS": "jstests/auth/auth1.js,jstests/core/where1.js", + } + change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) + + test_set = change_detector.find_changed_tests([mock_repo]) + + self.assertEqual(2, len(test_set)) + self.assertIn("jstests/auth/auth1.js", test_set) + self.assertIn("jstests/core/where1.js", test_set) + + def test_empty_env_should_not_add_extra_tests(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_repo = MagicMock() + mock_env = {} + change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env) + + test_set = change_detector.find_changed_tests([mock_repo]) + + self.assertEqual(set(), test_set) diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py new file mode 100644 index 00000000000..7ece404ab36 --- /dev/null +++ b/buildscripts/tests/test_evergreen_gen_build_variant.py @@ -0,0 +1,382 @@ +"""Unit tests for the generate_resmoke_suite script.""" +import unittest + +from mock import MagicMock + +from buildscripts import evergreen_gen_build_variant as under_test +from buildscripts.ciconfig.evergreen import Variant, Task + +# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access +# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter + + +def build_mock_build_variant(expansions=None, task_list=None): + task_spec_list = [{"name": task.name} for task in task_list] if task_list else [] + config = { + "tasks": task_spec_list, + } + if expansions: + config["expansions"] = expansions + + if task_list is None: + task_list = [] + task_map = {task.name: task for task in task_list} + + return Variant(config, task_map, {}) + + +def build_mock_task(name, run_vars=None, depends_on=None): + config = { + "name": + name, "commands": [ + {"func": "do setup"}, + { + "func": "generate resmoke tasks", + "vars": run_vars if run_vars else {}, + }, + ] + } + + if depends_on is not None: + config["depends_on"] = depends_on + return Task(config) + + +def build_mock_project_config(variant=None, task_defs=None): + mock_project = MagicMock() + if variant: + mock_project.get_variant.return_value = variant + + if task_defs: + mock_project.get_task.side_effect = task_defs + + return mock_project + + +def build_mock_expansions(): + mock_expansions = MagicMock() + mock_expansions.config_location.return_value = "/path/to/config" + mock_expansions.get_max_sub_suites.return_value = 998 + mock_expansions.task_name = "generating_task" + return mock_expansions + + +def build_mock_evg_api(build_task_list): + mock_evg_api = MagicMock() + mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list + return mock_evg_api + + +def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None): + if build_expansions is None: + build_expansions = {} + if task_def_list is None: + task_def_list = [] + if build_task_list is None: + build_task_list = [] + + mock_build_variant = build_mock_build_variant(build_expansions, task_def_list) + mock_project = build_mock_project_config(mock_build_variant, task_def_list) + mock_evg_expansions = build_mock_expansions() + mock_evg_api = build_mock_evg_api(build_task_list) + + return under_test.GenerateBuildVariantOrchestrator( + gen_task_validation=MagicMock(), + gen_task_options=MagicMock(), + evg_project_config=mock_project, + evg_expansions=mock_evg_expansions, + evg_api=mock_evg_api, + ) + + +class TestEvgExpansions(unittest.TestCase): + def test_get_max_sub_suites_should_use_patch_value_in_patches(self): + evg_expansions = under_test.EvgExpansions( + is_patch=True, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), evg_expansions.max_sub_suites) + + def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self): + evg_expansions = under_test.EvgExpansions( + is_patch=False, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), + evg_expansions.mainline_max_sub_suites) + + def test_get_max_sub_suites_should_use_mainline_value_if_patch_status_unknown(self): + evg_expansions = under_test.EvgExpansions( + is_patch=None, + max_sub_suites=5, + mainline_max_sub_suites=1, + build_id="build_id", + build_variant="build_variant", + project="project", + revision="revision", + task_name="task_name", + task_id="task_id", + ) + + self.assertEqual(evg_expansions.get_max_sub_suites(), + evg_expansions.mainline_max_sub_suites) + + +class TestTranslateRunVar(unittest.TestCase): + def test_normal_value_should_be_returned(self): + run_var = "some value" + mock_build_variant = build_mock_build_variant() + self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant(self): + run_var = "${my_expansion}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_none(self): + run_var = "${my_expansion}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_not_found_should_return_default(self): + run_var = "${my_expansion|default}" + mock_build_variant = build_mock_build_variant(expansions={}) + self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant)) + + def test_expansion_should_be_returned_from_build_variant_even_with_default(self): + run_var = "${my_expansion|default}" + value = "my value" + mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value}) + self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant)) + + +class TestTaskDefToSplitParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("my_task", split_param.suite_name) + self.assertEqual("my_task", split_param.filename) + + def test_params_should_allow_suite_to_be_overridden(self): + run_vars = { + "resmoke_args": "run tests", + "suite": "the suite", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant") + + self.assertEqual("build_variant", split_param.build_variant) + self.assertEqual("my_task", split_param.task_name) + self.assertEqual("the suite", split_param.suite_name) + self.assertEqual("the suite", split_param.filename) + + +class TestDetermineTaskDependencies(unittest.TestCase): + def test_running_task_should_not_be_included_in_depends_on(self): + run_vars = { + "resmoke_args": "run tests", + } + depends_on = [ + {"name": "compile"}, + {"name": "build_variant_gen"}, + ] + mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def) + + self.assertIn("compile", dependencies) + self.assertNotIn("generating_task", dependencies) + + +class TestTaskDefToGenParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "resmoke_args": "run tests", + } + mock_task_def = build_mock_task("my_task", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertFalse(gen_params.require_multiversion_setup) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertIsNone(gen_params.large_distro_name) + self.assertFalse(gen_params.use_large_distro) + + def test_params_should_be_overwritable(self): + run_vars = { + "resmoke_args": "run tests", + "use_large_distro": "true", + } + mock_task_def = build_mock_task("my_task", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant") + + self.assertFalse(gen_params.require_multiversion_setup) + self.assertEqual("run tests", gen_params.resmoke_args) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + gen_params.config_location) + self.assertEqual("my large distro", gen_params.large_distro_name) + self.assertTrue(gen_params.use_large_distro) + + +class TestTaskDefToFuzzerParams(unittest.TestCase): + def test_params_should_be_generated(self): + run_vars = { + "num_files": "5", + "num_tasks": "3", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(5, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("jstestfuzz", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertIsNone(fuzzer_params.large_distro_name) + self.assertFalse(fuzzer_params.use_large_distro) + + def test_num_tasks_respects_max_sub_suites(self): + run_vars = { + "num_files": "5", + "num_tasks": "3", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def]) + mock_orchestrator.evg_expansions.get_max_sub_suites.return_value = 1 + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual(5, fuzzer_params.num_files) + self.assertEqual(1, fuzzer_params.num_tasks) + + def test_params_should_be_overwritable(self): + run_vars = { + "num_files": "${file_count|8}", + "num_tasks": "3", + "use_large_distro": "true", + "npm_command": "aggfuzzer", + } + mock_task_def = build_mock_task("my_fuzzer_gen", run_vars) + build_expansions = {"large_distro_name": "my large distro"} + mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions, + task_def_list=[mock_task_def]) + + fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant") + + self.assertEqual("my_fuzzer", fuzzer_params.task_name) + self.assertEqual(8, fuzzer_params.num_files) + self.assertEqual(3, fuzzer_params.num_tasks) + self.assertEqual("aggfuzzer", fuzzer_params.npm_command) + self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value, + fuzzer_params.config_location) + self.assertEqual("my large distro", fuzzer_params.large_distro_name) + self.assertTrue(fuzzer_params.use_large_distro) + + +class TestGenerateBuildVariant(unittest.TestCase): + def test_a_whole_build_variant(self): + gen_run_vars = { + "resmoke_args": "run tests", + } + mv_gen_run_vars = { + "resmoke_args": "run tests", + "suite": "some suite", + } + fuzz_run_vars = { + "num_files": "5", + "num_tasks": "3", + "is_jstestfuzz": "true", + } + mv_fuzz_run_vars = { + "num_files": "5", + "num_tasks": "3", + "is_jstestfuzz": "true", + "suite": "aggfuzzer", + } + mock_task_defs = [ + build_mock_task("my_gen_task", gen_run_vars), + build_mock_task("my_fuzzer_task", fuzz_run_vars), + build_mock_task("my_mv_fuzzer_task", mv_fuzz_run_vars), + build_mock_task("my_mv_gen_task", mv_gen_run_vars), + ] + mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs) + builder = MagicMock() + + builder = mock_orchestrator.generate_build_variant(builder, "build variant") + + self.assertEqual(builder.generate_suite.call_count, 2) + self.assertEqual(builder.generate_fuzzer.call_count, 2) + + +class TestAdjustTaskPriority(unittest.TestCase): + def test_task_is_updates(self): + starting_priority = 42 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority + 1) + + def test_task_should_only_reach_99(self): + starting_priority = 99 + task_id = "task 314" + mock_task = MagicMock(task_id=task_id, priority=starting_priority) + mock_orchestrator = build_mock_orchestrator() + + mock_orchestrator.adjust_task_priority(mock_task) + + mock_orchestrator.evg_api.configure_task.assert_called_with(task_id, + priority=starting_priority) + + +class TestAdjustGenTasksPriority(unittest.TestCase): + def test_gen_tasks_in_task_list_are_adjusted(self): + gen_tasks = {"task_3", "task_8", "task_13"} + n_build_tasks = 25 + mock_task_list = [ + MagicMock(build_variant='dummy_variant', display_name=f"task_{i}", priority=0) + for i in range(n_build_tasks) + ] + mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list) + + n_tasks_adjusted = mock_orchestrator.adjust_gen_tasks_priority(gen_tasks) + + self.assertEqual(len(gen_tasks), n_tasks_adjusted) diff --git a/buildscripts/tests/test_evergreen_task_timeout.py b/buildscripts/tests/test_evergreen_task_timeout.py index 294c50d0a17..b3e4e201d96 100644 --- a/buildscripts/tests/test_evergreen_task_timeout.py +++ b/buildscripts/tests/test_evergreen_task_timeout.py @@ -110,7 +110,7 @@ class TestTimeoutOverrides(unittest.TestCase): class TestDetermineExecTimeout(unittest.TestCase): def _validate_exec_timeout(self, idle_timeout, exec_timeout, historic_timeout, evg_alias, - build_variant, display_name, timeout_override, expected_timeout): + build_variant, timeout_override, expected_timeout): task_name = "task_name" variant = build_variant overrides = {} @@ -121,9 +121,8 @@ class TestDetermineExecTimeout(unittest.TestCase): orchestrator = under_test.TaskTimeoutOrchestrator( timeout_service=MagicMock(spec_set=TimeoutService), - timeout_overrides=mock_timeout_overrides, evg_project_config=MagicMock( - spec_set=EvergreenProjectConfig, - get_variant=MagicMock(return_value=MagicMock(display_name=display_name)))) + timeout_overrides=mock_timeout_overrides, + evg_project_config=MagicMock(spec_set=EvergreenProjectConfig)) actual_timeout = orchestrator.determine_exec_timeout( task_name, variant, idle_timeout, exec_timeout, evg_alias, historic_timeout) @@ -133,83 +132,78 @@ class TestDetermineExecTimeout(unittest.TestCase): def test_timeout_used_if_specified(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=42), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, - expected_timeout=timedelta(seconds=42)) + timeout_override=None, expected_timeout=timedelta(seconds=42)) def test_default_is_returned_with_no_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, - evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, + evg_alias=None, build_variant="variant", timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_is_returned_with_timeout_at_zero(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, + timeout_override=None, expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT) def test_default_required_returned_on_required_variants(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - display_name="! required", timeout_override=None, + timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) def test_override_on_required_should_use_override(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=None, build_variant="variant-required", - display_name="! required", timeout_override=3 * 60, + timeout_override=3 * 60, expected_timeout=timedelta(minutes=3 * 60)) def test_task_specific_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0), historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=60, - expected_timeout=timedelta(minutes=60)) + timeout_override=60, expected_timeout=timedelta(minutes=60)) def test_commit_queue_items_use_commit_queue_timeout(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None, evg_alias=under_test.COMMIT_QUEUE_ALIAS, - build_variant="variant", display_name="not required", - timeout_override=None, + build_variant="variant", timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_use_idle_timeout_if_greater_than_exec_timeout(self): self._validate_exec_timeout( idle_timeout=timedelta(hours=2), exec_timeout=timedelta(minutes=10), - historic_timeout=None, evg_alias=None, build_variant="variant", - display_name="not required", timeout_override=None, expected_timeout=timedelta(hours=2)) + historic_timeout=None, evg_alias=None, build_variant="variant", timeout_override=None, + expected_timeout=timedelta(hours=2)) def test_historic_timeout_should_be_used_if_given(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", display_name="not required", - timeout_override=None, expected_timeout=timedelta(minutes=15)) + build_variant="variant", timeout_override=None, + expected_timeout=timedelta(minutes=15)) def test_commit_queue_should_override_historic_timeouts(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), - evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", - display_name="not required", timeout_override=None, + evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", timeout_override=None, expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT) def test_override_should_override_historic_timeouts(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant", display_name="not required", - timeout_override=33, expected_timeout=timedelta(minutes=33)) + build_variant="variant", timeout_override=33, + expected_timeout=timedelta(minutes=33)) def test_historic_timeout_should_not_be_overridden_by_required_bv(self): self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15), evg_alias=None, - build_variant="variant-required", display_name="! required", - timeout_override=None, expected_timeout=timedelta(minutes=15)) + build_variant="variant-required", timeout_override=None, + expected_timeout=timedelta(minutes=15)) def test_historic_timeout_should_not_be_increase_required_bv_timeout(self): self._validate_exec_timeout( idle_timeout=None, exec_timeout=None, historic_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT + timedelta(minutes=30), - evg_alias=None, build_variant="variant-required", display_name="! required", - timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) + evg_alias=None, build_variant="variant-required", timeout_override=None, + expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT) class TestDetermineIdleTimeout(unittest.TestCase): diff --git a/buildscripts/tests/test_jepsen_report.py b/buildscripts/tests/test_jepsen_report.py deleted file mode 100644 index cdfb095ba5a..00000000000 --- a/buildscripts/tests/test_jepsen_report.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Tests for jepsen report generator.""" -import unittest -import textwrap -import random -import os -from unittest.mock import patch, mock_open, MagicMock -from click.testing import CliRunner - -from buildscripts.jepsen_report import parse, ParserOutput, main - -_CORPUS = textwrap.dedent("""\ - "indeterminate: Command failed with error 251 (NoSuchTransaction): 'Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1' on server n9:27017. The full response is {\"writeConcernError\": {\"code\": 6, \"codeName\": \"HostUnreachable\", \"errmsg\": \"operation was interrupted\", \"errInfo\": {\"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"clientSupplied\"}}}, \"topologyVersion\": {\"processId\": {\"$oid\": \"625f0b0ef9d6a12d9b562ff9\"}, \"counter\": 21}, \"ok\": 0.0, \"errmsg\": \"Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1\", \"code\": 251, \"codeName\": \"NoSuchTransaction\", \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 16}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"operationTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 5}}, \"recoveryToken\": {\"recoveryShardId\": \"rs_shard1\"}}", - :index 1141}})}, - :workload {:valid? true}, - :valid? true} - - -Everything looks good! ヽ(‘ー`)ノ - - - -# Successful tests - -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T163539.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164724.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165317.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165910.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T170503.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171055.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171648.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172239.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172832.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T173634.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174227.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174820.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T175413.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180006.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180558.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181148.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181946.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T182539.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183720.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184313.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184906.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T185459.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190052.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190645.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191239.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191833.000Z - -# Crashed tests - -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition - -28 successes -0 unknown -2 crashed -0 failures -""").splitlines() - - -class TestParser(unittest.TestCase): - """TestParser.""" - - @classmethod - def _corpus_generator(cls): - n_pass = random.randint(1, 30) - n_fail = random.randint(1, 30) - n_unknown = random.randint(1, 30) - n_crash = random.randint(1, 30) - - corpus = textwrap.dedent("""\ - :index 2893}})}, - :workload {:valid? true}, - :valid? true} -Everything looks good! ヽ(‘ー`)ノ -""") - successful_tests = [] - if n_pass > 0: - corpus += "\n# Successful tests\n\n" - for i in range(0, n_pass): - corpus += f"test {i}\n" - successful_tests.append(f"test {i}") - - failed_tests = [] - if n_fail > 0: - corpus += "\n# Failed tests\n\n" - for i in range(0, n_fail): - corpus += f"test {i}\n" - failed_tests.append(f"test {i}") - - indeterminate_tests = [] - if n_unknown > 0: - corpus += "\n# Indeterminate tests\n\n" - for i in range(0, n_unknown): - corpus += f"test {i}\n" - indeterminate_tests.append(f"test {i}") - - crashed_tests = [] - if n_crash > 0: - corpus += "\n# Crashed tests\n\n" - for i in range(0, n_crash): - corpus += f"test {i}\n" - crashed_tests.append(f"test {i}") - # note leading newline for this block is required to match the actual - # logs - corpus += textwrap.dedent(f""" -{n_pass} successes -{n_unknown} unknown -{n_crash} crashed -{n_fail} failures -""") - - return { - 'expected': - ParserOutput({ - 'success': successful_tests, - 'unknown': indeterminate_tests, - 'crashed': crashed_tests, - 'failed': failed_tests, - }), 'corpus': - corpus - } - - def test_parser(self): - """Test with embedded corpus.""" - out = parse(_CORPUS) - self.assertEqual(len(out['success']), 28) - self.assertEqual(len(out['unknown']), 0) - self.assertEqual(len(out['crashed']), 2) - self.assertEqual(len(out['failed']), 0) - - def test_parser2(self): - """Test with jepsen.log file.""" - with open(os.path.join(os.path.dirname(__file__), - "test_jepsen_report_corpus.log.txt")) as fh: - corpus = fh.read().splitlines() - out = parse(corpus) - self.assertEqual(len(out['success']), 29) - self.assertEqual(len(out['unknown']), 0) - self.assertEqual(len(out['crashed']), 1) - self.assertEqual(len(out['failed']), 0) - - def test_generated_corpus(self): - """Generate 100 corpuses and test them.""" - for _ in range(0, 100): - self._test_generated_corpus() - - def _test_generated_corpus(self): - gen = self._corpus_generator() - corpus = gen['corpus'].splitlines() - out = parse(corpus) - self.assertDictEqual(out, gen['expected']) - - @patch('buildscripts.jepsen_report._try_find_log_file') - @patch('buildscripts.jepsen_report._get_log_lines') - @patch('buildscripts.jepsen_report._put_report') - def test_main(self, mock_put_report, mock_get_log_lines, mock_try_find_log_file): - """Test main function.""" - gen = self._corpus_generator() - corpus = gen['corpus'].splitlines() - mock_get_log_lines.return_value = corpus - - def _try_find_log_file(_store, _test): - if _try_find_log_file.counter == 0: - _try_find_log_file.counter += 1 - with open( - os.path.join( - os.path.dirname(__file__), "test_jepsen_report_corpus.log.txt")) as fh: - return fh.read() - return "" - - _try_find_log_file.counter = 0 - mock_try_find_log_file.side_effect = _try_find_log_file - - runner = CliRunner() - result = runner.invoke(main, - ["--start_time=0", "--end_time=10", "--elapsed=10", "test.log"]) - num_tests = len(gen['expected']['success']) + len(gen['expected']['unknown']) + len( - gen['expected']['crashed']) + len(gen['expected']['failed']) - num_fails = num_tests - len(gen['expected']['success']) - - callee_dict = mock_put_report.call_args[0][0] - self.assertEqual(callee_dict['failures'], num_fails) - self.assertEqual(len(callee_dict['results']), num_tests) - mock_get_log_lines.assert_called_once_with('test.log') - if gen['expected']['crashed']: - self.assertEqual(result.exit_code, 2) - elif gen['expected']['unknown'] or gen['expected']['failure']: - self.assertEqual(result.exit_code, 1) - else: - self.assertEqual(result.exit_code, 0) diff --git a/buildscripts/tests/test_jepsen_report_corpus.log.txt b/buildscripts/tests/test_jepsen_report_corpus.log.txt deleted file mode 100644 index 89d4e91892b..00000000000 --- a/buildscripts/tests/test_jepsen_report_corpus.log.txt +++ /dev/null @@ -1,67 +0,0 @@ -INFO [2022-04-20 02:46:34,505] jepsen test runner - jepsen.core {:perf - {:latency-graph {:valid? true}, - :rate-graph {:valid? true}, - :valid? true}, - :clock {:valid? true}, - :stats - {:valid? true, - :count 5768, - :ok-count 4930, - :fail-count 0, - :info-count 838, - :by-f - {:txn - {:valid? true, - :count 5768, - :ok-count 4930, - :fail-count 0, - :info-count 838}}}, - :exceptions {:valid? true}, - :workload {:valid? true}, - :valid? true} - - -Everything looks good! ヽ(‘ー`)ノ - - - -# Successful tests - -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235350.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235940.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T000533.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001131.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001724.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002316.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002909.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T003502.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004057.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004651.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005244.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005837.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T010429.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011021.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011617.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012210.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012802.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T013356.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014154.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014746.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015338.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015931.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T020525.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021116.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021708.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022301.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022854.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T023446.000Z -store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T024039.000Z - -# Crashed tests - -mongodb list-append w:majority r:majority tw:majority tr:snapshot partition - -29 successes -0 unknown -1 crashed -0 failures diff --git a/buildscripts/tests/test_mongosymb.py b/buildscripts/tests/test_mongosymb.py deleted file mode 100644 index 188ea385c5f..00000000000 --- a/buildscripts/tests/test_mongosymb.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Unit tests for buildscripts/mongosymb.py.""" -# pylint: disable=missing-docstring -import unittest - -from buildscripts import mongosymb as under_test - - -class TestGetVersion(unittest.TestCase): - def test_get_version_with_patch(self): - trace_doc = { - "processInfo": { - "mongodbVersion": "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169" - } - } - version = under_test.get_version(trace_doc) - self.assertEqual(version, "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169") - - def test_get_version_without_patch(self): - trace_doc = {"processInfo": {"mongodbVersion": "6.1.0-alpha-504-g0c8a142"}} - version = under_test.get_version(trace_doc) - self.assertEqual(version, "6.1.0-alpha-504-g0c8a142") - - def test_get_version_no_mongodb_version(self): - trace_doc = {"processInfo": {}} - version = under_test.get_version(trace_doc) - self.assertEqual(version, None) - - def test_get_version_no_process_info(self): - trace_doc = {} - version = under_test.get_version(trace_doc) - self.assertEqual(version, None) - - -class TestHasHighNotFoundPathsRatio(unittest.TestCase): - def test_not_found_paths_ratio_is_more_than_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": "some/path"}, - {"path": "some/path"}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, False) - - def test_not_found_paths_ratio_is_equal_to_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": "some/path"}, - {"path": None}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, True) - - def test_not_found_paths_ratio_is_less_than_0_5(self): - frames = [ - {"path": "some/path"}, - {"path": None}, - {"path": None}, - {"path": None}, - ] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, True) - - def test_no_frames(self): - frames = [] - ret = under_test.has_high_not_found_paths_ratio(frames) - self.assertEqual(ret, False) diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 85725e60858..70689345b69 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -24,7 +24,7 @@ from buildscripts.task_generation.task_types.gentask_options import GenTaskOptio from buildscripts.tests.test_burn_in_tests import get_evergreen_config, mock_changed_git_files from buildscripts import selected_tests as under_test -# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter,too-many-locals +# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter NS = "buildscripts.selected_tests" @@ -101,9 +101,7 @@ class TestAcceptance(unittest.TestCase): self.assertEqual(generated_config.file_list[0].file_name, "selected_tests_config.json") @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_when_test_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] + def test_when_test_mappings_are_found_for_changed_files(self): mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( @@ -135,7 +133,7 @@ class TestAcceptance(unittest.TestCase): # assert that generated suite files have the suite name and the variant name in the # filename, to prevent tasks on different variants from using the same suite file - self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate) + self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate) generated_evg_config_raw = [ gen_file.content for gen_file in generated_config.file_list @@ -147,13 +145,11 @@ class TestAcceptance(unittest.TestCase): # jstests/auth/auth1.js belongs to two suites, auth and auth_audit, rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2) @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") - @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3") - def test_when_task_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock): - get_stats_from_s3_mock.return_value = [] + def test_when_task_mappings_are_found_for_changed_files(self): mock_evg_api = self._mock_evg_api() mock_evg_config = get_evergreen_config("etc/evergreen.yml") mock_evg_expansions = under_test.EvgExpansions( @@ -168,7 +164,7 @@ class TestAcceptance(unittest.TestCase): mock_task_mapping = TaskMapping( branch="master", project="mongodb-mongo-master", repo="mongodb/mongo", source_file="src/file1.cpp", source_file_seen_count=8, - tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-8", flip_count=5)]) + tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)]) mock_selected_tests_client = MagicMock() mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse( task_mappings=[mock_task_mapping]) @@ -194,7 +190,7 @@ class TestAcceptance(unittest.TestCase): build_variants_with_generated_tasks = generated_evg_config["buildvariants"] rhel_80_with_generated_tasks = next( (variant for variant in build_variants_with_generated_tasks - if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None) + if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None) self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5) @@ -404,6 +400,13 @@ class TestGetTaskConfigsForTaskMappings(unittest.TestCase): class TestRemoveRepoPathPrefix(unittest.TestCase): - def test_path_handling(self): - filepath = under_test._remove_repo_path_prefix("sample_directory/src/file1.cpp") - self.assertEqual(filepath, "sample_directory/src/file1.cpp") + def test_file_is_in_enterprise_modules(self): + filepath = under_test._remove_repo_path_prefix( + "src/mongo/db/modules/enterprise/src/file1.cpp") + + self.assertEqual(filepath, "src/file1.cpp") + + def test_file_is_not_in_enterprise_modules(self): + filepath = under_test._remove_repo_path_prefix("other_directory/src/file1.cpp") + + self.assertEqual(filepath, "other_directory/src/file1.cpp") diff --git a/buildscripts/tests/test_validate_commit_message.py b/buildscripts/tests/test_validate_commit_message.py index 1bf0039188d..db341a5add1 100644 --- a/buildscripts/tests/test_validate_commit_message.py +++ b/buildscripts/tests/test_validate_commit_message.py @@ -1,31 +1,130 @@ """Unit tests for the evergreen_task_timeout script.""" +import itertools import unittest +from typing import List, Optional +from unittest.mock import MagicMock -from buildscripts.validate_commit_message import main, STATUS_OK, STATUS_ERROR +import buildscripts.validate_commit_message as under_test +from buildscripts.client.jiraclient import JiraClient, SecurityLevel +from evergreen import EvergreenApi # pylint: disable=missing-docstring,no-self-use +INVALID_MESSAGES = [ + "", # You must provide a message + "RevertEVG-1", # revert and ticket must be formatted + "revert EVG-1", # revert must be capitalized + "This is not a valid message", # message must be valid + "Fix Lint", # Fix lint is strict in terms of caps +] + + +def create_mock_code_change(code_change_messages: List[str], branch_name: Optional[str] = None): + mock_code_change = MagicMock( + commit_messages=code_change_messages, + branch_name=branch_name if branch_name else "mongodb-mongo-master", + ) + return mock_code_change + + +def create_mock_patch(code_change_messages: List[str], branch_name: Optional[str] = None): + mock_code_change = create_mock_code_change(code_change_messages, branch_name) + mock_patch = MagicMock(module_code_changes=[mock_code_change]) + return mock_patch + + +def create_mock_evg_client(code_change_messages: List[str], + branch_name: Optional[str] = None) -> MagicMock: + mock_patch = create_mock_patch(code_change_messages, branch_name) + + mock_evg_client = MagicMock(spec_set=EvergreenApi) + mock_evg_client.patch_by_id.return_value = mock_patch + return mock_evg_client + + +def create_mock_jira_client(): + mock_jira = MagicMock(spec_set=JiraClient) + mock_jira.get_ticket_security_level.return_value = SecurityLevel.NONE + return mock_jira + + +def interleave_new_format(older): + """Create a new list containing a new and old format copy of each string.""" + newer = [ + f"Commit Queue Merge: '{old}' into 'mongodb/mongo:SERVER-45949-validate-message-format'" + for old in older + ] + return list(itertools.chain(*zip(older, newer))) + class ValidateCommitMessageTest(unittest.TestCase): - def test_valid(self): + def test_valid_commits(self): messages = [ + "Fix lint", + "EVG-1", # Test valid projects with various number lengths + "SERVER-20", + "WT-300", "SERVER-44338", - "Revert \"SERVER-60", + "Revert EVG-5", + "Revert SERVER-60", + "Revert WT-700", + "Revert 'SERVER-8000", + 'Revert "SERVER-90000', "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", + "Import tools: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4", 'Revert "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4"', ] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_OK) - self.assertTrue(all(main([message]) == STATUS_OK for message in messages)) + def test_private(self): + messages = ["XYZ-1"] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - def test_invalid(self): + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_private_with_public(self): messages = [ - "SERVER-", # missing number - "Revert SERVER-60", # missing quote before SERVER - "", # empty value - "nonsense", # nonsense values + "Fix lint", + "EVG-1", # Test valid projects with various number lengths + "SERVER-20", + "XYZ-1", ] + mock_evg_api = create_mock_evg_client(interleave_new_format(messages)) + mock_jira = create_mock_jira_client() + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_internal_ticket_to_public_repo_should_fail(self): + message = "SERVER-20" + mock_evg_api = create_mock_evg_client(interleave_new_format([message])) + mock_jira = create_mock_jira_client() + mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) + + is_valid = orchestrator.validate_commit_messages("version_id") + + self.assertEqual(is_valid, under_test.STATUS_ERROR) + + def test_internal_ticket_to_private_repo_should_succeed(self): + message = "SERVER-20" + mock_evg_api = create_mock_evg_client(interleave_new_format([message]), "private-repo") + mock_jira = create_mock_jira_client() + mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL + orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira) - self.assertTrue(all(main([message]) == STATUS_ERROR for message in messages)) + is_valid = orchestrator.validate_commit_messages("version_id") - def test_message_is_empty_list(self): - self.assertEqual(main([]), STATUS_ERROR) + self.assertEqual(is_valid, under_test.STATUS_OK) diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py index 23e9c1f331f..bb0dd8a0c3e 100644 --- a/buildscripts/tests/timeouts/test_timeout_service.py +++ b/buildscripts/tests/timeouts/test_timeout_service.py @@ -1,42 +1,41 @@ """Unit tests for timeout_service.py.""" import random import unittest -from unittest.mock import MagicMock, patch +from datetime import datetime, timedelta +from unittest.mock import MagicMock from requests.exceptions import HTTPError +from evergreen import EvergreenApi import buildscripts.timeouts.timeout_service as under_test from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService -from buildscripts.util.teststats import HistoricTaskData, HistoricTestInfo +from buildscripts.util.teststats import HistoricTaskData # pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access -NS = "buildscripts.timeouts.timeout_service" - -def ns(relative_name): # pylint: disable=invalid-name - """Return a full name from a name relative to the test module"s name space.""" - return NS + "." + relative_name - - -def build_mock_service(resmoke_proxy=None): +def build_mock_service(evg_api=None, resmoke_proxy=None): + end_date = datetime.now() + start_date = end_date - timedelta(weeks=2) + timeout_settings = under_test.TimeoutSettings( + end_date=end_date, + start_date=start_date, + ) return under_test.TimeoutService( - resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService)) + evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi), + resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService), + timeout_settings=timeout_settings) def tst_stat_mock(file, duration, pass_count): - return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count, hooks=[]) - - -def tst_runtime_mock(file, duration, pass_count): - return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count) + return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count) class TestGetTimeoutEstimate(unittest.TestCase): - @patch(ns("HistoricTaskData.from_s3")) - def test_no_stats_should_return_default_timeout(self, from_s3_mock: MagicMock): - timeout_service = build_mock_service() - from_s3_mock.return_value = [] + def test_no_stats_should_return_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.return_value = [] + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -49,19 +48,13 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_too_many_tests_missing_history_should_cause_a_default_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(23) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_a_test_with_missing_history_should_cause_a_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - test_names = [ts.test_name for ts in test_stats] - test_names.extend([f"test_with_no_stats_{i}.js" for i in range(7)]) - mock_resmoke_proxy.list_tests.return_value = test_names - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -74,21 +67,14 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_too_many_tests_with_zero_runtime_history_should_cause_a_default_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(23) - ] - test_stats.extend([ - HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[]) - for i in range(7) - ]) - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)] + test_stats.append(tst_stat_mock("zero.js", 0.0, 1)) + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -101,73 +87,15 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertFalse(timeout.is_specified()) - @patch(ns("HistoricTaskData.from_s3")) - def test_enough_history_but_some_tests_missing_history_should_cause_custom_task_and_default_test_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(25) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) - mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - test_names = [ts.test_name for ts in test_stats] - test_names.extend([f"test_with_no_stats_{i}.js" for i in range(5)]) - mock_resmoke_proxy.list_tests.return_value = test_names - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) - timeout_params = under_test.TimeoutParams( - evg_project="my project", - build_variant="bv", - task_name="my task", - suite_name="my suite", - is_asan=False, - ) - - timeout = timeout_service.get_timeout_estimate(timeout_params) - - self.assertTrue(timeout.is_specified()) - self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) - - @patch(ns("HistoricTaskData.from_s3")) - def test_enough_history_but_some_tests_with_zero_runtime_should_cause_custom_task_and_default_test_timeout( - self, from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(25) - ] - test_stats.extend([ - HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[]) - for i in range(5) - ]) - from_s3_mock.return_value = HistoricTaskData(test_stats) - mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) - timeout_params = under_test.TimeoutParams( - evg_project="my project", - build_variant="bv", - task_name="my task", - suite_name="my suite", - is_asan=False, - ) - - timeout = timeout_service.get_timeout_estimate(timeout_params) - - self.assertTrue(timeout.is_specified()) - self.assertEqual(None, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) - - @patch(ns("HistoricTaskData.from_s3")) - def test_all_tests_with_runtime_history_should_use_custom_timeout(self, - from_s3_mock: MagicMock): - test_stats = [ - HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[]) - for i in range(30) - ] - from_s3_mock.return_value = HistoricTaskData(test_stats) + def test_all_tests_with_runtime_history_should_use_custom_timeout(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + n_tests = 30 + test_runtime = 600 + test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)] + mock_evg_api.test_stats_by_project.return_value = test_stats mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService) - mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats] - timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy) + mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats] + timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -180,7 +108,7 @@ class TestGetTimeoutEstimate(unittest.TestCase): self.assertTrue(timeout.is_specified()) self.assertEqual(1860, timeout.calculate_test_timeout(1)) - self.assertEqual(54360, timeout.calculate_task_timeout(1)) + self.assertEqual(54180, timeout.calculate_task_timeout(1)) class TestGetTaskHookOverhead(unittest.TestCase): @@ -221,10 +149,10 @@ class TestGetTaskHookOverhead(unittest.TestCase): class TestLookupHistoricStats(unittest.TestCase): - @patch(ns("HistoricTaskData.from_s3")) - def test_no_stats_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): - from_s3_mock.return_value = None - timeout_service = build_mock_service() + def test_no_stats_from_evergreen_should_return_none(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.return_value = [] + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -237,10 +165,10 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - @patch(ns("HistoricTaskData.from_s3")) - def test_errors_from_evergreen_should_return_none(self, from_s3_mock: MagicMock): - from_s3_mock.side_effect = HTTPError("failed to connect") - timeout_service = build_mock_service() + def test_errors_from_evergreen_should_return_none(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) + mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect") + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -253,11 +181,11 @@ class TestLookupHistoricStats(unittest.TestCase): self.assertIsNone(stats) - @patch(ns("HistoricTaskData.from_s3")) - def test_stats_from_evergreen_should_return_the_stats(self, from_s3_mock: MagicMock): + def test_stats_from_evergreen_should_return_the_stats(self): + mock_evg_api = MagicMock(spec_set=EvergreenApi) test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)] - from_s3_mock.return_value = HistoricTaskData(test_stats) - timeout_service = build_mock_service() + mock_evg_api.test_stats_by_project.return_value = test_stats + timeout_service = build_mock_service(evg_api=mock_evg_api) timeout_params = under_test.TimeoutParams( evg_project="my project", build_variant="bv", @@ -328,34 +256,3 @@ class TestGetCleanEveryNCadence(unittest.TestCase): cadence = timeout_service._get_clean_every_n_cadence("suite", False) self.assertEqual(1, cadence) - - -class TestHaveEnoughHistoricStats(unittest.TestCase): - def test_should_return_true_when_number_of_tests_equals_zero(self): - timeout_service = build_mock_service() - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=0, num_tests_missing_data=0)) - - def test_should_return_true_when_number_of_tests_with_historic_data_more_than_threshold(self): - timeout_service = build_mock_service() - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=19)) - self.assertTrue( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=0)) - - def test_should_return_false_when_number_of_tests_with_historic_data_less_or_equal_to_threshold( - self): - timeout_service = build_mock_service() - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=20)) - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=21)) - self.assertFalse( - timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=100)) - - def test_exception_raised_when_number_of_tests_less_than_zero(self): - timeout_service = build_mock_service() - with self.assertRaises(ValueError): - timeout_service._have_enough_historic_stats(num_tests=-1, num_tests_missing_data=0) - with self.assertRaises(ValueError): - timeout_service._have_enough_historic_stats(num_tests=-100, num_tests_missing_data=0) diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py index 637ee59321e..7f3296ca1aa 100644 --- a/buildscripts/tests/util/test_taskname.py +++ b/buildscripts/tests/util/test_taskname.py @@ -28,7 +28,7 @@ class TestRemoveGenSuffix(unittest.TestCase): class TestDetermineTaskBaseName(unittest.TestCase): def test_task_name_with_build_variant_should_strip_bv_and_sub_task_index(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = f"auth_23_{bv}" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -36,7 +36,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_should_strip_sub_task_index(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = "auth_314" base_task_name = under_test.determine_task_base_name(task_name, bv) @@ -44,7 +44,7 @@ class TestDetermineTaskBaseName(unittest.TestCase): self.assertEqual("auth", base_task_name) def test_task_name_without_build_variant_or_subtask_index_should_self(self): - bv = "enterprise-rhel-8-64-bit-dynamic-required" + bv = "enterprise-rhel-80-64-bit-dynamic-required" task_name = "auth" base_task_name = under_test.determine_task_base_name(task_name, bv) diff --git a/buildscripts/tests/util/test_teststats.py b/buildscripts/tests/util/test_teststats.py index 9e72d87a737..ebba930d032 100644 --- a/buildscripts/tests/util/test_teststats.py +++ b/buildscripts/tests/util/test_teststats.py @@ -2,12 +2,8 @@ import datetime import unittest -from json import JSONDecodeError -from unittest.mock import patch from mock import Mock -from mock.mock import MagicMock -from requests import Session import buildscripts.util.teststats as under_test @@ -25,60 +21,6 @@ class NormalizeTestNameTest(unittest.TestCase): under_test.normalize_test_name("\\home\\user\\test.js")) -class TestHistoricTestInfo(unittest.TestCase): - def test_total_test_runtime_not_passing_test_no_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=0, - avg_duration=0.0, - hooks=[], - ) - - self.assertEqual(0.0, test_info.total_test_runtime()) - - def test_total_test_runtime_not_passing_test_with_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=0, - avg_duration=0.0, - hooks=[ - under_test.HistoricHookInfo( - hook_id='test:hook', - num_pass=10, - avg_duration=5.0, - ), - ], - ) - - self.assertEqual(0.0, test_info.total_test_runtime()) - - def test_total_test_runtime_passing_test_no_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=10, - avg_duration=23.0, - hooks=[], - ) - - self.assertEqual(23.0, test_info.total_test_runtime()) - - def test_total_test_runtime_passing_test_with_hooks(self): - test_info = under_test.HistoricTestInfo( - test_name='jstests/test.js', - num_pass=10, - avg_duration=23.0, - hooks=[ - under_test.HistoricHookInfo( - hook_id='test:hook', - num_pass=10, - avg_duration=5.0, - ), - ], - ) - - self.assertEqual(28.0, test_info.total_test_runtime()) - - class TestHistoricTaskData(unittest.TestCase): def test_no_hooks(self): evg_results = [ @@ -138,7 +80,7 @@ class TestHistoricTaskData(unittest.TestCase): @staticmethod def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0): return Mock( - test_name=test_file, + test_file=test_file, task_name="task1", variant="variant1", distro="distro1", @@ -147,53 +89,3 @@ class TestHistoricTaskData(unittest.TestCase): num_fail=0, avg_duration_pass=duration, ) - - @patch.object(Session, 'get') - def test_get_stats_from_s3_returns_data(self, mock_get): - mock_response = MagicMock() - mock_response.json.return_value = [ - { - "test_name": "jstests/noPassthroughWithMongod/geo_near_random1.js", - "num_pass": 74, - "num_fail": 0, - "avg_duration_pass": 23.16216216216216, - "max_duration_pass": 27.123, - }, - { - "test_name": "shell_advance_cluster_time:ValidateCollections", - "num_pass": 74, - "num_fail": 0, - "avg_duration_pass": 1.662162162162162, - "max_duration_pass": 100.0987, - }, - ] - mock_get.return_value = mock_response - - result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") - - self.assertEqual(result, [ - under_test.HistoricalTestInformation( - test_name="jstests/noPassthroughWithMongod/geo_near_random1.js", - num_pass=74, - num_fail=0, - avg_duration_pass=23.16216216216216, - max_duration_pass=27.123, - ), - under_test.HistoricalTestInformation( - test_name="shell_advance_cluster_time:ValidateCollections", - num_pass=74, - num_fail=0, - avg_duration_pass=1.662162162162162, - max_duration_pass=100.0987, - ), - ]) - - @patch.object(Session, 'get') - def test_get_stats_from_s3_json_decode_error(self, mock_get): - mock_response = MagicMock() - mock_response.json.side_effect = JSONDecodeError("msg", "doc", 0) - mock_get.return_value = mock_response - - result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant") - - self.assertEqual(result, []) |