summaryrefslogtreecommitdiff
path: root/buildscripts/tests
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /buildscripts/tests
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'buildscripts/tests')
-rw-r--r--buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp7
-rw-r--r--buildscripts/tests/resmoke_end2end/test_resmoke.py27
-rw-r--r--buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py32
-rw-r--r--buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py2
-rw-r--r--buildscripts/tests/resmokelib/test_parser.py1
-rw-r--r--buildscripts/tests/resmokelib/testing/test_symbolizer_service.py250
-rw-r--r--buildscripts/tests/task_generation/test_suite_split.py67
-rw-r--r--buildscripts/tests/test_burn_in_tags.py261
-rw-r--r--buildscripts/tests/test_burn_in_tags_evergreen.yml138
-rw-r--r--buildscripts/tests/test_burn_in_tests.py18
-rw-r--r--buildscripts/tests/test_debugsymb_mapper.py115
-rw-r--r--buildscripts/tests/test_errorcodes.py2
-rw-r--r--buildscripts/tests/test_evergreen_activate_gen_tasks.py116
-rw-r--r--buildscripts/tests/test_evergreen_burn_in_tests.py408
-rw-r--r--buildscripts/tests/test_evergreen_gen_build_variant.py382
-rw-r--r--buildscripts/tests/test_evergreen_task_timeout.py48
-rw-r--r--buildscripts/tests/test_jepsen_report.py195
-rw-r--r--buildscripts/tests/test_jepsen_report_corpus.log.txt67
-rw-r--r--buildscripts/tests/test_mongosymb.py68
-rw-r--r--buildscripts/tests/test_selected_tests.py31
-rw-r--r--buildscripts/tests/test_validate_commit_message.py123
-rw-r--r--buildscripts/tests/timeouts/test_timeout_service.py209
-rw-r--r--buildscripts/tests/util/test_taskname.py6
-rw-r--r--buildscripts/tests/util/test_teststats.py110
24 files changed, 1476 insertions, 1207 deletions
diff --git a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp
index 24b0b42fed3..834d2052a98 100644
--- a/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp
+++ b/buildscripts/tests/data/errorcodes/regex_matching/regex_matching.cpp
@@ -27,9 +27,4 @@ LOGV2_ERROR(25,
"more words");
LOGV2_ERROR(26,
"words",
- "comma, more words words words words words words words words words words words words ");
-iassert(27, "words");
-iasserted(28, "words");
-iassertNoTrace(29, "words");
-iassertedNoTrace(30, "words");
-MONGO_UNREACHABLE_TASSERT(31);
+ "comma, more words words words words words words words words words words words words "
diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py
index 0cba8c8d82f..82434ee25c4 100644
--- a/buildscripts/tests/resmoke_end2end/test_resmoke.py
+++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py
@@ -4,7 +4,6 @@ import logging
import json
import os
import os.path
-import subprocess
import sys
import time
import unittest
@@ -172,7 +171,7 @@ class TestTimeout(_ResmokeSelftest):
]
self.execute_resmoke(resmoke_args)
- archival_dirs_to_expect = 8 # 2 tests * 2 nodes
+ archival_dirs_to_expect = 4 # 2 tests * 2 nodes
self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo)
@@ -190,11 +189,9 @@ class TestTimeout(_ResmokeSelftest):
self.execute_resmoke(resmoke_args, sleep_secs=25)
- archival_dirs_to_expect = 4
+ archival_dirs_to_expect = 2 # 2 tests * 2 nodes / 2 data_file directories
self.assert_dir_file_count(self.test_dir, self.archival_file, archival_dirs_to_expect)
- archival_inner_dirs_to_expect = 2
- self.assert_dir_file_count(self.test_dir_inner, self.archival_file,
- archival_inner_dirs_to_expect)
+ self.assert_dir_file_count(self.test_dir_inner, self.archival_file, archival_dirs_to_expect)
analysis_pids_to_expect = 6 # 2 tests * (2 mongod + 1 mongo)
self.assert_dir_file_count(self.test_dir, self.analysis_file, analysis_pids_to_expect)
@@ -428,21 +425,3 @@ class TestSetParameters(_ResmokeSelftest):
"""--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}""",
"""--mongodSetParameter={"mirrorReads": {samplingRate: 1.0}}"""
]).wait())
-
-
-class TestMultiversionConfig(unittest.TestCase):
- def test_valid_yaml(self):
- file_name = "multiversion-config.yml"
- subprocess.run([
- sys.executable, "buildscripts/resmoke.py", "multiversion-config",
- "--config-file-output", file_name
- ], check=True)
- with open(file_name, "r") as file:
- file_contents = file.read()
-
- try:
- yaml.safe_load(file_contents)
- except Exception: # pylint: disable=broad-except
- self.fail(msg="`resmoke.py multiversion-config` does not output valid yaml.")
-
- os.remove(file_name)
diff --git a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py
index 274b7e5ffbe..2f685f36c97 100644
--- a/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py
+++ b/buildscripts/tests/resmokelib/multiversion/test_multiversion_service.py
@@ -38,8 +38,6 @@ class TestCalculateFcvConstants(TestCase):
"100.0"
],
"longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0"],
- "eolVersions":
- ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"],
})
multiversion_service = under_test.MultiversionService(
@@ -47,15 +45,15 @@ class TestCalculateFcvConstants(TestCase):
mongo_releases=mongo_releases,
)
- version_constants = multiversion_service.calculate_version_constants()
+ fcv_constants = multiversion_service.calculate_fcv_constants()
- self.assertEqual(version_constants.latest, Version("6.0"))
- self.assertEqual(version_constants.last_continuous, Version("5.3"))
- self.assertEqual(version_constants.last_lts, Version("5.0"))
- self.assertEqual(version_constants.requires_fcv_tag_list,
+ self.assertEqual(fcv_constants.latest, Version("6.0"))
+ self.assertEqual(fcv_constants.last_continuous, Version("5.3"))
+ self.assertEqual(fcv_constants.last_lts, Version("5.0"))
+ self.assertEqual(fcv_constants.requires_fcv_tag_list,
[Version(v) for v in ["5.1", "5.2", "5.3", "6.0"]])
- self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("6.0")])
- self.assertEqual(version_constants.fcvs_less_than_latest, [
+ self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("6.0")])
+ self.assertEqual(fcv_constants.fcvs_less_than_latest, [
Version(v)
for v in ["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3"]
])
@@ -69,8 +67,6 @@ class TestCalculateFcvConstants(TestCase):
"6.1", "100.0"
],
"longTermSupportReleases": ["4.0", "4.2", "4.4", "5.0", "6.0"],
- "eolVersions":
- ["2.0", "2.2", "2.4", "2.6", "3.0", "3.2", "3.4", "3.6", "4.0", "5.1", "5.2"],
})
multiversion_service = under_test.MultiversionService(
@@ -78,15 +74,15 @@ class TestCalculateFcvConstants(TestCase):
mongo_releases=mongo_releases,
)
- version_constants = multiversion_service.calculate_version_constants()
+ fcv_constants = multiversion_service.calculate_fcv_constants()
- self.assertEqual(version_constants.latest, Version("100.0"))
- self.assertEqual(version_constants.last_continuous, Version("6.1"))
- self.assertEqual(version_constants.last_lts, Version("6.0"))
- self.assertEqual(version_constants.requires_fcv_tag_list,
+ self.assertEqual(fcv_constants.latest, Version("100.0"))
+ self.assertEqual(fcv_constants.last_continuous, Version("6.1"))
+ self.assertEqual(fcv_constants.last_lts, Version("6.0"))
+ self.assertEqual(fcv_constants.requires_fcv_tag_list,
[Version(v) for v in ["6.1", "100.0"]])
- self.assertEqual(version_constants.requires_fcv_tag_list_continuous, [Version("100.0")])
- self.assertEqual(version_constants.fcvs_less_than_latest, [
+ self.assertEqual(fcv_constants.requires_fcv_tag_list_continuous, [Version("100.0")])
+ self.assertEqual(fcv_constants.fcvs_less_than_latest, [
Version(v) for v in
["4.0", "4.2", "4.4", "4.7", "4.8", "4.9", "5.0", "5.1", "5.2", "5.3", "6.0", "6.1"]
])
diff --git a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py
index ec98f233840..decc81f4433 100644
--- a/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py
+++ b/buildscripts/tests/resmokelib/run/test_generate_multiversion_exclude_tags.py
@@ -11,7 +11,6 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags as un
from buildscripts.util.fileops import read_yaml_file
EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml"
-EXPANSIONS_FILE = "expansions.yml"
class TestGenerateExcludeYaml(unittest.TestCase):
@@ -43,7 +42,6 @@ class TestGenerateExcludeYaml(unittest.TestCase):
output = os.path.join(self._tmpdir.name, EXCLUDE_TAGS_FILE)
under_test.generate_exclude_yaml(old_bin_version=old_bin_version, output=output,
- expansions_file=EXPANSIONS_FILE,
logger=MagicMock())
mock_read_yaml.assert_called_once()
diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py
index c1ba97c6056..b9fc74f088b 100644
--- a/buildscripts/tests/resmokelib/test_parser.py
+++ b/buildscripts/tests/resmokelib/test_parser.py
@@ -248,6 +248,7 @@ class TestLocalCommandLine(unittest.TestCase):
cmdline = to_local_args([
"run",
"--suites=my_suite",
+ "--reportFailureStatus=fail",
"--reportFile=report.json",
"--perfReportFile=perf.json",
"--storageEngine=my_storage_engine",
diff --git a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py b/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py
deleted file mode 100644
index caeb81e1e4b..00000000000
--- a/buildscripts/tests/resmokelib/testing/test_symbolizer_service.py
+++ /dev/null
@@ -1,250 +0,0 @@
-"""Unit tests for buildscripts/resmokelib/testing/symbolizer_service.py."""
-# pylint: disable=missing-docstring
-import os
-import unittest
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from unittest.mock import MagicMock
-
-from buildscripts.resmokelib.testing import symbolizer_service as under_test
-
-
-def mock_resmoke_symbolizer_config():
- config_mock: under_test.ResmokeSymbolizerConfig = MagicMock(
- spec_set=under_test.ResmokeSymbolizerConfig)
- config_mock.evg_task_id = "evg_task_id"
- config_mock.client_id = "client_id"
- config_mock.client_secret = "client_secret"
- config_mock.is_windows.return_value = False
- config_mock.is_macos.return_value = False
- return config_mock
-
-
-class TestResmokeSymbolizer(unittest.TestCase):
- def setUp(self) -> None:
- self.config_mock = mock_resmoke_symbolizer_config()
- self.symbolizer_service_mock: under_test.SymbolizerService = MagicMock(
- spec_set=under_test.SymbolizerService)
- self.file_service_mock: under_test.FileService = MagicMock(spec_set=under_test.FileService)
- self.resmoke_symbolizer = under_test.ResmokeSymbolizer(
- self.config_mock, self.symbolizer_service_mock, self.file_service_mock)
-
- def test_symbolize_test_logs_process_all_files(self):
- stacktrace_files = [f"file{i}.stacktrace" for i in range(5)]
- self.file_service_mock.filter_out_non_files.return_value = stacktrace_files
-
- self.resmoke_symbolizer.symbolize_test_logs(MagicMock())
-
- self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 5)
- for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list):
- self.assertEqual(call.args[0], f"file{i}.stacktrace")
- self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files)
-
- def test_symbolize_test_logs_hit_timeout(self):
- stacktrace_files = [f"file{i}.stacktrace" for i in range(5)]
- self.file_service_mock.filter_out_non_files.return_value = stacktrace_files
-
- self.resmoke_symbolizer.symbolize_test_logs(MagicMock(), 0)
-
- self.assertEqual(self.symbolizer_service_mock.run_symbolizer_script.call_count, 1)
- for i, call in enumerate(self.symbolizer_service_mock.run_symbolizer_script.call_arg_list):
- self.assertEqual(call.args[0], f"file{i}.stacktrace")
- self.file_service_mock.remove_all.assert_called_once_with(stacktrace_files)
-
- def test_symbolize_test_logs_should_not_symbolize(self):
- self.config_mock.is_windows.return_value = True
-
- self.resmoke_symbolizer.symbolize_test_logs(MagicMock())
- self.symbolizer_service_mock.run_symbolizer_script.assert_not_called()
-
- def test_symbolize_test_logs_could_not_get_dbpath(self):
- self.file_service_mock.check_path_exists.return_value = False
-
- self.resmoke_symbolizer.symbolize_test_logs(MagicMock())
- self.symbolizer_service_mock.run_symbolizer_script.assert_not_called()
-
- def test_symbolize_test_logs_did_not_find_files(self):
- self.file_service_mock.find_all_children_recursively.return_value = []
-
- self.resmoke_symbolizer.symbolize_test_logs(MagicMock())
- self.symbolizer_service_mock.run_symbolizer_script.assert_not_called()
-
- def test_should_not_symbolize_if_not_in_evergreen(self):
- self.config_mock.evg_task_id = None
- ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
- self.assertFalse(ret)
-
- def test_should_not_symbolize_if_secrets_are_absent(self):
- self.config_mock.client_id = None
- self.config_mock.client_secret = None
- ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
- self.assertFalse(ret)
-
- def test_should_not_symbolize_if_on_windows(self):
- self.config_mock.is_windows.return_value = True
- ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
- self.assertFalse(ret)
-
- def test_should_not_symbolize_if_on_macos(self):
- self.config_mock.is_macos.return_value = True
- ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
- self.assertFalse(ret)
-
- def test_should_symbolize_return_true(self):
- ret = self.resmoke_symbolizer.should_symbolize(MagicMock())
- self.assertTrue(ret)
-
- def test_get_stacktrace_dir_returns_dir(self):
- dbpath = "dbpath"
- test = MagicMock(fixture=MagicMock(get_dbpath_prefix=MagicMock(return_value=dbpath)))
- self.file_service_mock.check_path_exists.return_value = True
-
- ret = self.resmoke_symbolizer.get_stacktrace_dir(test)
- self.assertEqual(ret, dbpath)
-
- def test_get_stacktrace_dir_if_dir_does_not_exist(self):
- test = MagicMock()
- self.file_service_mock.check_path_exists.return_value = False
-
- ret = self.resmoke_symbolizer.get_stacktrace_dir(test)
- self.assertEqual(ret, None)
-
- def test_get_stacktrace_dir_if_fixture_is_not_available(self):
- test = MagicMock(fixture=None)
-
- ret = self.resmoke_symbolizer.get_stacktrace_dir(test)
- self.assertEqual(ret, None)
-
-
-class TestFileService(unittest.TestCase):
- def setUp(self) -> None:
- self.file_service = under_test.FileService()
- self.relative_dir_paths = [
- os.path.join("dir_1"),
- os.path.join("dir_2", "dir_2_1"),
- os.path.join("dir_2", "dir_2_2"),
- os.path.join("dir_3"),
- ]
- self.relative_file_paths = [
- os.path.join("dir_1", "file_1.stacktrace"),
- os.path.join("dir_2", "file_2.stacktrace"),
- os.path.join("dir_2", "dir_2_1", "file_3.stacktrace"),
- os.path.join("dir_2", "dir_2_2", "file_4.stacktrace"),
- ]
-
- def test_find_all_children_recursively_returns_files(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- for file in abs_file_paths:
- Path(file).touch()
-
- ret = self.file_service.find_all_children_recursively(tmpdir)
- self.assertListEqual(sorted(ret), sorted(abs_file_paths))
-
- def test_find_all_children_recursively_no_files(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
-
- ret = self.file_service.find_all_children_recursively(tmpdir)
- self.assertListEqual(ret, [])
-
- def test_find_all_children_recursively_no_dirs(self):
- with TemporaryDirectory() as tmpdir:
- ret = self.file_service.find_all_children_recursively(tmpdir)
- self.assertListEqual(ret, [])
-
- def test_filter_out_non_files_if_all_files_absent(self):
- with TemporaryDirectory() as tmpdir:
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- ret = self.file_service.filter_out_non_files(abs_file_paths)
- self.assertListEqual(ret, [])
-
- def test_filter_out_non_files_if_some_files_present(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- Path(abs_file_paths[1]).touch()
- Path(abs_file_paths[3]).touch()
-
- ret = self.file_service.filter_out_non_files(abs_file_paths)
- self.assertListEqual(ret, [abs_file_paths[1], abs_file_paths[3]])
-
- def test_filter_out_non_files_if_all_files_present(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- for file in abs_file_paths:
- Path(file).touch()
-
- ret = self.file_service.filter_out_non_files(abs_file_paths)
- self.assertListEqual(ret, abs_file_paths)
-
- def test_remove_empty_files_if_no_empty(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- for file in abs_file_paths:
- with open(file, "w") as fstream:
- fstream.write("stacktrace")
-
- self.file_service.remove_empty(abs_file_paths)
- for file in abs_file_paths:
- self.assertTrue(os.path.exists(file))
-
- def test_remove_empty_files_if_partly_empty(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- with open(abs_file_paths[0], "w") as fstream:
- fstream.write("stacktrace")
- Path(abs_file_paths[1]).touch()
- with open(abs_file_paths[2], "w") as fstream:
- fstream.write("stacktrace")
- Path(abs_file_paths[3]).touch()
-
- self.file_service.remove_empty(abs_file_paths)
-
- self.assertTrue(os.path.exists(abs_file_paths[0]))
- self.assertFalse(os.path.exists(abs_file_paths[1]))
- self.assertTrue(os.path.exists(abs_file_paths[2]))
- self.assertFalse(os.path.exists(abs_file_paths[3]))
-
- def test_remove_empty_files_if_all_empty(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- for file in abs_file_paths:
- Path(file).touch()
-
- self.file_service.remove_empty(abs_file_paths)
- for file in abs_file_paths:
- self.assertFalse(os.path.exists(file))
-
- def test_remove_all_files(self):
- with TemporaryDirectory() as tmpdir:
- abs_dir_paths = [os.path.join(tmpdir, d) for d in self.relative_dir_paths]
- abs_file_paths = [os.path.join(tmpdir, f) for f in self.relative_file_paths]
- for dir_ in abs_dir_paths:
- Path(dir_).mkdir(parents=True)
- for file in abs_file_paths:
- with open(file, "w") as fstream:
- fstream.write("stacktrace")
-
- self.file_service.remove_all(abs_file_paths)
- for file in abs_file_paths:
- self.assertFalse(os.path.exists(file))
diff --git a/buildscripts/tests/task_generation/test_suite_split.py b/buildscripts/tests/task_generation/test_suite_split.py
index b2b197ed7c5..7c54ed489aa 100644
--- a/buildscripts/tests/task_generation/test_suite_split.py
+++ b/buildscripts/tests/task_generation/test_suite_split.py
@@ -8,7 +8,7 @@ import requests
import buildscripts.task_generation.suite_split as under_test
from buildscripts.task_generation.suite_split_strategies import greedy_division, \
round_robin_fallback
-from buildscripts.util.teststats import TestRuntime, HistoricalTestInformation
+from buildscripts.util.teststats import TestRuntime
# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
@@ -31,12 +31,7 @@ def build_mock_service(evg_api=None, split_config=None, resmoke_proxy=None):
def tst_stat_mock(file, duration, pass_count):
- return HistoricalTestInformation(
- test_name=file,
- num_pass=pass_count,
- num_fail=0,
- avg_duration_pass=duration,
- )
+ return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count)
def build_mock_split_config(target_resmoke_time=None, max_sub_suites=None):
@@ -120,16 +115,15 @@ class TestGeneratedSuite(unittest.TestCase):
class TestSplitSuite(unittest.TestCase):
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_calculate_suites(self, get_stats_from_s3_mock):
+ def test_calculate_suites(self):
mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
split_config = build_mock_split_config(target_resmoke_time=10)
split_params = build_mock_split_params()
suite_split_service = build_mock_service(split_config=split_config)
- get_stats_from_s3_mock.return_value = mock_test_stats
+ suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
suite_split_service.resmoke_proxy.list_tests.return_value = [
- stat.test_name for stat in mock_test_stats
+ stat.test_file for stat in mock_test_stats
]
suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
@@ -143,15 +137,32 @@ class TestSplitSuite(unittest.TestCase):
for sub_suite in suite.sub_suites:
self.assertEqual(10, len(sub_suite.test_list))
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_calculate_suites_uses_fallback_on_no_results(self, get_stats_from_s3_mock):
+ def test_calculate_suites_fallback_on_error(self):
+ n_tests = 100
+ max_sub_suites = 4
+ split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
+ split_params = build_mock_split_params()
+
+ suite_split_service = build_mock_service(split_config=split_config)
+ mock_evg_error(suite_split_service.evg_api)
+ suite_split_service.resmoke_proxy.list_tests.return_value = [
+ f"test_{i}.js" for i in range(n_tests)
+ ]
+
+ suite = suite_split_service.split_suite(split_params)
+
+ self.assertEqual(max_sub_suites, len(suite))
+ for sub_suite in suite.sub_suites:
+ self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
+
+ def test_calculate_suites_uses_fallback_on_no_results(self):
n_tests = 100
max_sub_suites = 5
split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
split_params = build_mock_split_params()
suite_split_service = build_mock_service(split_config=split_config)
- get_stats_from_s3_mock.return_value = []
+ suite_split_service.evg_api.test_stats_by_project.return_value = []
suite_split_service.resmoke_proxy.list_tests.return_value = [
f"test_{i}.js" for i in range(n_tests)
]
@@ -162,9 +173,7 @@ class TestSplitSuite(unittest.TestCase):
for sub_suite in suite.sub_suites:
self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_calculate_suites_uses_fallback_if_only_results_are_filtered(
- self, get_stats_from_s3_mock):
+ def test_calculate_suites_uses_fallback_if_only_results_are_filtered(self):
n_tests = 100
max_sub_suites = 10
mock_test_stats = [tst_stat_mock(f"test{i}.js", 60, 1) for i in range(100)]
@@ -173,7 +182,7 @@ class TestSplitSuite(unittest.TestCase):
split_params = build_mock_split_params()
suite_split_service = build_mock_service(split_config=split_config)
- get_stats_from_s3_mock.return_value = mock_test_stats
+ suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
suite_split_service.resmoke_proxy.list_tests.return_value = [
f"test_{i}.js" for i in range(n_tests)
]
@@ -189,17 +198,31 @@ class TestSplitSuite(unittest.TestCase):
for sub_suite in suite.sub_suites:
self.assertEqual(n_tests / max_sub_suites, len(sub_suite.test_list))
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_calculate_suites_will_filter_specified_tests(self, get_stats_from_s3_mock):
+ def test_calculate_suites_fail_on_unexpected_error(self):
+ n_tests = 100
+ max_sub_suites = 4
+ split_config = build_mock_split_config(max_sub_suites=max_sub_suites)
+ split_params = build_mock_split_params()
+
+ suite_split_service = build_mock_service(split_config=split_config)
+ mock_evg_error(suite_split_service.evg_api, error_code=requests.codes.INTERNAL_SERVER_ERROR)
+ suite_split_service.resmoke_proxy.list_tests.return_value = [
+ f"test_{i}.js" for i in range(n_tests)
+ ]
+
+ with self.assertRaises(requests.HTTPError):
+ suite_split_service.split_suite(split_params)
+
+ def test_calculate_suites_will_filter_specified_tests(self):
mock_test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
split_config = build_mock_split_config(target_resmoke_time=10)
split_params = build_mock_split_params(
test_filter=lambda t: t in {"test_1.js", "test_2.js"})
suite_split_service = build_mock_service(split_config=split_config)
- get_stats_from_s3_mock.return_value = mock_test_stats
+ suite_split_service.evg_api.test_stats_by_project.return_value = mock_test_stats
suite_split_service.resmoke_proxy.list_tests.return_value = [
- stat.test_name for stat in mock_test_stats
+ stat.test_file for stat in mock_test_stats
]
suite_split_service.resmoke_proxy.read_suite_config.return_value = {}
diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py
new file mode 100644
index 00000000000..ec53a02d161
--- /dev/null
+++ b/buildscripts/tests/test_burn_in_tags.py
@@ -0,0 +1,261 @@
+"""Unit tests for the burn_in_tags.py script."""
+from collections import defaultdict
+import json
+import os
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+from shrub.v2 import ShrubProject
+
+import buildscripts.ciconfig.evergreen as _evergreen
+from buildscripts.burn_in_tests import TaskInfo
+from buildscripts.tests.test_burn_in_tests import ns as burn_in_tests_ns
+from buildscripts.ciconfig.evergreen import EvergreenProjectConfig
+
+import buildscripts.burn_in_tags as under_test
+
+# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
+
+EMPTY_PROJECT = {
+ "buildvariants": [],
+ "tasks": [],
+}
+TEST_FILE_PATH = os.path.join(os.path.dirname(__file__), "test_burn_in_tags_evergreen.yml")
+
+NS = "buildscripts.burn_in_tags"
+
+
+def ns(relative_name): # pylint: disable-invalid-name
+ """Return a full name from a name relative to the test module"s name space."""
+ return NS + "." + relative_name
+
+
+def get_expansions_data():
+ return {
+ "branch_name": "fake_branch",
+ "build_variant": "enterprise-rhel-80-64-bit-suggested",
+ "check_evergreen": 2,
+ "distro_id": "rhel80-small",
+ "is_patch": "true",
+ "max_revisions": 25,
+ "repeat_tests_max": 1000,
+ "repeat_tests_min": 2,
+ "repeat_tests_secs": 600,
+ "revision": "fake_sha",
+ "project": "fake_project",
+ "task_id": "task id",
+ } # yapf: disable
+
+
+def get_evergreen_config() -> EvergreenProjectConfig:
+ return _evergreen.parse_evergreen_file(TEST_FILE_PATH, evergreen_binary=None)
+
+
+class TestCreateEvgBuildVariantMap(unittest.TestCase):
+ def test_create_evg_buildvariant_map(self):
+ expansions_file_data = {
+ "build_variant": "variant1", "burn_in_tag_buildvariants": "variant2 variant3"
+ }
+ buildvariant_map = under_test._create_evg_build_variant_map(expansions_file_data)
+
+ expected_buildvariant_map = {
+ "variant2": "variant2-required", "variant3": "variant3-required"
+ }
+ self.assertEqual(buildvariant_map, expected_buildvariant_map)
+
+
+class TestGenerateEvgBuildVariants(unittest.TestCase):
+ def test_generate_evg_buildvariant_one_base_variant(self):
+ evg_conf_mock = get_evergreen_config()
+ base_variant = "enterprise-rhel-80-64-bit-inmem"
+ generated_variant = "enterprise-rhel-80-64-bit-inmem-required"
+ burn_in_tags_gen_variant = "enterprise-rhel-80-64-bit"
+ variant = evg_conf_mock.get_variant(base_variant)
+
+ build_variant = under_test._generate_evg_build_variant(variant, generated_variant,
+ burn_in_tags_gen_variant)
+
+ generated_build_variant = build_variant.as_dict()
+ self.assertEqual(generated_build_variant["name"], generated_variant)
+ self.assertEqual(generated_build_variant["modules"], variant.modules)
+ generated_expansions = generated_build_variant["expansions"]
+ burn_in_bypass_expansion_value = generated_expansions.pop("burn_in_bypass")
+ self.assertEqual(burn_in_bypass_expansion_value, burn_in_tags_gen_variant)
+ self.assertEqual(generated_expansions, variant.expansions)
+
+
+class TestGenerateEvgTasks(unittest.TestCase):
+ @patch(ns("create_tests_by_task"))
+ def test_generate_evg_tasks_no_tests_changed(self, create_tests_by_task_mock):
+ evg_conf_mock = get_evergreen_config()
+ create_tests_by_task_mock.return_value = {}
+ expansions_file_data = get_expansions_data()
+ buildvariant_map = {
+ "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required",
+ "enterprise-rhel-80-64-bit-majority-read-concern-off":
+ "enterprise-rhel-80-64-bit-majority-read-concern-off-required",
+ } # yapf: disable
+ shrub_config = ShrubProject()
+ evergreen_api = MagicMock()
+ repo = MagicMock(working_dir=os.getcwd())
+ under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
+ buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin')
+
+ self.assertEqual(shrub_config.as_dict(), EMPTY_PROJECT)
+
+ @patch(ns("create_tests_by_task"))
+ def test_generate_evg_tasks_one_test_changed(self, create_tests_by_task_mock):
+ evg_conf_mock = get_evergreen_config()
+ create_tests_by_task_mock.return_value = {
+ "aggregation_mongos_passthrough": TaskInfo(
+ display_task_name="aggregation_mongos_passthrough",
+ suite="aggregation_mongos_passthrough",
+ resmoke_args="--suites=aggregation_mongos_passthrough --storageEngine=wiredTiger",
+ tests=["jstests/aggregation/ifnull.js"],
+ require_multiversion_setup=False,
+ distro="",
+ build_variant="enterprise-rhel-80-64-bit-inmem"
+ )
+ } # yapf: disable
+ expansions_file_data = get_expansions_data()
+ buildvariant_map = {
+ "enterprise-rhel-80-64-bit-inmem": "enterprise-rhel-80-64-bit-inmem-required",
+ "enterprise-rhel-80-64-bit-majority-read-concern-off":
+ "enterprise-rhel-80-64-bit-majority-read-concern-off-required",
+ } # yapf: disable
+ shrub_config = ShrubProject.empty()
+ evergreen_api = MagicMock()
+ repo = MagicMock(working_dir=os.getcwd())
+ evergreen_api.test_stats_by_project.return_value = [
+ MagicMock(test_file="dir/test2.js", avg_duration_pass=10)
+ ]
+ under_test._generate_evg_tasks(evergreen_api, shrub_config, expansions_file_data,
+ buildvariant_map, [repo], evg_conf_mock, 'install-dir/bin')
+
+ generated_config = shrub_config.as_dict()
+ self.assertEqual(len(generated_config["buildvariants"]), 2)
+ first_generated_build_variant = generated_config["buildvariants"][0]
+ self.assertIn(first_generated_build_variant["name"], buildvariant_map.values())
+ self.assertEqual(first_generated_build_variant["display_tasks"][0]["name"], "burn_in_tests")
+ self.assertEqual(
+ first_generated_build_variant["display_tasks"][0]["execution_tasks"][0],
+ f"burn_in:aggregation_mongos_passthrough_0_{first_generated_build_variant['name']}")
+
+
+EXPANSIONS_FILE_DATA = {
+ "build_variant": "enterprise-rhel-80-64-bit",
+ "revision": "badf00d000000000000000000000000000000000", "max_revisions": "1000",
+ "branch_name": "mongodb-mongo-master", "is_patch": "false", "distro_id": "rhel62-small",
+ "repeat_tests_min": "2", "repeat_tests_max": "1000", "repeat_tests_secs": "600", "project":
+ "mongodb-mongo-master", "task_id": "task id"
+}
+
+CREATE_EVG_BUILD_VARIANT_MAP = {
+ 'enterprise-rhel-80-64-bit-majority-read-concern-off':
+ 'enterprise-rhel-80-64-bit-majority-read-concern-off-required',
+ 'enterprise-rhel-80-64-bit-inmem':
+ 'enterprise-rhel-80-64-bit-inmem-required'
+}
+
+CREATE_TEST_MEMBERSHIP_MAP = {
+ "jstests/aggregation/accumulators/accumulator_js.js": [
+ "aggregation", "aggregation_auth", "aggregation_disabled_optimization", "aggregation_ese",
+ "aggregation_ese_gcm", "aggregation_facet_unwind_passthrough",
+ "aggregation_mongos_passthrough", "aggregation_one_shard_sharded_collections",
+ "aggregation_read_concern_majority_passthrough", "aggregation_secondary_reads",
+ "aggregation_sharded_collections_passthrough"
+ ], "jstests/core/create_collection.js": [
+ "core", "core_auth", "core_ese", "core_ese_gcm", "core_minimum_batch_size", "core_op_query",
+ "cwrwc_passthrough", "cwrwc_rc_majority_passthrough", "cwrwc_wc_majority_passthrough",
+ "logical_session_cache_replication_100ms_refresh_jscore_passthrough",
+ "logical_session_cache_replication_10sec_refresh_jscore_passthrough",
+ "logical_session_cache_replication_1sec_refresh_jscore_passthrough",
+ "logical_session_cache_replication_default_refresh_jscore_passthrough",
+ "logical_session_cache_standalone_100ms_refresh_jscore_passthrough",
+ "logical_session_cache_standalone_10sec_refresh_jscore_passthrough",
+ "logical_session_cache_standalone_1sec_refresh_jscore_passthrough",
+ "logical_session_cache_standalone_default_refresh_jscore_passthrough",
+ "read_concern_linearizable_passthrough", "read_concern_majority_passthrough",
+ "causally_consistent_read_concern_snapshot_passthrough",
+ "replica_sets_initsync_jscore_passthrough", "replica_sets_fcbis_jscore_passthrough",
+ "replica_sets_initsync_static_jscore_passthrough", "replica_sets_jscore_passthrough",
+ "replica_sets_kill_primary_jscore_passthrough",
+ "replica_sets_kill_secondaries_jscore_passthrough",
+ "replica_sets_reconfig_jscore_passthrough",
+ "replica_sets_terminate_primary_jscore_passthrough", "retryable_writes_jscore_passthrough",
+ "retryable_writes_jscore_stepdown_passthrough", "secondary_reads_passthrough",
+ "session_jscore_passthrough", "write_concern_majority_passthrough"
+ ]
+}
+
+
+class TestAcceptance(unittest.TestCase):
+ @patch(ns("write_file_to_dir"))
+ @patch(ns("_create_evg_build_variant_map"))
+ @patch(ns("EvergreenFileChangeDetector"))
+ def test_no_tests_run_if_none_changed(self, find_changed_tests_mock,
+ create_evg_build_variant_map_mock, write_to_file_mock):
+ """
+ Given a git repository with no changes,
+ When burn_in_tags is run,
+ Then no tests are discovered to run.
+ """
+ repos = [MagicMock(working_dir=os.getcwd())]
+ evg_conf_mock = MagicMock()
+ find_changed_tests_mock.return_value.find_changed_tests.return_value = {}
+
+ create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP
+
+ under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf_mock, MagicMock(), repos,
+ 'install_dir/bin')
+
+ write_to_file_mock.assert_called_once()
+ shrub_config = write_to_file_mock.call_args[0][2]
+ self.assertEqual(EMPTY_PROJECT, json.loads(shrub_config))
+
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ @patch(ns("write_file_to_dir"))
+ @patch(ns("_create_evg_build_variant_map"))
+ @patch(ns("EvergreenFileChangeDetector"))
+ @patch(burn_in_tests_ns("create_test_membership_map"))
+ def test_tests_generated_if_a_file_changed(
+ self, create_test_membership_map_mock, find_changed_tests_mock,
+ create_evg_build_variant_map_mock, write_to_file_mock):
+ """
+ Given a git repository with changes,
+ When burn_in_tags is run,
+ Then some tags are discovered to run.
+ """
+ create_test_membership_map_mock.return_value = defaultdict(list, CREATE_TEST_MEMBERSHIP_MAP)
+
+ repos = [MagicMock(working_dir=os.getcwd())]
+ evg_conf = get_evergreen_config()
+ create_evg_build_variant_map_mock.return_value = CREATE_EVG_BUILD_VARIANT_MAP
+ find_changed_tests_mock.return_value.find_changed_tests.return_value = {
+ 'jstests/slow1/large_role_chain.js',
+ 'jstests/aggregation/accumulators/accumulator_js.js'
+ }
+
+ under_test.burn_in(EXPANSIONS_FILE_DATA, evg_conf, MagicMock(), repos, 'install_dir/bin')
+
+ write_to_file_mock.assert_called_once()
+ written_config = write_to_file_mock.call_args[0][2]
+ written_config_map = json.loads(written_config)
+
+ n_tasks = len(written_config_map["tasks"])
+ # Ensure we are generating at least one task for the test.
+ self.assertGreaterEqual(n_tasks, 1)
+
+ written_build_variants = written_config_map["buildvariants"]
+ written_build_variants_name = [variant['name'] for variant in written_build_variants]
+ self.assertEqual(
+ set(CREATE_EVG_BUILD_VARIANT_MAP.values()), set(written_build_variants_name))
+
+ tasks = written_config_map["tasks"]
+ self.assertGreaterEqual(len(tasks), len(CREATE_EVG_BUILD_VARIANT_MAP))
+
+ self.assertTrue(
+ all(
+ len(display_tasks) == 1 for display_tasks in
+ [build_variant["display_tasks"] for build_variant in written_build_variants]))
diff --git a/buildscripts/tests/test_burn_in_tags_evergreen.yml b/buildscripts/tests/test_burn_in_tags_evergreen.yml
new file mode 100644
index 00000000000..d6a651fcde8
--- /dev/null
+++ b/buildscripts/tests/test_burn_in_tags_evergreen.yml
@@ -0,0 +1,138 @@
+functions:
+ "fetch source":
+ - command: git.get_project
+ params:
+ directory: src
+ - command: shell.exec
+ params:
+ working_dir: src
+ script: |
+ echo "this is a 2nd command in the function!"
+ ls
+
+tasks:
+- name: compile
+ depends_on: []
+ commands:
+ - func: "fetch source"
+- name: burn_in_tags_gen
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: compile_all_run_unittests_TG
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: clang_tidy_TG
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: stitch_support_lib_build_and_archive
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: lint_pylinters
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: lint_clang_format
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: burn_in_tests_gen
+ depends_on: []
+ commands:
+ - func: "fake command"
+- name: aggregation_multiversion_fuzzer_gen
+ depends_on: []
+ commands:
+ - func: "generate resmoke tasks"
+- name: aggregation_expression_multiversion_fuzzer_gen
+ depends_on: []
+ commands:
+ - func: "generate resmoke tasks"
+- name: aggregation
+ depends_on:
+ - name: compile
+ commands:
+ - func: run tests
+ vars:
+ resmoke_args: --suites=aggregation --storageEngine=wiredTiger
+
+buildvariants:
+- name: enterprise-rhel-80-64-bit
+ display_name: "! Enterprise RHEL 8.0"
+ expansions:
+ multiversion_platform: rhel80
+ burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-majority-read-concern-off enterprise-rhel-80-64-bit-inmem
+ tasks:
+ - name: compile_all_run_unittests_TG
+ distros:
+ - rhel80-large
+ - name: lint_pylinters
+ - name: burn_in_tests_gen
+ - name: aggregation_multiversion_fuzzer_gen
+ - name: aggregation
+ - name: burn_in_tags_gen
+- name: buildvariant-without-burn-in-tag-buildvariants
+ display_name: "Buildvariant without burn in tag buildvariants expansion"
+ expansions:
+ multiversion_platform: rhel80
+ tasks:
+ - name: burn_in_tags_gen
+- name: enterprise-rhel-80-64-bit-majority-read-concern-off
+ display_name: "Enterprise RHEL 8.0 (majority read concern off)"
+ modules: ["enterprise"]
+ run_on:
+ - rhel80-small
+ expansions: &enterprise-rhel-80-64-bit-majority-read-concern-off-expansions
+ multiversion_edition: enterprise
+ tasks:
+ - name: compile_all_run_unittests_TG
+ distros:
+ - rhel80-large
+ - name: aggregation_multiversion_fuzzer_gen
+ - name: aggregation
+- name: enterprise-rhel-80-64-bit-inmem
+ display_name: Enterprise RHEL 8.0 (inMemory)
+ modules: ["enterprise"]
+ run_on:
+ - rhel80-small
+ expansions: &enterprise-rhel-80-64-bit-inmem-expansions
+ test_flags: >-
+ --majorityReadConcern=off
+ --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime
+ compile_flags: >-
+ -j$(grep -c ^processor /proc/cpuinfo)
+ --ssl
+ --release
+ --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+ MONGO_DISTMOD=rhel80
+ multiversion_platform: rhel80
+ multiversion_edition: enterprise
+ scons_cache_scope: shared
+ tooltags: "ssl sasl gssapi"
+ large_distro_name: rhel80-large
+ tasks:
+ - name: compile
+- name: enterprise-rhel-80-64-bit-inmem
+ display_name: Enterprise RHEL 8.0 (inMemory)
+ expansions:
+ additional_targets: archive-mongocryptd archive-mongocryptd-debug
+ compile_flags: --ssl MONGO_DISTMOD=rhel80 -j$(grep -c ^processor /proc/cpuinfo)
+ --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+ large_distro_name: rhel80-large
+ multiversion_edition: enterprise
+ multiversion_platform: rhel80
+ scons_cache_scope: shared
+ test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling
+ modules:
+ - enterprise
+ run_on:
+ - rhel80-small
+ tasks:
+ - name: compile_all_run_unittests_TG
+ distros:
+ - rhel80-large
+ - name: aggregation_multiversion_fuzzer_gen
+ - name: aggregation
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index 369131eff60..c51f8c60ecf 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -4,14 +4,12 @@ from __future__ import absolute_import
import collections
import datetime
-from io import StringIO
import os
import sys
import subprocess
import unittest
from mock import Mock, patch, MagicMock
-import yaml
import buildscripts.burn_in_tests as under_test
from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask
@@ -558,19 +556,3 @@ class TestLocalFileChangeDetector(unittest.TestCase):
self.assertIn(file_list[2], found_tests)
self.assertNotIn(file_list[1], found_tests)
self.assertEqual(2, len(found_tests))
-
-
-class TestYamlBurnInExecutor(unittest.TestCase):
- @patch('sys.stdout', new_callable=StringIO)
- def test_found_tasks_should_be_reported_as_yaml(self, stdout):
- n_tasks = 5
- n_tests = 3
- tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
-
- yaml_executor = under_test.YamlBurnInExecutor()
- yaml_executor.execute(tests_by_task)
-
- yaml_raw = stdout.getvalue()
- results = yaml.safe_load(yaml_raw)
- self.assertEqual(n_tasks, len(results["discovered_tasks"]))
- self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"]))
diff --git a/buildscripts/tests/test_debugsymb_mapper.py b/buildscripts/tests/test_debugsymb_mapper.py
deleted file mode 100644
index b1b063f5331..00000000000
--- a/buildscripts/tests/test_debugsymb_mapper.py
+++ /dev/null
@@ -1,115 +0,0 @@
-"""Unit tests for debugsymb_mapper.py."""
-# pylint: disable=missing-docstring
-import unittest
-from unittest.mock import MagicMock
-
-import buildscripts.debugsymb_mapper as under_test
-
-
-def mock_cmd_client():
- cmd_client = MagicMock(spec_set=under_test.CmdClient)
- return cmd_client
-
-
-class TestCmdOutputExtractor(unittest.TestCase):
- def setUp(self):
- self.cmd_client_mock = mock_cmd_client()
- self.cmd_output_extractor = under_test.CmdOutputExtractor(self.cmd_client_mock)
-
-
-class TestGetBuildId(TestCmdOutputExtractor):
- def test_get_build_id_returns_build_id(self):
- readelf_output = (
- "Displaying notes found in: .note.gnu.build-id\n"
- " Owner Data size\tDescription\n"
- " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n"
- " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n"
- "\n"
- "Displaying notes found in: .gnu.build.attributes.hot\n"
- " Owner Data size\tDescription\n"
- " GA$<version>3h864 0x00000010\tOPEN\n"
- " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)\n"
- " GA$<version>3h864 0x00000010\tOPEN\n"
- " Applies to region from 0xb71 to 0xb71 (.annobin_init.c.hot)")
- self.cmd_client_mock.run.return_value = readelf_output
-
- build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin")
- self.assertEqual(build_id_output.build_id, "74c2322104428836f3d94af6cd7471ee7cb5c4ee")
- self.assertEqual(build_id_output.cmd_output, readelf_output)
-
- def test_get_build_id_raises_error(self):
- readelf_output = (
- " Owner Data size\tDescription\n"
- " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n"
- " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n"
- "\n"
- "Displaying notes found in: .gnu.build.attributes.hot\n"
- " Owner Data size\tDescription\n"
- " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)\n"
- " Build ID: 74c2322104428836f3d94af6cd7471ee7cb5c4ee\n"
- "\n"
- "Displaying notes found in: .gnu.build.attributes.hot")
- self.cmd_client_mock.run.return_value = readelf_output
-
- self.assertRaises(ValueError, self.cmd_output_extractor.get_build_id, "path/to/bin")
-
- def test_get_build_id_returns_none(self):
- readelf_output = (
- "Displaying notes found in: .note.gnu.build-id\n"
- " Owner Data size\tDescription\n"
- " GNU 0x00000014\tNT_GNU_BUILD_ID (unique build ID bitstring)")
- self.cmd_client_mock.run.return_value = readelf_output
-
- build_id_output = self.cmd_output_extractor.get_build_id("path/to/bin")
- self.assertIsNone(build_id_output.build_id)
- self.assertEqual(build_id_output.cmd_output, readelf_output)
-
-
-class TestGetBinVersion(TestCmdOutputExtractor):
- def test_get_bin_version_returns_version(self):
- # Newer versions command output
- version_cmd_output = ('db version v4.4.14-25-gb0475e2\n'
- 'Build Info: {\n'
- ' "version": "4.4.14-25-gb0475e2",\n'
- ' "gitVersion": "b0475e2657c3351b25499971d3340f054ea85b98",\n'
- ' "openSSLVersion": "OpenSSL 1.1.1 11 Sep 2018",\n'
- ' "modules": [\n'
- ' "enterprise"\n'
- ' ],\n'
- ' "allocator": "tcmalloc",\n'
- ' "environment": {\n'
- ' "distmod": "ubuntu1804",\n'
- ' "distarch": "x86_64",\n'
- ' "target_arch": "x86_64"\n'
- ' }\n'
- '}')
- self.cmd_client_mock.run.return_value = version_cmd_output
-
- bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin")
- self.assertEqual(bin_version_output.mongodb_version, "4.4.14-25-gb0475e2")
- self.assertEqual(bin_version_output.cmd_output, version_cmd_output)
-
- def test_get_bin_version_unsupported_output(self):
- # Versions prior to 5.0 are not supported
- version_cmd_output = ('db version v4.2.20-7-g5a81409\n'
- 'git version: 5a81409faf16f30f1189af6367eb3ceee50a02b5\n'
- 'OpenSSL version: OpenSSL 1.1.1 11 Sep 2018\n'
- 'allocator: tcmalloc\n'
- 'modules: enterprise \n'
- 'build environment:\n'
- ' distmod: ubuntu1804\n'
- ' distarch: x86_64\n'
- ' target_arch: x86_64')
- self.cmd_client_mock.run.return_value = version_cmd_output
-
- bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin")
- self.assertIsNone(bin_version_output.mongodb_version)
- self.assertEqual(bin_version_output.cmd_output, version_cmd_output)
-
- def test_get_bin_version_returns_none(self):
- version_cmd_output = "error: unrecognized arguments: --version"
- self.cmd_client_mock.run.return_value = version_cmd_output
-
- bin_version_output = self.cmd_output_extractor.get_bin_version("path/to/bin")
- self.assertIsNone(bin_version_output.mongodb_version)
- self.assertEqual(bin_version_output.cmd_output, version_cmd_output)
diff --git a/buildscripts/tests/test_errorcodes.py b/buildscripts/tests/test_errorcodes.py
index 17c102454b5..2a9c9ce1e3f 100644
--- a/buildscripts/tests/test_errorcodes.py
+++ b/buildscripts/tests/test_errorcodes.py
@@ -26,7 +26,7 @@ class TestErrorcodes(unittest.TestCase):
captured_error_codes.append(code)
errorcodes.parse_source_files(accumulate_files, TESTDATA_DIR + 'regex_matching/')
- self.assertEqual(31, len(captured_error_codes))
+ self.assertEqual(26, len(captured_error_codes))
def test_dup_checking(self):
"""Test dup checking."""
diff --git a/buildscripts/tests/test_evergreen_activate_gen_tasks.py b/buildscripts/tests/test_evergreen_activate_gen_tasks.py
index 5470828c6c3..dbb64380838 100644
--- a/buildscripts/tests/test_evergreen_activate_gen_tasks.py
+++ b/buildscripts/tests/test_evergreen_activate_gen_tasks.py
@@ -1,126 +1,42 @@
"""Unit tests for the generate_resmoke_suite script."""
-# pylint: disable=invalid-name
import unittest
-from mock import MagicMock, mock
+from mock import MagicMock
from buildscripts import evergreen_activate_gen_tasks as under_test
-from evergreen import Build, EvergreenApi, Task, Version
# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter
-def build_mock_task(display_name, task_id):
- mock_task = MagicMock(spec_set=Task, display_name=display_name, task_id=task_id)
+def build_mock_task(name, task_id):
+ mock_task = MagicMock(display_name=name, task_id=task_id)
return mock_task
-def build_mock_task_list(num_tasks):
- return [build_mock_task(f"task_{i}", f"id_{i}") for i in range(num_tasks)]
-
-
-class MockVariantData():
- """An object to help create a mock evg api."""
-
- def __init__(self, build_id, variant_name, task_list):
- self.build_id = build_id
- self.variant_name = variant_name
- self.task_list = task_list
-
-
-def build_mock_evg_api(variant_data_list):
- class VersionPatchedSpec(Version):
- """A patched `Version` with instance properties included for magic mock spec."""
- build_variants_map = MagicMock()
-
- mock_version = MagicMock(spec_set=VersionPatchedSpec)
- mock_version.build_variants_map = {
- variant_data.variant_name: variant_data.build_id
- for variant_data in variant_data_list
- }
-
- mock_evg_api = MagicMock(spec_set=EvergreenApi)
- mock_evg_api.version_by_id.return_value = mock_version
-
- build_id_mapping = {
- variant_data.build_id: variant_data.task_list
- for variant_data in variant_data_list
- }
-
- def tasks_by_build_side_effect(build_id):
- return build_id_mapping[build_id]
-
- mock_evg_api.tasks_by_build.side_effect = tasks_by_build_side_effect
+def build_mock_evg_api(mock_task_list):
+ mock_build = MagicMock()
+ mock_build.get_tasks.return_value = mock_task_list
+ mock_evg_api = MagicMock()
+ mock_evg_api.build_by_id.return_value = mock_build
return mock_evg_api
class TestActivateTask(unittest.TestCase):
def test_task_with_display_name_is_activated(self):
- expansions = under_test.EvgExpansions(**{
- "build_id": "build_id",
- "version_id": "version_id",
- "task_name": "task_3_gen",
- })
- mock_task_list = build_mock_task_list(5)
- mock_evg_api = build_mock_evg_api(
- [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)])
+ n_tasks = 5
+ mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)]
+ mock_evg_api = build_mock_evg_api(mock_task_list)
- under_test.activate_task(expansions, mock_evg_api)
+ under_test.activate_task("build_id", "task_3", mock_evg_api)
mock_evg_api.configure_task.assert_called_with("id_3", activated=True)
def test_task_with_no_matching_name(self):
- expansions = under_test.EvgExpansions(**{
- "build_id": "build_id",
- "version_id": "version_id",
- "task_name": "not_an_existing_task",
- })
- mock_task_list = build_mock_task_list(5)
- mock_evg_api = build_mock_evg_api(
- [MockVariantData("build_id", "non-burn-in-bv", mock_task_list)])
+ n_tasks = 5
+ mock_task_list = [build_mock_task(f"task_{i}", f"id_{i}") for i in range(n_tasks)]
+ mock_evg_api = build_mock_evg_api(mock_task_list)
- under_test.activate_task(expansions, mock_evg_api)
+ under_test.activate_task("build_id", "not_an_existing_task", mock_evg_api)
mock_evg_api.configure_task.assert_not_called()
-
- def test_burn_in_tags_tasks_are_activated(self):
- expansions = under_test.EvgExpansions(**{
- "build_id": "build_id",
- "version_id": "version_id",
- "task_name": "burn_in_tags_gen",
- })
- mock_task_list_2 = build_mock_task_list(5)
- mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2"))
- mock_task_list_3 = build_mock_task_list(5)
- mock_task_list_3.append(build_mock_task("burn_in_tests", "burn_in_tests_id_3"))
- mock_evg_api = build_mock_evg_api([
- MockVariantData("1", "variant1-generated-by-burn-in-tags", mock_task_list_2),
- MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_3)
- ])
-
- under_test.activate_task(expansions, mock_evg_api)
-
- mock_evg_api.configure_task.assert_has_calls([
- mock.call("burn_in_tests_id_2", activated=True),
- mock.call("burn_in_tests_id_3", activated=True)
- ])
-
- def test_burn_in_tags_task_skips_non_existing_build_variant(self):
- expansions = under_test.EvgExpansions(**{
- "build_id": "build_id",
- "version_id": "version_id",
- "task_name": "burn_in_tags_gen",
- })
- mock_task_list_1 = build_mock_task_list(5)
- mock_task_list_1.append(build_mock_task("burn_in_tags_gen", "burn_in_tags_gen_id_1"))
- mock_task_list_2 = build_mock_task_list(5)
- mock_task_list_2.append(build_mock_task("burn_in_tests", "burn_in_tests_id_2"))
- mock_evg_api = build_mock_evg_api([
- MockVariantData("1", "variant1-non-burn-in", mock_task_list_1),
- MockVariantData("2", "variant2-generated-by-burn-in-tags", mock_task_list_2)
- ])
-
- under_test.activate_task(expansions, mock_evg_api)
-
- mock_evg_api.configure_task.assert_called_once_with("burn_in_tests_id_2", activated=True)
diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py
new file mode 100644
index 00000000000..ee77ced2579
--- /dev/null
+++ b/buildscripts/tests/test_evergreen_burn_in_tests.py
@@ -0,0 +1,408 @@
+"""Unit tests for buildscripts/burn_in_tests.py."""
+
+from __future__ import absolute_import
+
+import json
+import os
+import sys
+import unittest
+from datetime import datetime, timedelta
+from math import ceil
+
+import requests
+from mock import patch, MagicMock
+from shrub.v2 import BuildVariant, ShrubProject
+from evergreen.api import EvergreenApi
+
+import buildscripts.evergreen_burn_in_tests as under_test
+from buildscripts.ciconfig.evergreen import parse_evergreen_file
+import buildscripts.resmokelib.parser as _parser
+import buildscripts.resmokelib.config as _config
+import buildscripts.util.teststats as teststats_utils
+_parser.set_run_options()
+
+# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
+
+NS = "buildscripts.evergreen_burn_in_tests"
+
+
+def ns(relative_name): # pylint: disable=invalid-name
+ """Return a full name from a name relative to the test module"s name space."""
+ return NS + "." + relative_name
+
+
+def mock_a_file(filename):
+ change = MagicMock(a_path=filename)
+ return change
+
+
+def mock_git_diff(change_list):
+ diff = MagicMock()
+ diff.iter_change_type.return_value = change_list
+ return diff
+
+
+def mock_changed_git_files(add_files):
+ repo = MagicMock()
+ repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files])
+ repo.working_dir = "."
+ return repo
+
+
+def get_evergreen_config(config_file_path):
+ evergreen_home = os.path.expanduser(os.path.join("~", "evergreen"))
+ if os.path.exists(evergreen_home):
+ return parse_evergreen_file(config_file_path, evergreen_home)
+ return parse_evergreen_file(config_file_path)
+
+
+class TestAcceptance(unittest.TestCase):
+ def tearDown(self):
+ _parser.set_run_options()
+
+ @patch(ns("write_file"))
+ def test_no_tests_run_if_none_changed(self, write_json_mock):
+ """
+ Given a git repository with no changes,
+ When burn_in_tests is run,
+ Then no tests are discovered to run.
+ """
+ variant = "build_variant"
+ repos = [mock_changed_git_files([])]
+ repeat_config = under_test.RepeatConfig()
+ gen_config = under_test.GenerateConfig(
+ variant,
+ "project",
+ ) # yapf: disable
+ mock_evg_conf = MagicMock()
+ mock_evg_conf.get_task_names_by_tag.return_value = set()
+ mock_evg_api = MagicMock()
+
+ under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api,
+ mock_evg_conf, repos, "testfile.json", "install-dir/bin")
+
+ write_json_mock.assert_called_once()
+ written_config = json.loads(write_json_mock.call_args[0][1])
+ display_task = written_config["buildvariants"][0]["display_tasks"][0]
+ self.assertEqual(1, len(display_task["execution_tasks"]))
+ self.assertEqual(under_test.BURN_IN_TESTS_GEN_TASK, display_task["execution_tasks"][0])
+
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ @patch(ns("write_file"))
+ def test_tests_generated_if_a_file_changed(self, write_json_mock):
+ """
+ Given a git repository with changes,
+ When burn_in_tests is run,
+ Then tests are discovered to run.
+ """
+ # Note: this test is using actual tests and suites. So changes to those suites could
+ # introduce failures and require this test to be updated.
+ # You can see the test file it is using below. This test is used in the 'auth' and
+ # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass.
+ variant = "enterprise-rhel-80-64-bit-inmem"
+ repos = [mock_changed_git_files(["jstests/auth/auth1.js"])]
+ repeat_config = under_test.RepeatConfig()
+ gen_config = under_test.GenerateConfig(
+ variant,
+ "project",
+ ) # yapf: disable
+ mock_evg_conf = get_evergreen_config("etc/evergreen.yml")
+ mock_evg_api = MagicMock()
+
+ under_test.burn_in("task_id", variant, gen_config, repeat_config, mock_evg_api,
+ mock_evg_conf, repos, "testfile.json", 'install-dir/bin')
+
+ write_json_mock.assert_called_once()
+ written_config = json.loads(write_json_mock.call_args[0][1])
+ n_tasks = len(written_config["tasks"])
+ # Ensure we are generating at least one task for the test.
+ self.assertGreaterEqual(n_tasks, 1)
+
+ written_build_variant = written_config["buildvariants"][0]
+ self.assertEqual(variant, written_build_variant["name"])
+ self.assertEqual(n_tasks, len(written_build_variant["tasks"]))
+
+ display_task = written_build_variant["display_tasks"][0]
+ # The display task should contain all the generated tasks as well as 1 extra task for
+ # the burn_in_test_gen task.
+ self.assertEqual(n_tasks + 1, len(display_task["execution_tasks"]))
+
+
+class TestGenerateConfig(unittest.TestCase):
+ def test_run_build_variant_with_no_run_build_variant(self):
+ gen_config = under_test.GenerateConfig("build_variant", "project")
+
+ self.assertEqual(gen_config.build_variant, gen_config.run_build_variant)
+
+ def test_run_build_variant_with_run_build_variant(self):
+ gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant")
+
+ self.assertNotEqual(gen_config.build_variant, gen_config.run_build_variant)
+ self.assertEqual(gen_config.run_build_variant, "run_build_variant")
+
+ def test_validate_non_existing_build_variant(self):
+ evg_conf_mock = MagicMock()
+ evg_conf_mock.get_variant.return_value = None
+
+ gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant")
+
+ with self.assertRaises(ValueError):
+ gen_config.validate(evg_conf_mock)
+
+ def test_validate_existing_build_variant(self):
+ evg_conf_mock = MagicMock()
+
+ gen_config = under_test.GenerateConfig("build_variant", "project", "run_build_variant")
+ gen_config.validate(evg_conf_mock)
+
+ def test_validate_non_existing_run_build_variant(self):
+ evg_conf_mock = MagicMock()
+
+ gen_config = under_test.GenerateConfig("build_variant", "project")
+ gen_config.validate(evg_conf_mock)
+
+
+class TestParseAvgTestRuntime(unittest.TestCase):
+ def test__parse_avg_test_runtime(self):
+ task_avg_test_runtime_stats = [
+ teststats_utils.TestRuntime(test_name="dir/test1.js", runtime=30.2),
+ teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)
+ ]
+ result = under_test._parse_avg_test_runtime("dir/test2.js", task_avg_test_runtime_stats)
+ self.assertEqual(result, 455.1)
+
+
+class TestCalculateTimeout(unittest.TestCase):
+ def test__calculate_timeout(self):
+ avg_test_runtime = 455.1
+ expected_result = ceil(avg_test_runtime * under_test.AVG_TEST_TIME_MULTIPLIER)
+ self.assertEqual(expected_result, under_test._calculate_timeout(avg_test_runtime))
+
+ def test__calculate_timeout_avg_is_less_than_min(self):
+ avg_test_runtime = 10
+ self.assertEqual(under_test.MIN_AVG_TEST_TIME_SEC,
+ under_test._calculate_timeout(avg_test_runtime))
+
+
+class TestCalculateExecTimeout(unittest.TestCase):
+ def test__calculate_exec_timeout(self):
+ repeat_config = under_test.RepeatConfig(repeat_tests_secs=600)
+ avg_test_runtime = 455.1
+
+ exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime)
+
+ self.assertEqual(1771, exec_timeout)
+
+ def test_average_timeout_greater_than_execution_time(self):
+ repeat_config = under_test.RepeatConfig(repeat_tests_secs=600, repeat_tests_min=2)
+ avg_test_runtime = 750
+
+ exec_timeout = under_test._calculate_exec_timeout(repeat_config, avg_test_runtime)
+
+ # The timeout needs to be greater than the number of the test * the minimum number of runs.
+ minimum_expected_timeout = avg_test_runtime * repeat_config.repeat_tests_min
+
+ self.assertGreater(exec_timeout, minimum_expected_timeout)
+
+
+class TestGenerateTimeouts(unittest.TestCase):
+ def test__generate_timeouts(self):
+ repeat_config = under_test.RepeatConfig(repeat_tests_secs=600)
+ runtime_stats = [teststats_utils.TestRuntime(test_name="dir/test2.js", runtime=455.1)]
+ test_name = "dir/test2.js"
+
+ task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats)
+ timeout_info = task_generator.generate_timeouts(test_name)
+
+ self.assertEqual(timeout_info.exec_timeout, 1771)
+ self.assertEqual(timeout_info.timeout, 1366)
+
+ def test__generate_timeouts_no_results(self):
+ repeat_config = under_test.RepeatConfig(repeat_tests_secs=600)
+ runtime_stats = []
+ test_name = "dir/new_test.js"
+
+ task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats)
+ timeout_info = task_generator.generate_timeouts(test_name)
+
+ self.assertIsNone(timeout_info.cmd)
+
+ def test__generate_timeouts_avg_runtime_is_zero(self):
+ repeat_config = under_test.RepeatConfig(repeat_tests_secs=600)
+ runtime_stats = [
+ teststats_utils.TestRuntime(test_name="dir/test_with_zero_runtime.js", runtime=0)
+ ]
+ test_name = "dir/test_with_zero_runtime.js"
+
+ task_generator = under_test.BurnInGenTaskService(MagicMock(), repeat_config, runtime_stats)
+ timeout_info = task_generator.generate_timeouts(test_name)
+
+ self.assertIsNone(timeout_info.cmd)
+
+
+class TestGetTaskRuntimeHistory(unittest.TestCase):
+ def test_get_task_runtime_history(self):
+ mock_evg_api = MagicMock()
+ mock_evg_api.test_stats_by_project.return_value = [
+ MagicMock(
+ test_file="dir/test2.js",
+ task_name="task1",
+ variant="variant1",
+ distro="distro1",
+ date=datetime.utcnow().date(),
+ num_pass=1,
+ num_fail=0,
+ avg_duration_pass=10.1,
+ )
+ ]
+ analysis_duration = under_test.AVG_TEST_RUNTIME_ANALYSIS_DAYS
+ end_date = datetime.utcnow().replace(microsecond=0)
+ start_date = end_date - timedelta(days=analysis_duration)
+ mock_gen_config = MagicMock(project="project1", build_variant="variant1")
+
+ executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api,
+ history_end_date=end_date)
+ result = executor.get_task_runtime_history("task1")
+
+ self.assertEqual(result, [("dir/test2.js", 10.1)])
+ mock_evg_api.test_stats_by_project.assert_called_with(
+ "project1", after_date=start_date, before_date=end_date, group_by="test",
+ group_num_days=14, tasks=["task1"], variants=["variant1"])
+
+ def test_get_task_runtime_history_evg_degraded_mode_error(self):
+ mock_response = MagicMock(status_code=requests.codes.SERVICE_UNAVAILABLE)
+ mock_evg_api = MagicMock()
+ mock_evg_api.test_stats_by_project.side_effect = requests.HTTPError(response=mock_response)
+ mock_gen_config = MagicMock(project="project1", build_variant="variant1")
+
+ executor = under_test.GenerateBurnInExecutor(mock_gen_config, MagicMock(), mock_evg_api)
+ result = executor.get_task_runtime_history("task1")
+
+ self.assertEqual(result, [])
+
+
+TESTS_BY_TASK = {
+ "task1": {
+ "resmoke_args": "--suites=suite1",
+ "tests": ["jstests/test1.js", "jstests/test2.js"]},
+ "task2": {
+ "resmoke_args": "--suites=suite1",
+ "tests": ["jstests/test1.js", "jstests/test3.js"]},
+ "task3": {
+ "resmoke_args": "--suites=suite3",
+ "tests": ["jstests/test4.js", "jstests/test5.js"]},
+ "task4": {
+ "resmoke_args": "--suites=suite4", "tests": []},
+} # yapf: disable
+
+
+def create_tests_by_task_mock(n_tasks, n_tests):
+ return {
+ f"task_{i}_gen": under_test.TaskInfo(display_task_name=f"task_{i}", resmoke_args="", tests=[
+ f"jstests/tests_{j}" for j in range(n_tests)
+ ], require_multiversion_setup=False, distro=f"distro_{i}", build_variant="variant",
+ suite=f"suite_{i}")
+ for i in range(n_tasks)
+ }
+
+
+class TestCreateGenerateTasksConfig(unittest.TestCase):
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ def test_no_tasks_given(self):
+ build_variant = BuildVariant("build variant")
+ gen_config = MagicMock(run_build_variant="variant")
+ repeat_config = MagicMock()
+ mock_evg_api = MagicMock()
+
+ executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api)
+ executor.generate_tasks_for_variant({}, build_variant)
+
+ evg_config_dict = build_variant.as_dict()
+ self.assertEqual(0, len(evg_config_dict["tasks"]))
+
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ def test_one_task_one_test(self):
+ n_tasks = 1
+ n_tests = 1
+ resmoke_options = "options for resmoke"
+ build_variant = BuildVariant("build variant")
+ gen_config = MagicMock(run_build_variant="variant", distro=None)
+ repeat_config = MagicMock()
+ repeat_config.generate_resmoke_options.return_value = resmoke_options
+ mock_evg_api = MagicMock()
+ tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
+
+ executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api)
+ executor.generate_tasks_for_variant(tests_by_task, build_variant)
+
+ shrub_config = ShrubProject.empty().add_build_variant(build_variant)
+ evg_config_dict = shrub_config.as_dict()
+ tasks = evg_config_dict["tasks"]
+ self.assertEqual(n_tasks * n_tests, len(tasks))
+ cmd = tasks[0]["commands"]
+ self.assertIn(resmoke_options, cmd[2]["vars"]["resmoke_args"])
+ self.assertEqual("suite_0", cmd[2]["vars"]["suite"])
+ self.assertIn("tests_0", cmd[2]["vars"]["resmoke_args"])
+
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ def test_n_task_m_test(self):
+ n_tasks = 3
+ n_tests = 5
+ build_variant = BuildVariant("build variant")
+ gen_config = MagicMock(run_build_variant="variant", distro=None)
+ repeat_config = MagicMock()
+ tests_by_task = create_tests_by_task_mock(n_tasks, n_tests)
+ mock_evg_api = MagicMock()
+
+ executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api)
+ executor.generate_tasks_for_variant(tests_by_task, build_variant)
+
+ evg_config_dict = build_variant.as_dict()
+ self.assertEqual(n_tasks * n_tests, len(evg_config_dict["tasks"]))
+
+
+class TestCreateGenerateTasksFile(unittest.TestCase):
+ @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
+ @patch(ns("sys.exit"))
+ @patch(ns("validate_task_generation_limit"))
+ def test_cap_on_task_generate(self, validate_mock, exit_mock):
+ gen_config = MagicMock(require_multiversion_setup=False)
+ repeat_config = MagicMock()
+ tests_by_task = MagicMock()
+ mock_evg_api = MagicMock()
+
+ validate_mock.return_value = False
+
+ exit_mock.side_effect = ValueError("exiting")
+ with self.assertRaises(ValueError):
+ executor = under_test.GenerateBurnInExecutor(gen_config, repeat_config, mock_evg_api,
+ "gen_file.json")
+ executor.execute(tests_by_task)
+
+ exit_mock.assert_called_once()
+
+
+class TestFindChangedTests(unittest.TestCase):
+ def test_manual_tests_should_be_specifiable_via_env_vars(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ mock_repo = MagicMock()
+ mock_env = {
+ "BURN_IN_TESTS": "jstests/auth/auth1.js,jstests/core/where1.js",
+ }
+ change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env)
+
+ test_set = change_detector.find_changed_tests([mock_repo])
+
+ self.assertEqual(2, len(test_set))
+ self.assertIn("jstests/auth/auth1.js", test_set)
+ self.assertIn("jstests/core/where1.js", test_set)
+
+ def test_empty_env_should_not_add_extra_tests(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ mock_repo = MagicMock()
+ mock_env = {}
+ change_detector = under_test.EvergreenFileChangeDetector("task_id", mock_evg_api, mock_env)
+
+ test_set = change_detector.find_changed_tests([mock_repo])
+
+ self.assertEqual(set(), test_set)
diff --git a/buildscripts/tests/test_evergreen_gen_build_variant.py b/buildscripts/tests/test_evergreen_gen_build_variant.py
new file mode 100644
index 00000000000..7ece404ab36
--- /dev/null
+++ b/buildscripts/tests/test_evergreen_gen_build_variant.py
@@ -0,0 +1,382 @@
+"""Unit tests for the generate_resmoke_suite script."""
+import unittest
+
+from mock import MagicMock
+
+from buildscripts import evergreen_gen_build_variant as under_test
+from buildscripts.ciconfig.evergreen import Variant, Task
+
+# pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use,protected-access
+# pylint: disable=too-many-locals,too-many-lines,too-many-public-methods,no-value-for-parameter
+
+
+def build_mock_build_variant(expansions=None, task_list=None):
+ task_spec_list = [{"name": task.name} for task in task_list] if task_list else []
+ config = {
+ "tasks": task_spec_list,
+ }
+ if expansions:
+ config["expansions"] = expansions
+
+ if task_list is None:
+ task_list = []
+ task_map = {task.name: task for task in task_list}
+
+ return Variant(config, task_map, {})
+
+
+def build_mock_task(name, run_vars=None, depends_on=None):
+ config = {
+ "name":
+ name, "commands": [
+ {"func": "do setup"},
+ {
+ "func": "generate resmoke tasks",
+ "vars": run_vars if run_vars else {},
+ },
+ ]
+ }
+
+ if depends_on is not None:
+ config["depends_on"] = depends_on
+ return Task(config)
+
+
+def build_mock_project_config(variant=None, task_defs=None):
+ mock_project = MagicMock()
+ if variant:
+ mock_project.get_variant.return_value = variant
+
+ if task_defs:
+ mock_project.get_task.side_effect = task_defs
+
+ return mock_project
+
+
+def build_mock_expansions():
+ mock_expansions = MagicMock()
+ mock_expansions.config_location.return_value = "/path/to/config"
+ mock_expansions.get_max_sub_suites.return_value = 998
+ mock_expansions.task_name = "generating_task"
+ return mock_expansions
+
+
+def build_mock_evg_api(build_task_list):
+ mock_evg_api = MagicMock()
+ mock_evg_api.build_by_id.return_value.get_tasks.return_value = build_task_list
+ return mock_evg_api
+
+
+def build_mock_orchestrator(build_expansions=None, task_def_list=None, build_task_list=None):
+ if build_expansions is None:
+ build_expansions = {}
+ if task_def_list is None:
+ task_def_list = []
+ if build_task_list is None:
+ build_task_list = []
+
+ mock_build_variant = build_mock_build_variant(build_expansions, task_def_list)
+ mock_project = build_mock_project_config(mock_build_variant, task_def_list)
+ mock_evg_expansions = build_mock_expansions()
+ mock_evg_api = build_mock_evg_api(build_task_list)
+
+ return under_test.GenerateBuildVariantOrchestrator(
+ gen_task_validation=MagicMock(),
+ gen_task_options=MagicMock(),
+ evg_project_config=mock_project,
+ evg_expansions=mock_evg_expansions,
+ evg_api=mock_evg_api,
+ )
+
+
+class TestEvgExpansions(unittest.TestCase):
+ def test_get_max_sub_suites_should_use_patch_value_in_patches(self):
+ evg_expansions = under_test.EvgExpansions(
+ is_patch=True,
+ max_sub_suites=5,
+ mainline_max_sub_suites=1,
+ build_id="build_id",
+ build_variant="build_variant",
+ project="project",
+ revision="revision",
+ task_name="task_name",
+ task_id="task_id",
+ )
+
+ self.assertEqual(evg_expansions.get_max_sub_suites(), evg_expansions.max_sub_suites)
+
+ def test_get_max_sub_suites_should_use_mainline_value_in_non_patches(self):
+ evg_expansions = under_test.EvgExpansions(
+ is_patch=False,
+ max_sub_suites=5,
+ mainline_max_sub_suites=1,
+ build_id="build_id",
+ build_variant="build_variant",
+ project="project",
+ revision="revision",
+ task_name="task_name",
+ task_id="task_id",
+ )
+
+ self.assertEqual(evg_expansions.get_max_sub_suites(),
+ evg_expansions.mainline_max_sub_suites)
+
+ def test_get_max_sub_suites_should_use_mainline_value_if_patch_status_unknown(self):
+ evg_expansions = under_test.EvgExpansions(
+ is_patch=None,
+ max_sub_suites=5,
+ mainline_max_sub_suites=1,
+ build_id="build_id",
+ build_variant="build_variant",
+ project="project",
+ revision="revision",
+ task_name="task_name",
+ task_id="task_id",
+ )
+
+ self.assertEqual(evg_expansions.get_max_sub_suites(),
+ evg_expansions.mainline_max_sub_suites)
+
+
+class TestTranslateRunVar(unittest.TestCase):
+ def test_normal_value_should_be_returned(self):
+ run_var = "some value"
+ mock_build_variant = build_mock_build_variant()
+ self.assertEqual(run_var, under_test.translate_run_var(run_var, mock_build_variant))
+
+ def test_expansion_should_be_returned_from_build_variant(self):
+ run_var = "${my_expansion}"
+ value = "my value"
+ mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value})
+ self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant))
+
+ def test_expansion_not_found_should_return_none(self):
+ run_var = "${my_expansion}"
+ mock_build_variant = build_mock_build_variant(expansions={})
+ self.assertIsNone(under_test.translate_run_var(run_var, mock_build_variant))
+
+ def test_expansion_not_found_should_return_default(self):
+ run_var = "${my_expansion|default}"
+ mock_build_variant = build_mock_build_variant(expansions={})
+ self.assertEqual("default", under_test.translate_run_var(run_var, mock_build_variant))
+
+ def test_expansion_should_be_returned_from_build_variant_even_with_default(self):
+ run_var = "${my_expansion|default}"
+ value = "my value"
+ mock_build_variant = build_mock_build_variant(expansions={"my_expansion": value})
+ self.assertEqual(value, under_test.translate_run_var(run_var, mock_build_variant))
+
+
+class TestTaskDefToSplitParams(unittest.TestCase):
+ def test_params_should_be_generated(self):
+ run_vars = {
+ "resmoke_args": "run tests",
+ }
+ mock_task_def = build_mock_task("my_task", run_vars)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+
+ split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant")
+
+ self.assertEqual("build_variant", split_param.build_variant)
+ self.assertEqual("my_task", split_param.task_name)
+ self.assertEqual("my_task", split_param.suite_name)
+ self.assertEqual("my_task", split_param.filename)
+
+ def test_params_should_allow_suite_to_be_overridden(self):
+ run_vars = {
+ "resmoke_args": "run tests",
+ "suite": "the suite",
+ }
+ mock_task_def = build_mock_task("my_task", run_vars)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+
+ split_param = mock_orchestrator.task_def_to_split_params(mock_task_def, "build_variant")
+
+ self.assertEqual("build_variant", split_param.build_variant)
+ self.assertEqual("my_task", split_param.task_name)
+ self.assertEqual("the suite", split_param.suite_name)
+ self.assertEqual("the suite", split_param.filename)
+
+
+class TestDetermineTaskDependencies(unittest.TestCase):
+ def test_running_task_should_not_be_included_in_depends_on(self):
+ run_vars = {
+ "resmoke_args": "run tests",
+ }
+ depends_on = [
+ {"name": "compile"},
+ {"name": "build_variant_gen"},
+ ]
+ mock_task_def = build_mock_task("my_task", run_vars, depends_on=depends_on)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+
+ dependencies = mock_orchestrator.determine_task_dependencies(mock_task_def)
+
+ self.assertIn("compile", dependencies)
+ self.assertNotIn("generating_task", dependencies)
+
+
+class TestTaskDefToGenParams(unittest.TestCase):
+ def test_params_should_be_generated(self):
+ run_vars = {
+ "resmoke_args": "run tests",
+ }
+ mock_task_def = build_mock_task("my_task", run_vars)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+
+ gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant")
+
+ self.assertFalse(gen_params.require_multiversion_setup)
+ self.assertEqual("run tests", gen_params.resmoke_args)
+ self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value,
+ gen_params.config_location)
+ self.assertIsNone(gen_params.large_distro_name)
+ self.assertFalse(gen_params.use_large_distro)
+
+ def test_params_should_be_overwritable(self):
+ run_vars = {
+ "resmoke_args": "run tests",
+ "use_large_distro": "true",
+ }
+ mock_task_def = build_mock_task("my_task", run_vars)
+ build_expansions = {"large_distro_name": "my large distro"}
+ mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions,
+ task_def_list=[mock_task_def])
+ gen_params = mock_orchestrator.task_def_to_gen_params(mock_task_def, "build_variant")
+
+ self.assertFalse(gen_params.require_multiversion_setup)
+ self.assertEqual("run tests", gen_params.resmoke_args)
+ self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value,
+ gen_params.config_location)
+ self.assertEqual("my large distro", gen_params.large_distro_name)
+ self.assertTrue(gen_params.use_large_distro)
+
+
+class TestTaskDefToFuzzerParams(unittest.TestCase):
+ def test_params_should_be_generated(self):
+ run_vars = {
+ "num_files": "5",
+ "num_tasks": "3",
+ }
+ mock_task_def = build_mock_task("my_fuzzer_gen", run_vars)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+ fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant")
+
+ self.assertEqual("my_fuzzer", fuzzer_params.task_name)
+ self.assertEqual(5, fuzzer_params.num_files)
+ self.assertEqual(3, fuzzer_params.num_tasks)
+ self.assertEqual("jstestfuzz", fuzzer_params.npm_command)
+ self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value,
+ fuzzer_params.config_location)
+ self.assertIsNone(fuzzer_params.large_distro_name)
+ self.assertFalse(fuzzer_params.use_large_distro)
+
+ def test_num_tasks_respects_max_sub_suites(self):
+ run_vars = {
+ "num_files": "5",
+ "num_tasks": "3",
+ }
+ mock_task_def = build_mock_task("my_fuzzer_gen", run_vars)
+ mock_orchestrator = build_mock_orchestrator(task_def_list=[mock_task_def])
+ mock_orchestrator.evg_expansions.get_max_sub_suites.return_value = 1
+ fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant")
+
+ self.assertEqual(5, fuzzer_params.num_files)
+ self.assertEqual(1, fuzzer_params.num_tasks)
+
+ def test_params_should_be_overwritable(self):
+ run_vars = {
+ "num_files": "${file_count|8}",
+ "num_tasks": "3",
+ "use_large_distro": "true",
+ "npm_command": "aggfuzzer",
+ }
+ mock_task_def = build_mock_task("my_fuzzer_gen", run_vars)
+ build_expansions = {"large_distro_name": "my large distro"}
+ mock_orchestrator = build_mock_orchestrator(build_expansions=build_expansions,
+ task_def_list=[mock_task_def])
+
+ fuzzer_params = mock_orchestrator.task_def_to_fuzzer_params(mock_task_def, "build_variant")
+
+ self.assertEqual("my_fuzzer", fuzzer_params.task_name)
+ self.assertEqual(8, fuzzer_params.num_files)
+ self.assertEqual(3, fuzzer_params.num_tasks)
+ self.assertEqual("aggfuzzer", fuzzer_params.npm_command)
+ self.assertEqual(mock_orchestrator.evg_expansions.config_location.return_value,
+ fuzzer_params.config_location)
+ self.assertEqual("my large distro", fuzzer_params.large_distro_name)
+ self.assertTrue(fuzzer_params.use_large_distro)
+
+
+class TestGenerateBuildVariant(unittest.TestCase):
+ def test_a_whole_build_variant(self):
+ gen_run_vars = {
+ "resmoke_args": "run tests",
+ }
+ mv_gen_run_vars = {
+ "resmoke_args": "run tests",
+ "suite": "some suite",
+ }
+ fuzz_run_vars = {
+ "num_files": "5",
+ "num_tasks": "3",
+ "is_jstestfuzz": "true",
+ }
+ mv_fuzz_run_vars = {
+ "num_files": "5",
+ "num_tasks": "3",
+ "is_jstestfuzz": "true",
+ "suite": "aggfuzzer",
+ }
+ mock_task_defs = [
+ build_mock_task("my_gen_task", gen_run_vars),
+ build_mock_task("my_fuzzer_task", fuzz_run_vars),
+ build_mock_task("my_mv_fuzzer_task", mv_fuzz_run_vars),
+ build_mock_task("my_mv_gen_task", mv_gen_run_vars),
+ ]
+ mock_orchestrator = build_mock_orchestrator(task_def_list=mock_task_defs)
+ builder = MagicMock()
+
+ builder = mock_orchestrator.generate_build_variant(builder, "build variant")
+
+ self.assertEqual(builder.generate_suite.call_count, 2)
+ self.assertEqual(builder.generate_fuzzer.call_count, 2)
+
+
+class TestAdjustTaskPriority(unittest.TestCase):
+ def test_task_is_updates(self):
+ starting_priority = 42
+ task_id = "task 314"
+ mock_task = MagicMock(task_id=task_id, priority=starting_priority)
+ mock_orchestrator = build_mock_orchestrator()
+
+ mock_orchestrator.adjust_task_priority(mock_task)
+
+ mock_orchestrator.evg_api.configure_task.assert_called_with(task_id,
+ priority=starting_priority + 1)
+
+ def test_task_should_only_reach_99(self):
+ starting_priority = 99
+ task_id = "task 314"
+ mock_task = MagicMock(task_id=task_id, priority=starting_priority)
+ mock_orchestrator = build_mock_orchestrator()
+
+ mock_orchestrator.adjust_task_priority(mock_task)
+
+ mock_orchestrator.evg_api.configure_task.assert_called_with(task_id,
+ priority=starting_priority)
+
+
+class TestAdjustGenTasksPriority(unittest.TestCase):
+ def test_gen_tasks_in_task_list_are_adjusted(self):
+ gen_tasks = {"task_3", "task_8", "task_13"}
+ n_build_tasks = 25
+ mock_task_list = [
+ MagicMock(build_variant='dummy_variant', display_name=f"task_{i}", priority=0)
+ for i in range(n_build_tasks)
+ ]
+ mock_orchestrator = build_mock_orchestrator(build_task_list=mock_task_list)
+
+ n_tasks_adjusted = mock_orchestrator.adjust_gen_tasks_priority(gen_tasks)
+
+ self.assertEqual(len(gen_tasks), n_tasks_adjusted)
diff --git a/buildscripts/tests/test_evergreen_task_timeout.py b/buildscripts/tests/test_evergreen_task_timeout.py
index 294c50d0a17..b3e4e201d96 100644
--- a/buildscripts/tests/test_evergreen_task_timeout.py
+++ b/buildscripts/tests/test_evergreen_task_timeout.py
@@ -110,7 +110,7 @@ class TestTimeoutOverrides(unittest.TestCase):
class TestDetermineExecTimeout(unittest.TestCase):
def _validate_exec_timeout(self, idle_timeout, exec_timeout, historic_timeout, evg_alias,
- build_variant, display_name, timeout_override, expected_timeout):
+ build_variant, timeout_override, expected_timeout):
task_name = "task_name"
variant = build_variant
overrides = {}
@@ -121,9 +121,8 @@ class TestDetermineExecTimeout(unittest.TestCase):
orchestrator = under_test.TaskTimeoutOrchestrator(
timeout_service=MagicMock(spec_set=TimeoutService),
- timeout_overrides=mock_timeout_overrides, evg_project_config=MagicMock(
- spec_set=EvergreenProjectConfig,
- get_variant=MagicMock(return_value=MagicMock(display_name=display_name))))
+ timeout_overrides=mock_timeout_overrides,
+ evg_project_config=MagicMock(spec_set=EvergreenProjectConfig))
actual_timeout = orchestrator.determine_exec_timeout(
task_name, variant, idle_timeout, exec_timeout, evg_alias, historic_timeout)
@@ -133,83 +132,78 @@ class TestDetermineExecTimeout(unittest.TestCase):
def test_timeout_used_if_specified(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=42),
historic_timeout=None, evg_alias=None, build_variant="variant",
- display_name="not required", timeout_override=None,
- expected_timeout=timedelta(seconds=42))
+ timeout_override=None, expected_timeout=timedelta(seconds=42))
def test_default_is_returned_with_no_timeout(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None,
- evg_alias=None, build_variant="variant",
- display_name="not required", timeout_override=None,
+ evg_alias=None, build_variant="variant", timeout_override=None,
expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT)
def test_default_is_returned_with_timeout_at_zero(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0),
historic_timeout=None, evg_alias=None, build_variant="variant",
- display_name="not required", timeout_override=None,
+ timeout_override=None,
expected_timeout=under_test.DEFAULT_NON_REQUIRED_BUILD_TIMEOUT)
def test_default_required_returned_on_required_variants(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None,
evg_alias=None, build_variant="variant-required",
- display_name="! required", timeout_override=None,
+ timeout_override=None,
expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT)
def test_override_on_required_should_use_override(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None,
evg_alias=None, build_variant="variant-required",
- display_name="! required", timeout_override=3 * 60,
+ timeout_override=3 * 60,
expected_timeout=timedelta(minutes=3 * 60))
def test_task_specific_timeout(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=timedelta(seconds=0),
historic_timeout=None, evg_alias=None, build_variant="variant",
- display_name="not required", timeout_override=60,
- expected_timeout=timedelta(minutes=60))
+ timeout_override=60, expected_timeout=timedelta(minutes=60))
def test_commit_queue_items_use_commit_queue_timeout(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None, historic_timeout=None,
evg_alias=under_test.COMMIT_QUEUE_ALIAS,
- build_variant="variant", display_name="not required",
- timeout_override=None,
+ build_variant="variant", timeout_override=None,
expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT)
def test_use_idle_timeout_if_greater_than_exec_timeout(self):
self._validate_exec_timeout(
idle_timeout=timedelta(hours=2), exec_timeout=timedelta(minutes=10),
- historic_timeout=None, evg_alias=None, build_variant="variant",
- display_name="not required", timeout_override=None, expected_timeout=timedelta(hours=2))
+ historic_timeout=None, evg_alias=None, build_variant="variant", timeout_override=None,
+ expected_timeout=timedelta(hours=2))
def test_historic_timeout_should_be_used_if_given(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None,
historic_timeout=timedelta(minutes=15), evg_alias=None,
- build_variant="variant", display_name="not required",
- timeout_override=None, expected_timeout=timedelta(minutes=15))
+ build_variant="variant", timeout_override=None,
+ expected_timeout=timedelta(minutes=15))
def test_commit_queue_should_override_historic_timeouts(self):
self._validate_exec_timeout(
idle_timeout=None, exec_timeout=None, historic_timeout=timedelta(minutes=15),
- evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant",
- display_name="not required", timeout_override=None,
+ evg_alias=under_test.COMMIT_QUEUE_ALIAS, build_variant="variant", timeout_override=None,
expected_timeout=under_test.COMMIT_QUEUE_TIMEOUT)
def test_override_should_override_historic_timeouts(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None,
historic_timeout=timedelta(minutes=15), evg_alias=None,
- build_variant="variant", display_name="not required",
- timeout_override=33, expected_timeout=timedelta(minutes=33))
+ build_variant="variant", timeout_override=33,
+ expected_timeout=timedelta(minutes=33))
def test_historic_timeout_should_not_be_overridden_by_required_bv(self):
self._validate_exec_timeout(idle_timeout=None, exec_timeout=None,
historic_timeout=timedelta(minutes=15), evg_alias=None,
- build_variant="variant-required", display_name="! required",
- timeout_override=None, expected_timeout=timedelta(minutes=15))
+ build_variant="variant-required", timeout_override=None,
+ expected_timeout=timedelta(minutes=15))
def test_historic_timeout_should_not_be_increase_required_bv_timeout(self):
self._validate_exec_timeout(
idle_timeout=None, exec_timeout=None,
historic_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT + timedelta(minutes=30),
- evg_alias=None, build_variant="variant-required", display_name="! required",
- timeout_override=None, expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT)
+ evg_alias=None, build_variant="variant-required", timeout_override=None,
+ expected_timeout=under_test.DEFAULT_REQUIRED_BUILD_TIMEOUT)
class TestDetermineIdleTimeout(unittest.TestCase):
diff --git a/buildscripts/tests/test_jepsen_report.py b/buildscripts/tests/test_jepsen_report.py
deleted file mode 100644
index cdfb095ba5a..00000000000
--- a/buildscripts/tests/test_jepsen_report.py
+++ /dev/null
@@ -1,195 +0,0 @@
-"""Tests for jepsen report generator."""
-import unittest
-import textwrap
-import random
-import os
-from unittest.mock import patch, mock_open, MagicMock
-from click.testing import CliRunner
-
-from buildscripts.jepsen_report import parse, ParserOutput, main
-
-_CORPUS = textwrap.dedent("""\
- "indeterminate: Command failed with error 251 (NoSuchTransaction): 'Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1' on server n9:27017. The full response is {\"writeConcernError\": {\"code\": 6, \"codeName\": \"HostUnreachable\", \"errmsg\": \"operation was interrupted\", \"errInfo\": {\"writeConcern\": {\"w\": \"majority\", \"wtimeout\": 0, \"provenance\": \"clientSupplied\"}}}, \"topologyVersion\": {\"processId\": {\"$oid\": \"625f0b0ef9d6a12d9b562ff9\"}, \"counter\": 21}, \"ok\": 0.0, \"errmsg\": \"Transaction was aborted :: caused by :: from shard rs_shard2 :: caused by :: Given transaction number 53 does not match any in-progress transactions. The active transaction number is -1\", \"code\": 251, \"codeName\": \"NoSuchTransaction\", \"$clusterTime\": {\"clusterTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 16}}, \"signature\": {\"hash\": {\"$binary\": {\"base64\": \"AAAAAAAAAAAAAAAAAAAAAAAAAAA=\", \"subType\": \"00\"}}, \"keyId\": 0}}, \"operationTime\": {\"$timestamp\": {\"t\": 1650395950, \"i\": 5}}, \"recoveryToken\": {\"recoveryShardId\": \"rs_shard1\"}}",
- :index 1141}})},
- :workload {:valid? true},
- :valid? true}
-
-
-Everything looks good! ヽ(‘ー`)ノ
-
-
-
-# Successful tests
-
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T163539.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164131.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T164724.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165317.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T165910.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T170503.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171055.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T171648.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172239.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T172832.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T173634.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174227.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T174820.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T175413.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180006.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T180558.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181148.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T181946.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T182539.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183131.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T183720.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184313.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T184906.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T185459.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190052.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T190645.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191239.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T191833.000Z
-
-# Crashed tests
-
-mongodb list-append w:majority r:majority tw:majority tr:snapshot partition
-mongodb list-append w:majority r:majority tw:majority tr:snapshot partition
-
-28 successes
-0 unknown
-2 crashed
-0 failures
-""").splitlines()
-
-
-class TestParser(unittest.TestCase):
- """TestParser."""
-
- @classmethod
- def _corpus_generator(cls):
- n_pass = random.randint(1, 30)
- n_fail = random.randint(1, 30)
- n_unknown = random.randint(1, 30)
- n_crash = random.randint(1, 30)
-
- corpus = textwrap.dedent("""\
- :index 2893}})},
- :workload {:valid? true},
- :valid? true}
-Everything looks good! ヽ(‘ー`)ノ
-""")
- successful_tests = []
- if n_pass > 0:
- corpus += "\n# Successful tests\n\n"
- for i in range(0, n_pass):
- corpus += f"test {i}\n"
- successful_tests.append(f"test {i}")
-
- failed_tests = []
- if n_fail > 0:
- corpus += "\n# Failed tests\n\n"
- for i in range(0, n_fail):
- corpus += f"test {i}\n"
- failed_tests.append(f"test {i}")
-
- indeterminate_tests = []
- if n_unknown > 0:
- corpus += "\n# Indeterminate tests\n\n"
- for i in range(0, n_unknown):
- corpus += f"test {i}\n"
- indeterminate_tests.append(f"test {i}")
-
- crashed_tests = []
- if n_crash > 0:
- corpus += "\n# Crashed tests\n\n"
- for i in range(0, n_crash):
- corpus += f"test {i}\n"
- crashed_tests.append(f"test {i}")
- # note leading newline for this block is required to match the actual
- # logs
- corpus += textwrap.dedent(f"""
-{n_pass} successes
-{n_unknown} unknown
-{n_crash} crashed
-{n_fail} failures
-""")
-
- return {
- 'expected':
- ParserOutput({
- 'success': successful_tests,
- 'unknown': indeterminate_tests,
- 'crashed': crashed_tests,
- 'failed': failed_tests,
- }), 'corpus':
- corpus
- }
-
- def test_parser(self):
- """Test with embedded corpus."""
- out = parse(_CORPUS)
- self.assertEqual(len(out['success']), 28)
- self.assertEqual(len(out['unknown']), 0)
- self.assertEqual(len(out['crashed']), 2)
- self.assertEqual(len(out['failed']), 0)
-
- def test_parser2(self):
- """Test with jepsen.log file."""
- with open(os.path.join(os.path.dirname(__file__),
- "test_jepsen_report_corpus.log.txt")) as fh:
- corpus = fh.read().splitlines()
- out = parse(corpus)
- self.assertEqual(len(out['success']), 29)
- self.assertEqual(len(out['unknown']), 0)
- self.assertEqual(len(out['crashed']), 1)
- self.assertEqual(len(out['failed']), 0)
-
- def test_generated_corpus(self):
- """Generate 100 corpuses and test them."""
- for _ in range(0, 100):
- self._test_generated_corpus()
-
- def _test_generated_corpus(self):
- gen = self._corpus_generator()
- corpus = gen['corpus'].splitlines()
- out = parse(corpus)
- self.assertDictEqual(out, gen['expected'])
-
- @patch('buildscripts.jepsen_report._try_find_log_file')
- @patch('buildscripts.jepsen_report._get_log_lines')
- @patch('buildscripts.jepsen_report._put_report')
- def test_main(self, mock_put_report, mock_get_log_lines, mock_try_find_log_file):
- """Test main function."""
- gen = self._corpus_generator()
- corpus = gen['corpus'].splitlines()
- mock_get_log_lines.return_value = corpus
-
- def _try_find_log_file(_store, _test):
- if _try_find_log_file.counter == 0:
- _try_find_log_file.counter += 1
- with open(
- os.path.join(
- os.path.dirname(__file__), "test_jepsen_report_corpus.log.txt")) as fh:
- return fh.read()
- return ""
-
- _try_find_log_file.counter = 0
- mock_try_find_log_file.side_effect = _try_find_log_file
-
- runner = CliRunner()
- result = runner.invoke(main,
- ["--start_time=0", "--end_time=10", "--elapsed=10", "test.log"])
- num_tests = len(gen['expected']['success']) + len(gen['expected']['unknown']) + len(
- gen['expected']['crashed']) + len(gen['expected']['failed'])
- num_fails = num_tests - len(gen['expected']['success'])
-
- callee_dict = mock_put_report.call_args[0][0]
- self.assertEqual(callee_dict['failures'], num_fails)
- self.assertEqual(len(callee_dict['results']), num_tests)
- mock_get_log_lines.assert_called_once_with('test.log')
- if gen['expected']['crashed']:
- self.assertEqual(result.exit_code, 2)
- elif gen['expected']['unknown'] or gen['expected']['failure']:
- self.assertEqual(result.exit_code, 1)
- else:
- self.assertEqual(result.exit_code, 0)
diff --git a/buildscripts/tests/test_jepsen_report_corpus.log.txt b/buildscripts/tests/test_jepsen_report_corpus.log.txt
deleted file mode 100644
index 89d4e91892b..00000000000
--- a/buildscripts/tests/test_jepsen_report_corpus.log.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-INFO [2022-04-20 02:46:34,505] jepsen test runner - jepsen.core {:perf
- {:latency-graph {:valid? true},
- :rate-graph {:valid? true},
- :valid? true},
- :clock {:valid? true},
- :stats
- {:valid? true,
- :count 5768,
- :ok-count 4930,
- :fail-count 0,
- :info-count 838,
- :by-f
- {:txn
- {:valid? true,
- :count 5768,
- :ok-count 4930,
- :fail-count 0,
- :info-count 838}}},
- :exceptions {:valid? true},
- :workload {:valid? true},
- :valid? true}
-
-
-Everything looks good! ヽ(‘ー`)ノ
-
-
-
-# Successful tests
-
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235350.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220419T235940.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T000533.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001131.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T001724.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002316.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T002909.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T003502.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004057.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T004651.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005244.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T005837.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T010429.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011021.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T011617.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012210.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T012802.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T013356.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014154.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T014746.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015338.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T015931.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T020525.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021116.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T021708.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022301.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T022854.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T023446.000Z
-store/mongodb list-append w:majority r:majority tw:majority tr:snapshot partition/20220420T024039.000Z
-
-# Crashed tests
-
-mongodb list-append w:majority r:majority tw:majority tr:snapshot partition
-
-29 successes
-0 unknown
-1 crashed
-0 failures
diff --git a/buildscripts/tests/test_mongosymb.py b/buildscripts/tests/test_mongosymb.py
deleted file mode 100644
index 188ea385c5f..00000000000
--- a/buildscripts/tests/test_mongosymb.py
+++ /dev/null
@@ -1,68 +0,0 @@
-"""Unit tests for buildscripts/mongosymb.py."""
-# pylint: disable=missing-docstring
-import unittest
-
-from buildscripts import mongosymb as under_test
-
-
-class TestGetVersion(unittest.TestCase):
- def test_get_version_with_patch(self):
- trace_doc = {
- "processInfo": {
- "mongodbVersion": "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169"
- }
- }
- version = under_test.get_version(trace_doc)
- self.assertEqual(version, "6.0.0-alpha0-37-ge1d28c1-patch-6257e60a32f417196bc25169")
-
- def test_get_version_without_patch(self):
- trace_doc = {"processInfo": {"mongodbVersion": "6.1.0-alpha-504-g0c8a142"}}
- version = under_test.get_version(trace_doc)
- self.assertEqual(version, "6.1.0-alpha-504-g0c8a142")
-
- def test_get_version_no_mongodb_version(self):
- trace_doc = {"processInfo": {}}
- version = under_test.get_version(trace_doc)
- self.assertEqual(version, None)
-
- def test_get_version_no_process_info(self):
- trace_doc = {}
- version = under_test.get_version(trace_doc)
- self.assertEqual(version, None)
-
-
-class TestHasHighNotFoundPathsRatio(unittest.TestCase):
- def test_not_found_paths_ratio_is_more_than_0_5(self):
- frames = [
- {"path": "some/path"},
- {"path": "some/path"},
- {"path": "some/path"},
- {"path": None},
- ]
- ret = under_test.has_high_not_found_paths_ratio(frames)
- self.assertEqual(ret, False)
-
- def test_not_found_paths_ratio_is_equal_to_0_5(self):
- frames = [
- {"path": "some/path"},
- {"path": "some/path"},
- {"path": None},
- {"path": None},
- ]
- ret = under_test.has_high_not_found_paths_ratio(frames)
- self.assertEqual(ret, True)
-
- def test_not_found_paths_ratio_is_less_than_0_5(self):
- frames = [
- {"path": "some/path"},
- {"path": None},
- {"path": None},
- {"path": None},
- ]
- ret = under_test.has_high_not_found_paths_ratio(frames)
- self.assertEqual(ret, True)
-
- def test_no_frames(self):
- frames = []
- ret = under_test.has_high_not_found_paths_ratio(frames)
- self.assertEqual(ret, False)
diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py
index 85725e60858..70689345b69 100644
--- a/buildscripts/tests/test_selected_tests.py
+++ b/buildscripts/tests/test_selected_tests.py
@@ -24,7 +24,7 @@ from buildscripts.task_generation.task_types.gentask_options import GenTaskOptio
from buildscripts.tests.test_burn_in_tests import get_evergreen_config, mock_changed_git_files
from buildscripts import selected_tests as under_test
-# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter,too-many-locals
+# pylint: disable=missing-docstring,invalid-name,unused-argument,protected-access,no-value-for-parameter
NS = "buildscripts.selected_tests"
@@ -101,9 +101,7 @@ class TestAcceptance(unittest.TestCase):
self.assertEqual(generated_config.file_list[0].file_name, "selected_tests_config.json")
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_when_test_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock):
- get_stats_from_s3_mock.return_value = []
+ def test_when_test_mappings_are_found_for_changed_files(self):
mock_evg_api = self._mock_evg_api()
mock_evg_config = get_evergreen_config("etc/evergreen.yml")
mock_evg_expansions = under_test.EvgExpansions(
@@ -135,7 +133,7 @@ class TestAcceptance(unittest.TestCase):
# assert that generated suite files have the suite name and the variant name in the
# filename, to prevent tasks on different variants from using the same suite file
- self.assertIn("auth_enterprise-rhel-8-64-bit-dynamic-required_0.yml", files_to_generate)
+ self.assertIn("auth_enterprise-rhel-80-64-bit-dynamic-required_0.yml", files_to_generate)
generated_evg_config_raw = [
gen_file.content for gen_file in generated_config.file_list
@@ -147,13 +145,11 @@ class TestAcceptance(unittest.TestCase):
# jstests/auth/auth1.js belongs to two suites, auth and auth_audit,
rhel_80_with_generated_tasks = next(
(variant for variant in build_variants_with_generated_tasks
- if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None)
+ if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None)
self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 2)
@unittest.skipIf(sys.platform.startswith("win"), "not supported on windows")
- @patch("buildscripts.util.teststats.HistoricTaskData.get_stats_from_s3")
- def test_when_task_mappings_are_found_for_changed_files(self, get_stats_from_s3_mock):
- get_stats_from_s3_mock.return_value = []
+ def test_when_task_mappings_are_found_for_changed_files(self):
mock_evg_api = self._mock_evg_api()
mock_evg_config = get_evergreen_config("etc/evergreen.yml")
mock_evg_expansions = under_test.EvgExpansions(
@@ -168,7 +164,7 @@ class TestAcceptance(unittest.TestCase):
mock_task_mapping = TaskMapping(
branch="master", project="mongodb-mongo-master", repo="mongodb/mongo",
source_file="src/file1.cpp", source_file_seen_count=8,
- tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-8", flip_count=5)])
+ tasks=[TaskMapInstance(name="auth", variant="enterprise-rhel-80", flip_count=5)])
mock_selected_tests_client = MagicMock()
mock_selected_tests_client.get_task_mappings.return_value = TaskMappingsResponse(
task_mappings=[mock_task_mapping])
@@ -194,7 +190,7 @@ class TestAcceptance(unittest.TestCase):
build_variants_with_generated_tasks = generated_evg_config["buildvariants"]
rhel_80_with_generated_tasks = next(
(variant for variant in build_variants_with_generated_tasks
- if variant["name"] == "enterprise-rhel-8-64-bit-dynamic-required"), None)
+ if variant["name"] == "enterprise-rhel-80-64-bit-dynamic-required"), None)
self.assertEqual(len(rhel_80_with_generated_tasks["tasks"]), 5)
@@ -404,6 +400,13 @@ class TestGetTaskConfigsForTaskMappings(unittest.TestCase):
class TestRemoveRepoPathPrefix(unittest.TestCase):
- def test_path_handling(self):
- filepath = under_test._remove_repo_path_prefix("sample_directory/src/file1.cpp")
- self.assertEqual(filepath, "sample_directory/src/file1.cpp")
+ def test_file_is_in_enterprise_modules(self):
+ filepath = under_test._remove_repo_path_prefix(
+ "src/mongo/db/modules/enterprise/src/file1.cpp")
+
+ self.assertEqual(filepath, "src/file1.cpp")
+
+ def test_file_is_not_in_enterprise_modules(self):
+ filepath = under_test._remove_repo_path_prefix("other_directory/src/file1.cpp")
+
+ self.assertEqual(filepath, "other_directory/src/file1.cpp")
diff --git a/buildscripts/tests/test_validate_commit_message.py b/buildscripts/tests/test_validate_commit_message.py
index 1bf0039188d..db341a5add1 100644
--- a/buildscripts/tests/test_validate_commit_message.py
+++ b/buildscripts/tests/test_validate_commit_message.py
@@ -1,31 +1,130 @@
"""Unit tests for the evergreen_task_timeout script."""
+import itertools
import unittest
+from typing import List, Optional
+from unittest.mock import MagicMock
-from buildscripts.validate_commit_message import main, STATUS_OK, STATUS_ERROR
+import buildscripts.validate_commit_message as under_test
+from buildscripts.client.jiraclient import JiraClient, SecurityLevel
+from evergreen import EvergreenApi
# pylint: disable=missing-docstring,no-self-use
+INVALID_MESSAGES = [
+ "", # You must provide a message
+ "RevertEVG-1", # revert and ticket must be formatted
+ "revert EVG-1", # revert must be capitalized
+ "This is not a valid message", # message must be valid
+ "Fix Lint", # Fix lint is strict in terms of caps
+]
+
+
+def create_mock_code_change(code_change_messages: List[str], branch_name: Optional[str] = None):
+ mock_code_change = MagicMock(
+ commit_messages=code_change_messages,
+ branch_name=branch_name if branch_name else "mongodb-mongo-master",
+ )
+ return mock_code_change
+
+
+def create_mock_patch(code_change_messages: List[str], branch_name: Optional[str] = None):
+ mock_code_change = create_mock_code_change(code_change_messages, branch_name)
+ mock_patch = MagicMock(module_code_changes=[mock_code_change])
+ return mock_patch
+
+
+def create_mock_evg_client(code_change_messages: List[str],
+ branch_name: Optional[str] = None) -> MagicMock:
+ mock_patch = create_mock_patch(code_change_messages, branch_name)
+
+ mock_evg_client = MagicMock(spec_set=EvergreenApi)
+ mock_evg_client.patch_by_id.return_value = mock_patch
+ return mock_evg_client
+
+
+def create_mock_jira_client():
+ mock_jira = MagicMock(spec_set=JiraClient)
+ mock_jira.get_ticket_security_level.return_value = SecurityLevel.NONE
+ return mock_jira
+
+
+def interleave_new_format(older):
+ """Create a new list containing a new and old format copy of each string."""
+ newer = [
+ f"Commit Queue Merge: '{old}' into 'mongodb/mongo:SERVER-45949-validate-message-format'"
+ for old in older
+ ]
+ return list(itertools.chain(*zip(older, newer)))
+
class ValidateCommitMessageTest(unittest.TestCase):
- def test_valid(self):
+ def test_valid_commits(self):
messages = [
+ "Fix lint",
+ "EVG-1", # Test valid projects with various number lengths
+ "SERVER-20",
+ "WT-300",
"SERVER-44338",
- "Revert \"SERVER-60",
+ "Revert EVG-5",
+ "Revert SERVER-60",
+ "Revert WT-700",
+ "Revert 'SERVER-8000",
+ 'Revert "SERVER-90000',
"Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4",
+ "Import tools: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4",
'Revert "Import wiredtiger: 58115abb6fbb3c1cc7bfd087d41a47347bce9a69 from branch mongodb-4.4"',
]
+ mock_evg_api = create_mock_evg_client(interleave_new_format(messages))
+ mock_jira = create_mock_jira_client()
+ orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira)
+
+ is_valid = orchestrator.validate_commit_messages("version_id")
+
+ self.assertEqual(is_valid, under_test.STATUS_OK)
- self.assertTrue(all(main([message]) == STATUS_OK for message in messages))
+ def test_private(self):
+ messages = ["XYZ-1"]
+ mock_evg_api = create_mock_evg_client(interleave_new_format(messages))
+ mock_jira = create_mock_jira_client()
+ orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira)
- def test_invalid(self):
+ is_valid = orchestrator.validate_commit_messages("version_id")
+
+ self.assertEqual(is_valid, under_test.STATUS_ERROR)
+
+ def test_private_with_public(self):
messages = [
- "SERVER-", # missing number
- "Revert SERVER-60", # missing quote before SERVER
- "", # empty value
- "nonsense", # nonsense values
+ "Fix lint",
+ "EVG-1", # Test valid projects with various number lengths
+ "SERVER-20",
+ "XYZ-1",
]
+ mock_evg_api = create_mock_evg_client(interleave_new_format(messages))
+ mock_jira = create_mock_jira_client()
+ orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira)
+
+ is_valid = orchestrator.validate_commit_messages("version_id")
+
+ self.assertEqual(is_valid, under_test.STATUS_ERROR)
+
+ def test_internal_ticket_to_public_repo_should_fail(self):
+ message = "SERVER-20"
+ mock_evg_api = create_mock_evg_client(interleave_new_format([message]))
+ mock_jira = create_mock_jira_client()
+ mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL
+ orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira)
+
+ is_valid = orchestrator.validate_commit_messages("version_id")
+
+ self.assertEqual(is_valid, under_test.STATUS_ERROR)
+
+ def test_internal_ticket_to_private_repo_should_succeed(self):
+ message = "SERVER-20"
+ mock_evg_api = create_mock_evg_client(interleave_new_format([message]), "private-repo")
+ mock_jira = create_mock_jira_client()
+ mock_jira.get_ticket_security_level.return_value = SecurityLevel.MONGO_INTERNAL
+ orchestrator = under_test.CommitMessageValidationOrchestrator(mock_evg_api, mock_jira)
- self.assertTrue(all(main([message]) == STATUS_ERROR for message in messages))
+ is_valid = orchestrator.validate_commit_messages("version_id")
- def test_message_is_empty_list(self):
- self.assertEqual(main([]), STATUS_ERROR)
+ self.assertEqual(is_valid, under_test.STATUS_OK)
diff --git a/buildscripts/tests/timeouts/test_timeout_service.py b/buildscripts/tests/timeouts/test_timeout_service.py
index 23e9c1f331f..bb0dd8a0c3e 100644
--- a/buildscripts/tests/timeouts/test_timeout_service.py
+++ b/buildscripts/tests/timeouts/test_timeout_service.py
@@ -1,42 +1,41 @@
"""Unit tests for timeout_service.py."""
import random
import unittest
-from unittest.mock import MagicMock, patch
+from datetime import datetime, timedelta
+from unittest.mock import MagicMock
from requests.exceptions import HTTPError
+from evergreen import EvergreenApi
import buildscripts.timeouts.timeout_service as under_test
from buildscripts.task_generation.resmoke_proxy import ResmokeProxyService
-from buildscripts.util.teststats import HistoricTaskData, HistoricTestInfo
+from buildscripts.util.teststats import HistoricTaskData
# pylint: disable=missing-docstring,no-self-use,invalid-name,protected-access
-NS = "buildscripts.timeouts.timeout_service"
-
-def ns(relative_name): # pylint: disable=invalid-name
- """Return a full name from a name relative to the test module"s name space."""
- return NS + "." + relative_name
-
-
-def build_mock_service(resmoke_proxy=None):
+def build_mock_service(evg_api=None, resmoke_proxy=None):
+ end_date = datetime.now()
+ start_date = end_date - timedelta(weeks=2)
+ timeout_settings = under_test.TimeoutSettings(
+ end_date=end_date,
+ start_date=start_date,
+ )
return under_test.TimeoutService(
- resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService))
+ evg_api=evg_api if evg_api else MagicMock(spec_set=EvergreenApi),
+ resmoke_proxy=resmoke_proxy if resmoke_proxy else MagicMock(spec_set=ResmokeProxyService),
+ timeout_settings=timeout_settings)
def tst_stat_mock(file, duration, pass_count):
- return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count, hooks=[])
-
-
-def tst_runtime_mock(file, duration, pass_count):
- return MagicMock(test_name=file, avg_duration_pass=duration, num_pass=pass_count)
+ return MagicMock(test_file=file, avg_duration_pass=duration, num_pass=pass_count)
class TestGetTimeoutEstimate(unittest.TestCase):
- @patch(ns("HistoricTaskData.from_s3"))
- def test_no_stats_should_return_default_timeout(self, from_s3_mock: MagicMock):
- timeout_service = build_mock_service()
- from_s3_mock.return_value = []
+ def test_no_stats_should_return_default_timeout(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ mock_evg_api.test_stats_by_project.return_value = []
+ timeout_service = build_mock_service(evg_api=mock_evg_api)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -49,19 +48,13 @@ class TestGetTimeoutEstimate(unittest.TestCase):
self.assertFalse(timeout.is_specified())
- @patch(ns("HistoricTaskData.from_s3"))
- def test_too_many_tests_missing_history_should_cause_a_default_timeout(
- self, from_s3_mock: MagicMock):
- test_stats = [
- HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[])
- for i in range(23)
- ]
- from_s3_mock.return_value = HistoricTaskData(test_stats)
+ def test_a_test_with_missing_history_should_cause_a_default_timeout(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
+ mock_evg_api.test_stats_by_project.return_value = test_stats
mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
- test_names = [ts.test_name for ts in test_stats]
- test_names.extend([f"test_with_no_stats_{i}.js" for i in range(7)])
- mock_resmoke_proxy.list_tests.return_value = test_names
- timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
+ mock_resmoke_proxy.list_tests.return_value = ["test_with_no_stats.js"]
+ timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -74,21 +67,14 @@ class TestGetTimeoutEstimate(unittest.TestCase):
self.assertFalse(timeout.is_specified())
- @patch(ns("HistoricTaskData.from_s3"))
- def test_too_many_tests_with_zero_runtime_history_should_cause_a_default_timeout(
- self, from_s3_mock: MagicMock):
- test_stats = [
- HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[])
- for i in range(23)
- ]
- test_stats.extend([
- HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[])
- for i in range(7)
- ])
- from_s3_mock.return_value = HistoricTaskData(test_stats)
+ def test_a_test_with_zero_runtime_history_should_cause_a_default_timeout(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(30)]
+ test_stats.append(tst_stat_mock("zero.js", 0.0, 1))
+ mock_evg_api.test_stats_by_project.return_value = test_stats
mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
- mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats]
- timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
+ mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
+ timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -101,73 +87,15 @@ class TestGetTimeoutEstimate(unittest.TestCase):
self.assertFalse(timeout.is_specified())
- @patch(ns("HistoricTaskData.from_s3"))
- def test_enough_history_but_some_tests_missing_history_should_cause_custom_task_and_default_test_timeout(
- self, from_s3_mock: MagicMock):
- test_stats = [
- HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[])
- for i in range(25)
- ]
- from_s3_mock.return_value = HistoricTaskData(test_stats)
- mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
- test_names = [ts.test_name for ts in test_stats]
- test_names.extend([f"test_with_no_stats_{i}.js" for i in range(5)])
- mock_resmoke_proxy.list_tests.return_value = test_names
- timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
- timeout_params = under_test.TimeoutParams(
- evg_project="my project",
- build_variant="bv",
- task_name="my task",
- suite_name="my suite",
- is_asan=False,
- )
-
- timeout = timeout_service.get_timeout_estimate(timeout_params)
-
- self.assertTrue(timeout.is_specified())
- self.assertEqual(None, timeout.calculate_test_timeout(1))
- self.assertEqual(54360, timeout.calculate_task_timeout(1))
-
- @patch(ns("HistoricTaskData.from_s3"))
- def test_enough_history_but_some_tests_with_zero_runtime_should_cause_custom_task_and_default_test_timeout(
- self, from_s3_mock: MagicMock):
- test_stats = [
- HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[])
- for i in range(25)
- ]
- test_stats.extend([
- HistoricTestInfo(test_name=f"zero_{i}.js", avg_duration=0.0, num_pass=1, hooks=[])
- for i in range(5)
- ])
- from_s3_mock.return_value = HistoricTaskData(test_stats)
- mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
- mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats]
- timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
- timeout_params = under_test.TimeoutParams(
- evg_project="my project",
- build_variant="bv",
- task_name="my task",
- suite_name="my suite",
- is_asan=False,
- )
-
- timeout = timeout_service.get_timeout_estimate(timeout_params)
-
- self.assertTrue(timeout.is_specified())
- self.assertEqual(None, timeout.calculate_test_timeout(1))
- self.assertEqual(54360, timeout.calculate_task_timeout(1))
-
- @patch(ns("HistoricTaskData.from_s3"))
- def test_all_tests_with_runtime_history_should_use_custom_timeout(self,
- from_s3_mock: MagicMock):
- test_stats = [
- HistoricTestInfo(test_name=f"test_{i}.js", avg_duration=600.0, num_pass=1, hooks=[])
- for i in range(30)
- ]
- from_s3_mock.return_value = HistoricTaskData(test_stats)
+ def test_all_tests_with_runtime_history_should_use_custom_timeout(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ n_tests = 30
+ test_runtime = 600
+ test_stats = [tst_stat_mock(f"test_{i}.js", test_runtime, 1) for i in range(n_tests)]
+ mock_evg_api.test_stats_by_project.return_value = test_stats
mock_resmoke_proxy = MagicMock(spec_set=ResmokeProxyService)
- mock_resmoke_proxy.list_tests.return_value = [ts.test_name for ts in test_stats]
- timeout_service = build_mock_service(resmoke_proxy=mock_resmoke_proxy)
+ mock_resmoke_proxy.list_tests.return_value = [ts.test_file for ts in test_stats]
+ timeout_service = build_mock_service(evg_api=mock_evg_api, resmoke_proxy=mock_resmoke_proxy)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -180,7 +108,7 @@ class TestGetTimeoutEstimate(unittest.TestCase):
self.assertTrue(timeout.is_specified())
self.assertEqual(1860, timeout.calculate_test_timeout(1))
- self.assertEqual(54360, timeout.calculate_task_timeout(1))
+ self.assertEqual(54180, timeout.calculate_task_timeout(1))
class TestGetTaskHookOverhead(unittest.TestCase):
@@ -221,10 +149,10 @@ class TestGetTaskHookOverhead(unittest.TestCase):
class TestLookupHistoricStats(unittest.TestCase):
- @patch(ns("HistoricTaskData.from_s3"))
- def test_no_stats_from_evergreen_should_return_none(self, from_s3_mock: MagicMock):
- from_s3_mock.return_value = None
- timeout_service = build_mock_service()
+ def test_no_stats_from_evergreen_should_return_none(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ mock_evg_api.test_stats_by_project.return_value = []
+ timeout_service = build_mock_service(evg_api=mock_evg_api)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -237,10 +165,10 @@ class TestLookupHistoricStats(unittest.TestCase):
self.assertIsNone(stats)
- @patch(ns("HistoricTaskData.from_s3"))
- def test_errors_from_evergreen_should_return_none(self, from_s3_mock: MagicMock):
- from_s3_mock.side_effect = HTTPError("failed to connect")
- timeout_service = build_mock_service()
+ def test_errors_from_evergreen_should_return_none(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
+ mock_evg_api.test_stats_by_project.side_effect = HTTPError("failed to connect")
+ timeout_service = build_mock_service(evg_api=mock_evg_api)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -253,11 +181,11 @@ class TestLookupHistoricStats(unittest.TestCase):
self.assertIsNone(stats)
- @patch(ns("HistoricTaskData.from_s3"))
- def test_stats_from_evergreen_should_return_the_stats(self, from_s3_mock: MagicMock):
+ def test_stats_from_evergreen_should_return_the_stats(self):
+ mock_evg_api = MagicMock(spec_set=EvergreenApi)
test_stats = [tst_stat_mock(f"test_{i}.js", 60, 1) for i in range(100)]
- from_s3_mock.return_value = HistoricTaskData(test_stats)
- timeout_service = build_mock_service()
+ mock_evg_api.test_stats_by_project.return_value = test_stats
+ timeout_service = build_mock_service(evg_api=mock_evg_api)
timeout_params = under_test.TimeoutParams(
evg_project="my project",
build_variant="bv",
@@ -328,34 +256,3 @@ class TestGetCleanEveryNCadence(unittest.TestCase):
cadence = timeout_service._get_clean_every_n_cadence("suite", False)
self.assertEqual(1, cadence)
-
-
-class TestHaveEnoughHistoricStats(unittest.TestCase):
- def test_should_return_true_when_number_of_tests_equals_zero(self):
- timeout_service = build_mock_service()
- self.assertTrue(
- timeout_service._have_enough_historic_stats(num_tests=0, num_tests_missing_data=0))
-
- def test_should_return_true_when_number_of_tests_with_historic_data_more_than_threshold(self):
- timeout_service = build_mock_service()
- self.assertTrue(
- timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=19))
- self.assertTrue(
- timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=0))
-
- def test_should_return_false_when_number_of_tests_with_historic_data_less_or_equal_to_threshold(
- self):
- timeout_service = build_mock_service()
- self.assertFalse(
- timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=20))
- self.assertFalse(
- timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=21))
- self.assertFalse(
- timeout_service._have_enough_historic_stats(num_tests=100, num_tests_missing_data=100))
-
- def test_exception_raised_when_number_of_tests_less_than_zero(self):
- timeout_service = build_mock_service()
- with self.assertRaises(ValueError):
- timeout_service._have_enough_historic_stats(num_tests=-1, num_tests_missing_data=0)
- with self.assertRaises(ValueError):
- timeout_service._have_enough_historic_stats(num_tests=-100, num_tests_missing_data=0)
diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py
index 637ee59321e..7f3296ca1aa 100644
--- a/buildscripts/tests/util/test_taskname.py
+++ b/buildscripts/tests/util/test_taskname.py
@@ -28,7 +28,7 @@ class TestRemoveGenSuffix(unittest.TestCase):
class TestDetermineTaskBaseName(unittest.TestCase):
def test_task_name_with_build_variant_should_strip_bv_and_sub_task_index(self):
- bv = "enterprise-rhel-8-64-bit-dynamic-required"
+ bv = "enterprise-rhel-80-64-bit-dynamic-required"
task_name = f"auth_23_{bv}"
base_task_name = under_test.determine_task_base_name(task_name, bv)
@@ -36,7 +36,7 @@ class TestDetermineTaskBaseName(unittest.TestCase):
self.assertEqual("auth", base_task_name)
def test_task_name_without_build_variant_should_strip_sub_task_index(self):
- bv = "enterprise-rhel-8-64-bit-dynamic-required"
+ bv = "enterprise-rhel-80-64-bit-dynamic-required"
task_name = "auth_314"
base_task_name = under_test.determine_task_base_name(task_name, bv)
@@ -44,7 +44,7 @@ class TestDetermineTaskBaseName(unittest.TestCase):
self.assertEqual("auth", base_task_name)
def test_task_name_without_build_variant_or_subtask_index_should_self(self):
- bv = "enterprise-rhel-8-64-bit-dynamic-required"
+ bv = "enterprise-rhel-80-64-bit-dynamic-required"
task_name = "auth"
base_task_name = under_test.determine_task_base_name(task_name, bv)
diff --git a/buildscripts/tests/util/test_teststats.py b/buildscripts/tests/util/test_teststats.py
index 9e72d87a737..ebba930d032 100644
--- a/buildscripts/tests/util/test_teststats.py
+++ b/buildscripts/tests/util/test_teststats.py
@@ -2,12 +2,8 @@
import datetime
import unittest
-from json import JSONDecodeError
-from unittest.mock import patch
from mock import Mock
-from mock.mock import MagicMock
-from requests import Session
import buildscripts.util.teststats as under_test
@@ -25,60 +21,6 @@ class NormalizeTestNameTest(unittest.TestCase):
under_test.normalize_test_name("\\home\\user\\test.js"))
-class TestHistoricTestInfo(unittest.TestCase):
- def test_total_test_runtime_not_passing_test_no_hooks(self):
- test_info = under_test.HistoricTestInfo(
- test_name='jstests/test.js',
- num_pass=0,
- avg_duration=0.0,
- hooks=[],
- )
-
- self.assertEqual(0.0, test_info.total_test_runtime())
-
- def test_total_test_runtime_not_passing_test_with_hooks(self):
- test_info = under_test.HistoricTestInfo(
- test_name='jstests/test.js',
- num_pass=0,
- avg_duration=0.0,
- hooks=[
- under_test.HistoricHookInfo(
- hook_id='test:hook',
- num_pass=10,
- avg_duration=5.0,
- ),
- ],
- )
-
- self.assertEqual(0.0, test_info.total_test_runtime())
-
- def test_total_test_runtime_passing_test_no_hooks(self):
- test_info = under_test.HistoricTestInfo(
- test_name='jstests/test.js',
- num_pass=10,
- avg_duration=23.0,
- hooks=[],
- )
-
- self.assertEqual(23.0, test_info.total_test_runtime())
-
- def test_total_test_runtime_passing_test_with_hooks(self):
- test_info = under_test.HistoricTestInfo(
- test_name='jstests/test.js',
- num_pass=10,
- avg_duration=23.0,
- hooks=[
- under_test.HistoricHookInfo(
- hook_id='test:hook',
- num_pass=10,
- avg_duration=5.0,
- ),
- ],
- )
-
- self.assertEqual(28.0, test_info.total_test_runtime())
-
-
class TestHistoricTaskData(unittest.TestCase):
def test_no_hooks(self):
evg_results = [
@@ -138,7 +80,7 @@ class TestHistoricTaskData(unittest.TestCase):
@staticmethod
def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0):
return Mock(
- test_name=test_file,
+ test_file=test_file,
task_name="task1",
variant="variant1",
distro="distro1",
@@ -147,53 +89,3 @@ class TestHistoricTaskData(unittest.TestCase):
num_fail=0,
avg_duration_pass=duration,
)
-
- @patch.object(Session, 'get')
- def test_get_stats_from_s3_returns_data(self, mock_get):
- mock_response = MagicMock()
- mock_response.json.return_value = [
- {
- "test_name": "jstests/noPassthroughWithMongod/geo_near_random1.js",
- "num_pass": 74,
- "num_fail": 0,
- "avg_duration_pass": 23.16216216216216,
- "max_duration_pass": 27.123,
- },
- {
- "test_name": "shell_advance_cluster_time:ValidateCollections",
- "num_pass": 74,
- "num_fail": 0,
- "avg_duration_pass": 1.662162162162162,
- "max_duration_pass": 100.0987,
- },
- ]
- mock_get.return_value = mock_response
-
- result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant")
-
- self.assertEqual(result, [
- under_test.HistoricalTestInformation(
- test_name="jstests/noPassthroughWithMongod/geo_near_random1.js",
- num_pass=74,
- num_fail=0,
- avg_duration_pass=23.16216216216216,
- max_duration_pass=27.123,
- ),
- under_test.HistoricalTestInformation(
- test_name="shell_advance_cluster_time:ValidateCollections",
- num_pass=74,
- num_fail=0,
- avg_duration_pass=1.662162162162162,
- max_duration_pass=100.0987,
- ),
- ])
-
- @patch.object(Session, 'get')
- def test_get_stats_from_s3_json_decode_error(self, mock_get):
- mock_response = MagicMock()
- mock_response.json.side_effect = JSONDecodeError("msg", "doc", 0)
- mock_get.return_value = mock_response
-
- result = under_test.HistoricTaskData.get_stats_from_s3("project", "task", "variant")
-
- self.assertEqual(result, [])