diff options
Diffstat (limited to 'site_scons/site_tools')
| -rw-r--r-- | site_scons/site_tools/mongo_test_list.py | 5 | ||||
| -rw-r--r-- | site_scons/site_tools/mongo_unittest.py | 24 | ||||
| -rw-r--r-- | site_scons/site_tools/ninja.py | 223 | ||||
| -rw-r--r-- | site_scons/site_tools/oom_auto_retry.py | 113 |
4 files changed, 63 insertions, 302 deletions
diff --git a/site_scons/site_tools/mongo_test_list.py b/site_scons/site_tools/mongo_test_list.py index 9b1a8ae6a3b..1b02c52eb8e 100644 --- a/site_scons/site_tools/mongo_test_list.py +++ b/site_scons/site_tools/mongo_test_list.py @@ -28,7 +28,7 @@ from collections import defaultdict TEST_REGISTRY = defaultdict(list) -def register_test(env, file, test, generate_alias=True): +def register_test(env, file, test): """Register test into the dictionary of tests for file_name""" test_path = test if env.get("AUTO_INSTALL_ENABLED", False) and env.GetAutoInstalledFiles(test): @@ -40,8 +40,7 @@ def register_test(env, file, test, generate_alias=True): env.Depends(file, test_path) file_name = file.path TEST_REGISTRY[file_name].append(test_path) - if generate_alias: - env.GenerateTestExecutionAliases(test) + env.GenerateTestExecutionAliases(test) def test_list_builder_action(env, target, source): diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py index 417d8ddce47..33373282606 100644 --- a/site_scons/site_tools/mongo_unittest.py +++ b/site_scons/site_tools/mongo_unittest.py @@ -25,19 +25,11 @@ from SCons.Script import Action from site_scons.mongo import insort_wrapper -LAST_TEST_GROUP = 0 - -TEST_GROUPS = ['first', 'second', 'third', 'fourth'] - - def exists(env): return True def build_cpp_unit_test(env, target, source, **kwargs): - - global LAST_TEST_GROUP - if not isinstance(target, list): target = [target] @@ -45,11 +37,6 @@ def build_cpp_unit_test(env, target, source, **kwargs): if not t.endswith('_test'): env.ConfError(f"CppUnitTest target `{t}' does not end in `_test'") - test_group = TEST_GROUPS[LAST_TEST_GROUP] - LAST_TEST_GROUP += 1 - if LAST_TEST_GROUP > len(TEST_GROUPS) - 1: - LAST_TEST_GROUP = 0 - if not kwargs.get("UNITTEST_HAS_CUSTOM_MAINLINE", False): libdeps = kwargs.get("LIBDEPS", env.get("LIBDEPS", [])).copy() insort_wrapper(libdeps, "$BUILD_DIR/mongo/unittest/unittest_main") @@ -62,7 +49,8 @@ def build_cpp_unit_test(env, target, source, **kwargs): elif primary_component: kwargs["AIB_COMPONENT"] = primary_component else: - kwargs["AIB_COMPONENT"] = f"{test_group}-quarter-unittests" + kwargs["AIB_COMPONENT"] = "unittests" + unit_test_components = {"tests"} if "AIB_COMPONENTS_EXTRA" in kwargs: kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union( @@ -75,18 +63,10 @@ def build_cpp_unit_test(env, target, source, **kwargs): env.RegisterTest("$UNITTEST_LIST", result[0]) env.Alias("$UNITTEST_ALIAS", result[0]) - env.RegisterTest(f"$BUILD_ROOT/{test_group}_quarter_unittests.txt", result[0], - generate_alias=False) - env.Alias(f"install-{test_group}-quarter-unittests", result[0]) - return result def generate(env): - for test_group in TEST_GROUPS: - env.TestList(f"$BUILD_ROOT/{test_group}_quarter_unittests.txt", source=[]) - env.Alias(f"install-{test_group}-quarter-unittests", - f"$BUILD_ROOT/{test_group}_quarter_unittests.txt") env.TestList("$UNITTEST_LIST", source=[]) env.AddMethod(build_cpp_unit_test, "CppUnitTest") env.Alias("$UNITTEST_ALIAS", "$UNITTEST_LIST") diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py index 30c4dbe9d58..8505ce8e7ba 100644 --- a/site_scons/site_tools/ninja.py +++ b/site_scons/site_tools/ninja.py @@ -57,24 +57,6 @@ COMMAND_TYPES = ( SCons.Action.CommandGeneratorAction, ) -ninja_compdb_adjust = """\ -import json -import sys - -compdb = {} -with open(sys.argv[1]) as f: - compdb = json.load(f) - -for command in compdb: - if command['output'].endswith('.compdb'): - command['output'] = command['output'][:-(len('.compdb'))] - else: - print(f"compdb entry does not contain '.compdb': {command['output']}") - -with open(sys.argv[1], 'w') as f: - json.dump(compdb, f, indent=2) -""" - def _install_action_function(_env, node): """Install files using the install or copy commands""" @@ -83,7 +65,6 @@ def _install_action_function(_env, node): "rule": "INSTALL", "inputs": [get_path(src_file(s)) for s in node.sources], "implicit": get_dependencies(node), - "variables": {"precious": node.precious}, } @@ -98,11 +79,9 @@ def _mkdir_action_function(env, node): # to an invalid ninja file. "variables": { # On Windows mkdir "-p" is always on - "cmd": - "mkdir {args}".format( - args=' '.join(get_outputs(node)) + " & exit /b 0" - if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ), - "variables": {"precious": node.precious}, + "cmd": "{mkdir} $out".format( + mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p", + ), }, } @@ -122,7 +101,6 @@ def _lib_symlink_action_function(_env, node): "inputs": inputs, "rule": "SYMLINK", "implicit": get_dependencies(node), - "variables": {"precious": node.precious}, } @@ -327,11 +305,6 @@ class SConsToNinjaTranslator: if callable(node_callback): node_callback(env, node, build) - if build is not None and node.precious: - if not build.get('variables'): - build['variables'] = {} - build['variables']['precious'] = node.precious - return build def handle_func_action(self, node, action): @@ -436,6 +409,14 @@ class SConsToNinjaTranslator: "implicit": dependencies, } + elif results[0]["rule"] == "INSTALL": + return { + "outputs": all_outputs, + "rule": "INSTALL", + "inputs": [get_path(src_file(s)) for s in node.sources], + "implicit": dependencies, + } + raise Exception("Unhandled list action with rule: " + results[0]["rule"]) @@ -465,21 +446,14 @@ class NinjaState: scons_escape = env.get("ESCAPE", lambda x: x) self.variables = { - # The /b option here will make sure that windows updates the mtime - # when copying the file. This allows to not need to use restat for windows - # copy commands. - "COPY": - "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp", - "NOOP": - "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null", - "SCONS_INVOCATION": - "{} {} __NINJA_NO=1 $out".format( - sys.executable, - " ".join([ - ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv - if arg not in COMMAND_LINE_TARGETS - ]), + "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp", + "NOOP": "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null", + "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format( + sys.executable, + " ".join( + [ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS] ), + ), "SCONS_INVOCATION_W_TARGETS": "{} {}".format( sys.executable, " ".join([ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv]) ), @@ -513,18 +487,6 @@ class NinjaState: "rspfile": "$out.rsp", "rspfile_content": "$rspc", }, - "COMPDB_CC": { - "command": "$CC @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, - "COMPDB_CXX": { - "command": "$CXX @$out.rsp", - "description": "Compiling $out", - "rspfile": "$out.rsp", - "rspfile_content": "$rspc", - }, "LINK": { "command": "$env$LINK @$out.rsp", "description": "Linking $out", @@ -532,9 +494,17 @@ class NinjaState: "rspfile_content": "$rspc", "pool": "local_pool", }, + # Ninja does not automatically delete the archive before + # invoking ar. The ar utility will append to an existing archive, which + # can cause duplicate symbols if the symbols moved between object files. + # Native SCons will perform this operation so we need to force ninja + # to do the same. See related for more info: + # https://jira.mongodb.org/browse/SERVER-49457 "AR": { - "command": "$env$AR @$out.rsp", - "description": "Archived $out", + "command": "{}$env$AR @$out.rsp".format( + '' if sys.platform == "win32" else "rm -f $out && " + ), + "description": "Archiving $out", "rspfile": "$out.rsp", "rspfile_content": "$rspc", "pool": "local_pool", @@ -556,6 +526,15 @@ class NinjaState: "command": "$COPY $in $out", "description": "Install $out", "pool": "install_pool", + # On Windows cmd.exe /c copy does not always correctly + # update the timestamp on the output file. This leads + # to a stuck constant timestamp in the Ninja database + # and needless rebuilds. + # + # Adding restat here ensures that Ninja always checks + # the copy updated the timestamp and that Ninja has + # the correct information. + "restat": 1, }, "TEMPLATE": { "command": "$SCONS_INVOCATION $out", @@ -668,9 +647,6 @@ class NinjaState: ninja.comment("Generated by scons. DO NOT EDIT.") - # This version is needed because it is easy to get from pip and it support compile_commands.json - ninja.variable("ninja_required_version", "1.10") - ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR'])) for pool_name, size in self.pools.items(): @@ -679,34 +655,10 @@ class NinjaState: for var, val in self.variables.items(): ninja.variable(var, val) - # This is the command that is used to clean a target before building it, - # excluding precious targets. - if sys.platform == "win32": - rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 &' - else: - rm_cmd = 'rm -f $rm_outs;' - - precious_rule_suffix = "_PRECIOUS" - - # Make two sets of rules to honor scons Precious setting. The build nodes themselves - # will then reselect their rule according to the precious being set for that node. - precious_rules = {} for rule, kwargs in self.rules.items(): if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs: kwargs['pool'] = 'local_pool' - # Do not worry about precious for commands that don't have targets (phony) - # or that will callback to scons (which maintains its own precious). - if rule not in ['phony', 'TEMPLATE', 'REGENERATE', 'COMPDB_CC', 'COMPDB_CXX']: - precious_rule = rule + precious_rule_suffix - precious_rules[precious_rule] = kwargs.copy() - ninja.rule(precious_rule, **precious_rules[precious_rule]) - - kwargs['command'] = f"{rm_cmd} " + kwargs['command'] - ninja.rule(rule, **kwargs) - else: - - ninja.rule(rule, **kwargs) - self.rules.update(precious_rules) + ninja.rule(rule, **kwargs) # If the user supplied an alias to determine generated sources, use that, otherwise # determine what the generated sources are dynamically. @@ -761,45 +713,6 @@ class NinjaState: template_builders = [] - # If we ever change the name/s of the rules that include - # compile commands (i.e. something like CC) we will need to - # update this build to reflect that complete list. - compile_commands = "compile_commands.json" - compdb_expand = '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '' - adjust_script_out = os.path.join( - get_path(self.env['NINJA_BUILDDIR']), 'ninja_compdb_adjust.py') - os.makedirs(os.path.dirname(adjust_script_out), exist_ok=True) - with open(adjust_script_out, 'w') as f: - f.write(ninja_compdb_adjust) - self.builds[compile_commands] = { - 'rule': "CMD", - 'outputs': [compile_commands], - 'pool': "console", - 'implicit': [ninja_file], - 'variables': { - "cmd": - f"ninja -f {ninja_file} -t compdb {compdb_expand}COMPDB_CC COMPDB_CXX > {compile_commands};" - + f"{sys.executable} {adjust_script_out} {compile_commands}" - }, - } - self.builds["compiledb"] = { - 'rule': "phony", - "outputs": ["compiledb"], - 'implicit': [compile_commands], - } - - # Now for all build nodes, we want to select the precious rule or not. - # If it's not precious, we need to save all the outputs into a variable - # on that node. Later we will be removing outputs and switching them to - # phonies so that we can generate response and depfiles correctly. - for build, kwargs in self.builds.items(): - if kwargs.get('variables') and kwargs['variables'].get('precious'): - kwargs['rule'] = kwargs['rule'] + precious_rule_suffix - elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']: - if not kwargs.get('variables'): - kwargs['variables'] = {} - kwargs['variables']['rm_outs'] = kwargs['outputs'].copy() - for build in [self.builds[key] for key in sorted(self.builds.keys())]: if build["rule"] == "TEMPLATE": template_builders.append(build) @@ -890,20 +803,6 @@ class NinjaState: ninja.build(**build) - for build, kwargs in self.builds.items(): - if kwargs['rule'] in [ - 'CC', f'CC{precious_rule_suffix}', 'CXX', f'CXX{precious_rule_suffix}' - ]: - rule = kwargs['rule'].replace( - precious_rule_suffix - ) if precious_rule_suffix in kwargs['rule'] else kwargs['rule'] - rule = "COMPDB_" + rule - compdb_build = kwargs.copy() - - compdb_build['rule'] = rule - compdb_build['outputs'] = [kwargs['outputs'] + ".compdb"] - ninja.build(**compdb_build) - template_builds = {'rule': "TEMPLATE"} for template_builder in template_builders: @@ -964,6 +863,25 @@ class NinjaState: implicit=[__file__], ) + # If we ever change the name/s of the rules that include + # compile commands (i.e. something like CC) we will need to + # update this build to reflect that complete list. + ninja.build( + "compile_commands.json", + rule="CMD", + pool="console", + implicit=[ninja_file], + variables={ + "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format( + ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else '' + ) + }, + ) + + ninja.build( + "compiledb", rule="phony", implicit=["compile_commands.json"], + ) + # Look in SCons's list of DEFAULT_TARGETS, find the ones that # we generated a ninja build rule for. scons_default_targets = [ @@ -1336,9 +1254,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule): __NINJA_RULE_MAPPING[pre_subst_string] = rule -def register_custom_rule(env, rule, command, description="", deps=None, pool=None, - use_depfile=False, use_response_file=False, response_file_content="$rspc", - restat=False): +def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"): """Allows specification of Ninja rules from inside SCons files.""" rule_obj = { "command": command, @@ -1356,15 +1272,8 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non if use_response_file: rule_obj["rspfile"] = "$out.rsp" - if rule_obj["rspfile"] not in command: - raise Exception( - f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}' - ) rule_obj["rspfile_content"] = response_file_content - if restat: - rule_obj["restat"] = 1 - env[NINJA_RULES][rule] = rule_obj @@ -1541,6 +1450,7 @@ def generate(env): ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}") ninja_file = env.Ninja(target=ninja_file_name, source=[]) env.AlwaysBuild(ninja_file) + env.Alias("$NINJA_ALIAS_NAME", ninja_file) # TODO: API for getting the SConscripts programmatically # exists upstream: https://github.com/SCons/scons/issues/3625 @@ -1566,15 +1476,6 @@ def generate(env): env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider") env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback") - # Expose ninja node path converstion functions to make writing - # custom function action handlers easier. - env.AddMethod(lambda _env, node: get_outputs(node), "NinjaGetOutputs") - env.AddMethod(lambda _env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types), - "NinjaGetInputs") - env.AddMethod(lambda _env, node, skip_sources=False: get_dependencies(node), - "NinjaGetDependencies") - env.AddMethod(lambda _env, node: get_order_only(node), "NinjaGetOrderOnly") - # Provides a way for users to handle custom FunctionActions they # want to translate to Ninja. env[NINJA_CUSTOM_HANDLERS] = {} @@ -1669,12 +1570,6 @@ def generate(env): if not exists(env): return - # There is a target called generate-ninja which needs to be included - # with the --ninja flag in order to generate the ninja file. Because the --ninja - # flag is ONLY used with generate-ninja, we have combined the two by making the --ninja flag - # implicitly build the generate-ninja target. - SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file)) - # Set a known variable that other tools can query so they can # behave correctly during ninja generation. env["GENERATING_NINJA"] = True diff --git a/site_scons/site_tools/oom_auto_retry.py b/site_scons/site_tools/oom_auto_retry.py deleted file mode 100644 index 7ff457d2798..00000000000 --- a/site_scons/site_tools/oom_auto_retry.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2023 MongoDB Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -import SCons - -import functools -import subprocess -import sys -import time -import random -import os -import re - -from typing import Callable, List, Dict - - -def command_spawn_func(sh: str, escape: Callable[[str], str], cmd: str, args: List, env: Dict, - target: List, source: List): - retries = 0 - success = False - - build_env = target[0].get_build_env() - oom_messages = [ - re.compile(msg, re.MULTILINE | re.DOTALL) - for msg in build_env.get('OOM_RETRY_MESSAGES', []) - ] - oom_returncodes = [int(returncode) for returncode in build_env.get('OOM_RETRY_RETURNCODES', [])] - max_retries = build_env.get('OOM_RETRY_ATTEMPTS', 10) - oom_max_retry_delay = build_env.get('OOM_RETRY_MAX_DELAY_SECONDS', 120) - - while not success and retries <= max_retries: - - try: - start_time = time.time() - if sys.platform[:3] == 'win': - # have to use shell=True for windows because of https://github.com/python/cpython/issues/53908 - proc = subprocess.run(' '.join(args), env=env, close_fds=True, shell=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - check=True) - else: - proc = subprocess.run([sh, '-c', ' '.join(args)], env=env, close_fds=True, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - check=True) - except subprocess.CalledProcessError as exc: - print(f"{os.path.basename(__file__)} captured error:") - print(exc.stdout) - if any([re.findall(oom_message, exc.stdout) for oom_message in oom_messages]) or any( - [oom_returncode == exc.returncode for oom_returncode in oom_returncodes]): - retries += 1 - retry_delay = int((time.time() - start_time) + - oom_max_retry_delay * random.random()) - print(f"Ran out of memory while trying to build {target[0]}", ) - if retries <= max_retries: - print(f"trying again in {retry_delay} seconds with retry attempt {retries}") - time.sleep(retry_delay) - continue - - # There was no OOM error or no more OOM retries left - return exc.returncode - else: - if proc.stdout: - print(proc.stdout) - return proc.returncode - - -def generate(env): - - original_command_execute = SCons.Action.CommandAction.execute - - def oom_retry_execute(command_action_instance, target, source, env, executor=None): - - if 'conftest' not in str(target[0]) and target[0].has_builder() and target[0].get_builder( - ).get_name(env) in [ - 'Object', 'SharedObject', 'StaticObject', 'Program', 'StaticLibrary', - 'SharedLibrary' - ]: - - original_spawn = env['SPAWN'] - - env['SPAWN'] = functools.partial(command_spawn_func, target=target, source=source) - result = original_command_execute(command_action_instance, target, source, env, - executor) - env['SPAWN'] = original_spawn - - else: - result = original_command_execute(command_action_instance, target, source, env, - executor) - return result - - SCons.Action.CommandAction.execute = oom_retry_execute - - -def exists(env): - return True |