summaryrefslogtreecommitdiff
path: root/site_scons
diff options
context:
space:
mode:
authorLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
committerLucas de Castro Borges <lucas@gnuabordo.com.br>2025-02-11 15:07:35 -0300
commit4cb8841196d0625dfa3825aa326f071cd27c7b8b (patch)
tree1682a647d4463397c119183369ae6f750d5fdcff /site_scons
parentaa03c6362cbaa767638e6eed9b031d86dd2643d1 (diff)
parent8f0827553e09872941945a093b647a4211a9db7f (diff)
Update upstream source from tag 'upstream/6.0.0'master
Update to upstream version '6.0.0' with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322
Diffstat (limited to 'site_scons')
-rw-r--r--site_scons/mongo/build_profiles.py117
-rw-r--r--site_scons/mongo/generators.py1
-rw-r--r--site_scons/site_tools/mongo_test_list.py5
-rw-r--r--site_scons/site_tools/mongo_unittest.py24
-rw-r--r--site_scons/site_tools/ninja.py223
-rw-r--r--site_scons/site_tools/oom_auto_retry.py113
6 files changed, 63 insertions, 420 deletions
diff --git a/site_scons/mongo/build_profiles.py b/site_scons/mongo/build_profiles.py
deleted file mode 100644
index 925b8c118a0..00000000000
--- a/site_scons/mongo/build_profiles.py
+++ /dev/null
@@ -1,117 +0,0 @@
-"""Dictionary to store available build profiles."""
-from dataclasses import dataclass
-from typing import Any, List, Optional
-import mongo.generators as mongo_generators
-
-
-@dataclass
-class BuildProfile:
- ninja: str
- variables_files: List
- allocator: str
- sanitize: Optional[str]
- link_model: str
- dbg: str
- opt: str
- ICECC: Optional[str]
- CCACHE: Optional[str]
- NINJA_PREFIX: str
- VARIANT_DIR: Any
- disable_warnings_as_errors: Optional[List]
-
-
-BUILD_PROFILES = {
- # These options were the default settings before implementing build profiles.
- "default":
- BuildProfile(
- ninja="disabled",
- variables_files=[],
- allocator="auto",
- sanitize=None,
- link_model="auto",
- dbg="off",
- opt="auto",
- ICECC=None,
- CCACHE=None,
- NINJA_PREFIX="build",
- VARIANT_DIR=mongo_generators.default_variant_dir_generator,
- disable_warnings_as_errors=[],
- ),
- # This build has fast runtime speed & fast build time at the cost of debuggability.
- "fast":
- BuildProfile(
- ninja="enabled",
- variables_files=[
- './etc/scons/mongodbtoolchain_stable_clang.vars',
- './etc/scons/developer_versions.vars',
- ],
- allocator="auto",
- sanitize=None,
- link_model="dynamic",
- dbg="off",
- opt="off",
- ICECC="icecc",
- CCACHE="ccache",
- NINJA_PREFIX="fast",
- VARIANT_DIR="fast",
- disable_warnings_as_errors=[],
- ),
- # This build has fast runtime speed & debuggability at the cost of build time.
- "opt":
- BuildProfile(
- ninja="enabled",
- variables_files=[
- './etc/scons/mongodbtoolchain_stable_clang.vars',
- './etc/scons/developer_versions.vars',
- ],
- allocator="auto",
- sanitize=None,
- link_model="dynamic",
- dbg="off",
- opt="on",
- ICECC="icecc",
- CCACHE="ccache",
- NINJA_PREFIX="opt",
- VARIANT_DIR="opt",
- disable_warnings_as_errors=[],
- ),
- # This build leverages santizers & is the suggested build profile to use for development.
- "san":
- BuildProfile(
- ninja="enabled",
- variables_files=[
- './etc/scons/mongodbtoolchain_stable_clang.vars',
- './etc/scons/developer_versions.vars',
- ],
- allocator="system",
- sanitize="undefined,address",
- link_model="dynamic",
- dbg="on",
- opt="off",
- ICECC="icecc",
- CCACHE="ccache",
- NINJA_PREFIX="san",
- VARIANT_DIR="san",
- disable_warnings_as_errors=[],
- ),
-
- #These options are the preferred settings for compiledb to generating compile_commands.json
- "compiledb":
- BuildProfile(
- ninja="disabled",
- variables_files=[
- './etc/scons/mongodbtoolchain_stable_clang.vars',
- './etc/scons/developer_versions.vars',
- ],
- allocator="auto",
- sanitize=None,
- link_model="dynamic",
- dbg="on",
- opt="off",
- ICECC=None,
- CCACHE=None,
- NINJA_PREFIX="build",
- VARIANT_DIR=mongo_generators.default_variant_dir_generator,
- disable_warnings_as_errors=['source'],
- ),
-}
diff --git a/site_scons/mongo/generators.py b/site_scons/mongo/generators.py
index 415486587f1..e2b401a5eae 100644
--- a/site_scons/mongo/generators.py
+++ b/site_scons/mongo/generators.py
@@ -88,7 +88,6 @@ def empty_buildinfo_environment_data():
return {}
-# TODO: SERVER-69064 Improve default_variant_dir_generator in Build System
def default_variant_dir_generator(target, source, env, for_signature):
if env.GetOption('cache') != None:
diff --git a/site_scons/site_tools/mongo_test_list.py b/site_scons/site_tools/mongo_test_list.py
index 9b1a8ae6a3b..1b02c52eb8e 100644
--- a/site_scons/site_tools/mongo_test_list.py
+++ b/site_scons/site_tools/mongo_test_list.py
@@ -28,7 +28,7 @@ from collections import defaultdict
TEST_REGISTRY = defaultdict(list)
-def register_test(env, file, test, generate_alias=True):
+def register_test(env, file, test):
"""Register test into the dictionary of tests for file_name"""
test_path = test
if env.get("AUTO_INSTALL_ENABLED", False) and env.GetAutoInstalledFiles(test):
@@ -40,8 +40,7 @@ def register_test(env, file, test, generate_alias=True):
env.Depends(file, test_path)
file_name = file.path
TEST_REGISTRY[file_name].append(test_path)
- if generate_alias:
- env.GenerateTestExecutionAliases(test)
+ env.GenerateTestExecutionAliases(test)
def test_list_builder_action(env, target, source):
diff --git a/site_scons/site_tools/mongo_unittest.py b/site_scons/site_tools/mongo_unittest.py
index 417d8ddce47..33373282606 100644
--- a/site_scons/site_tools/mongo_unittest.py
+++ b/site_scons/site_tools/mongo_unittest.py
@@ -25,19 +25,11 @@ from SCons.Script import Action
from site_scons.mongo import insort_wrapper
-LAST_TEST_GROUP = 0
-
-TEST_GROUPS = ['first', 'second', 'third', 'fourth']
-
-
def exists(env):
return True
def build_cpp_unit_test(env, target, source, **kwargs):
-
- global LAST_TEST_GROUP
-
if not isinstance(target, list):
target = [target]
@@ -45,11 +37,6 @@ def build_cpp_unit_test(env, target, source, **kwargs):
if not t.endswith('_test'):
env.ConfError(f"CppUnitTest target `{t}' does not end in `_test'")
- test_group = TEST_GROUPS[LAST_TEST_GROUP]
- LAST_TEST_GROUP += 1
- if LAST_TEST_GROUP > len(TEST_GROUPS) - 1:
- LAST_TEST_GROUP = 0
-
if not kwargs.get("UNITTEST_HAS_CUSTOM_MAINLINE", False):
libdeps = kwargs.get("LIBDEPS", env.get("LIBDEPS", [])).copy()
insort_wrapper(libdeps, "$BUILD_DIR/mongo/unittest/unittest_main")
@@ -62,7 +49,8 @@ def build_cpp_unit_test(env, target, source, **kwargs):
elif primary_component:
kwargs["AIB_COMPONENT"] = primary_component
else:
- kwargs["AIB_COMPONENT"] = f"{test_group}-quarter-unittests"
+ kwargs["AIB_COMPONENT"] = "unittests"
+ unit_test_components = {"tests"}
if "AIB_COMPONENTS_EXTRA" in kwargs:
kwargs["AIB_COMPONENTS_EXTRA"] = set(kwargs["AIB_COMPONENTS_EXTRA"]).union(
@@ -75,18 +63,10 @@ def build_cpp_unit_test(env, target, source, **kwargs):
env.RegisterTest("$UNITTEST_LIST", result[0])
env.Alias("$UNITTEST_ALIAS", result[0])
- env.RegisterTest(f"$BUILD_ROOT/{test_group}_quarter_unittests.txt", result[0],
- generate_alias=False)
- env.Alias(f"install-{test_group}-quarter-unittests", result[0])
-
return result
def generate(env):
- for test_group in TEST_GROUPS:
- env.TestList(f"$BUILD_ROOT/{test_group}_quarter_unittests.txt", source=[])
- env.Alias(f"install-{test_group}-quarter-unittests",
- f"$BUILD_ROOT/{test_group}_quarter_unittests.txt")
env.TestList("$UNITTEST_LIST", source=[])
env.AddMethod(build_cpp_unit_test, "CppUnitTest")
env.Alias("$UNITTEST_ALIAS", "$UNITTEST_LIST")
diff --git a/site_scons/site_tools/ninja.py b/site_scons/site_tools/ninja.py
index 30c4dbe9d58..8505ce8e7ba 100644
--- a/site_scons/site_tools/ninja.py
+++ b/site_scons/site_tools/ninja.py
@@ -57,24 +57,6 @@ COMMAND_TYPES = (
SCons.Action.CommandGeneratorAction,
)
-ninja_compdb_adjust = """\
-import json
-import sys
-
-compdb = {}
-with open(sys.argv[1]) as f:
- compdb = json.load(f)
-
-for command in compdb:
- if command['output'].endswith('.compdb'):
- command['output'] = command['output'][:-(len('.compdb'))]
- else:
- print(f"compdb entry does not contain '.compdb': {command['output']}")
-
-with open(sys.argv[1], 'w') as f:
- json.dump(compdb, f, indent=2)
-"""
-
def _install_action_function(_env, node):
"""Install files using the install or copy commands"""
@@ -83,7 +65,6 @@ def _install_action_function(_env, node):
"rule": "INSTALL",
"inputs": [get_path(src_file(s)) for s in node.sources],
"implicit": get_dependencies(node),
- "variables": {"precious": node.precious},
}
@@ -98,11 +79,9 @@ def _mkdir_action_function(env, node):
# to an invalid ninja file.
"variables": {
# On Windows mkdir "-p" is always on
- "cmd":
- "mkdir {args}".format(
- args=' '.join(get_outputs(node)) + " & exit /b 0"
- if env["PLATFORM"] == "win32" else "-p " + ' '.join(get_outputs(node)), ),
- "variables": {"precious": node.precious},
+ "cmd": "{mkdir} $out".format(
+ mkdir="mkdir" if env["PLATFORM"] == "win32" else "mkdir -p",
+ ),
},
}
@@ -122,7 +101,6 @@ def _lib_symlink_action_function(_env, node):
"inputs": inputs,
"rule": "SYMLINK",
"implicit": get_dependencies(node),
- "variables": {"precious": node.precious},
}
@@ -327,11 +305,6 @@ class SConsToNinjaTranslator:
if callable(node_callback):
node_callback(env, node, build)
- if build is not None and node.precious:
- if not build.get('variables'):
- build['variables'] = {}
- build['variables']['precious'] = node.precious
-
return build
def handle_func_action(self, node, action):
@@ -436,6 +409,14 @@ class SConsToNinjaTranslator:
"implicit": dependencies,
}
+ elif results[0]["rule"] == "INSTALL":
+ return {
+ "outputs": all_outputs,
+ "rule": "INSTALL",
+ "inputs": [get_path(src_file(s)) for s in node.sources],
+ "implicit": dependencies,
+ }
+
raise Exception("Unhandled list action with rule: " + results[0]["rule"])
@@ -465,21 +446,14 @@ class NinjaState:
scons_escape = env.get("ESCAPE", lambda x: x)
self.variables = {
- # The /b option here will make sure that windows updates the mtime
- # when copying the file. This allows to not need to use restat for windows
- # copy commands.
- "COPY":
- "cmd.exe /c 1>NUL copy /b" if sys.platform == "win32" else "cp",
- "NOOP":
- "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null",
- "SCONS_INVOCATION":
- "{} {} __NINJA_NO=1 $out".format(
- sys.executable,
- " ".join([
- ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv
- if arg not in COMMAND_LINE_TARGETS
- ]),
+ "COPY": "cmd.exe /c 1>NUL copy" if sys.platform == "win32" else "cp",
+ "NOOP": "cmd.exe /c 1>NUL echo 0" if sys.platform == "win32" else "echo 0 >/dev/null",
+ "SCONS_INVOCATION": "{} {} __NINJA_NO=1 $out".format(
+ sys.executable,
+ " ".join(
+ [ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv if arg not in COMMAND_LINE_TARGETS]
),
+ ),
"SCONS_INVOCATION_W_TARGETS": "{} {}".format(
sys.executable, " ".join([ninja_syntax.escape(scons_escape(arg)) for arg in sys.argv])
),
@@ -513,18 +487,6 @@ class NinjaState:
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
},
- "COMPDB_CC": {
- "command": "$CC @$out.rsp",
- "description": "Compiling $out",
- "rspfile": "$out.rsp",
- "rspfile_content": "$rspc",
- },
- "COMPDB_CXX": {
- "command": "$CXX @$out.rsp",
- "description": "Compiling $out",
- "rspfile": "$out.rsp",
- "rspfile_content": "$rspc",
- },
"LINK": {
"command": "$env$LINK @$out.rsp",
"description": "Linking $out",
@@ -532,9 +494,17 @@ class NinjaState:
"rspfile_content": "$rspc",
"pool": "local_pool",
},
+ # Ninja does not automatically delete the archive before
+ # invoking ar. The ar utility will append to an existing archive, which
+ # can cause duplicate symbols if the symbols moved between object files.
+ # Native SCons will perform this operation so we need to force ninja
+ # to do the same. See related for more info:
+ # https://jira.mongodb.org/browse/SERVER-49457
"AR": {
- "command": "$env$AR @$out.rsp",
- "description": "Archived $out",
+ "command": "{}$env$AR @$out.rsp".format(
+ '' if sys.platform == "win32" else "rm -f $out && "
+ ),
+ "description": "Archiving $out",
"rspfile": "$out.rsp",
"rspfile_content": "$rspc",
"pool": "local_pool",
@@ -556,6 +526,15 @@ class NinjaState:
"command": "$COPY $in $out",
"description": "Install $out",
"pool": "install_pool",
+ # On Windows cmd.exe /c copy does not always correctly
+ # update the timestamp on the output file. This leads
+ # to a stuck constant timestamp in the Ninja database
+ # and needless rebuilds.
+ #
+ # Adding restat here ensures that Ninja always checks
+ # the copy updated the timestamp and that Ninja has
+ # the correct information.
+ "restat": 1,
},
"TEMPLATE": {
"command": "$SCONS_INVOCATION $out",
@@ -668,9 +647,6 @@ class NinjaState:
ninja.comment("Generated by scons. DO NOT EDIT.")
- # This version is needed because it is easy to get from pip and it support compile_commands.json
- ninja.variable("ninja_required_version", "1.10")
-
ninja.variable("builddir", get_path(self.env['NINJA_BUILDDIR']))
for pool_name, size in self.pools.items():
@@ -679,34 +655,10 @@ class NinjaState:
for var, val in self.variables.items():
ninja.variable(var, val)
- # This is the command that is used to clean a target before building it,
- # excluding precious targets.
- if sys.platform == "win32":
- rm_cmd = f'cmd.exe /c del /q $rm_outs >nul 2>&1 &'
- else:
- rm_cmd = 'rm -f $rm_outs;'
-
- precious_rule_suffix = "_PRECIOUS"
-
- # Make two sets of rules to honor scons Precious setting. The build nodes themselves
- # will then reselect their rule according to the precious being set for that node.
- precious_rules = {}
for rule, kwargs in self.rules.items():
if self.env.get('NINJA_MAX_JOBS') is not None and 'pool' not in kwargs:
kwargs['pool'] = 'local_pool'
- # Do not worry about precious for commands that don't have targets (phony)
- # or that will callback to scons (which maintains its own precious).
- if rule not in ['phony', 'TEMPLATE', 'REGENERATE', 'COMPDB_CC', 'COMPDB_CXX']:
- precious_rule = rule + precious_rule_suffix
- precious_rules[precious_rule] = kwargs.copy()
- ninja.rule(precious_rule, **precious_rules[precious_rule])
-
- kwargs['command'] = f"{rm_cmd} " + kwargs['command']
- ninja.rule(rule, **kwargs)
- else:
-
- ninja.rule(rule, **kwargs)
- self.rules.update(precious_rules)
+ ninja.rule(rule, **kwargs)
# If the user supplied an alias to determine generated sources, use that, otherwise
# determine what the generated sources are dynamically.
@@ -761,45 +713,6 @@ class NinjaState:
template_builders = []
- # If we ever change the name/s of the rules that include
- # compile commands (i.e. something like CC) we will need to
- # update this build to reflect that complete list.
- compile_commands = "compile_commands.json"
- compdb_expand = '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
- adjust_script_out = os.path.join(
- get_path(self.env['NINJA_BUILDDIR']), 'ninja_compdb_adjust.py')
- os.makedirs(os.path.dirname(adjust_script_out), exist_ok=True)
- with open(adjust_script_out, 'w') as f:
- f.write(ninja_compdb_adjust)
- self.builds[compile_commands] = {
- 'rule': "CMD",
- 'outputs': [compile_commands],
- 'pool': "console",
- 'implicit': [ninja_file],
- 'variables': {
- "cmd":
- f"ninja -f {ninja_file} -t compdb {compdb_expand}COMPDB_CC COMPDB_CXX > {compile_commands};"
- + f"{sys.executable} {adjust_script_out} {compile_commands}"
- },
- }
- self.builds["compiledb"] = {
- 'rule': "phony",
- "outputs": ["compiledb"],
- 'implicit': [compile_commands],
- }
-
- # Now for all build nodes, we want to select the precious rule or not.
- # If it's not precious, we need to save all the outputs into a variable
- # on that node. Later we will be removing outputs and switching them to
- # phonies so that we can generate response and depfiles correctly.
- for build, kwargs in self.builds.items():
- if kwargs.get('variables') and kwargs['variables'].get('precious'):
- kwargs['rule'] = kwargs['rule'] + precious_rule_suffix
- elif kwargs['rule'] not in ['phony', 'TEMPLATE', 'REGENERATE']:
- if not kwargs.get('variables'):
- kwargs['variables'] = {}
- kwargs['variables']['rm_outs'] = kwargs['outputs'].copy()
-
for build in [self.builds[key] for key in sorted(self.builds.keys())]:
if build["rule"] == "TEMPLATE":
template_builders.append(build)
@@ -890,20 +803,6 @@ class NinjaState:
ninja.build(**build)
- for build, kwargs in self.builds.items():
- if kwargs['rule'] in [
- 'CC', f'CC{precious_rule_suffix}', 'CXX', f'CXX{precious_rule_suffix}'
- ]:
- rule = kwargs['rule'].replace(
- precious_rule_suffix
- ) if precious_rule_suffix in kwargs['rule'] else kwargs['rule']
- rule = "COMPDB_" + rule
- compdb_build = kwargs.copy()
-
- compdb_build['rule'] = rule
- compdb_build['outputs'] = [kwargs['outputs'] + ".compdb"]
- ninja.build(**compdb_build)
-
template_builds = {'rule': "TEMPLATE"}
for template_builder in template_builders:
@@ -964,6 +863,25 @@ class NinjaState:
implicit=[__file__],
)
+ # If we ever change the name/s of the rules that include
+ # compile commands (i.e. something like CC) we will need to
+ # update this build to reflect that complete list.
+ ninja.build(
+ "compile_commands.json",
+ rule="CMD",
+ pool="console",
+ implicit=[ninja_file],
+ variables={
+ "cmd": "ninja -f {} -t compdb {}CC CXX > compile_commands.json".format(
+ ninja_file, '-x ' if self.env.get('NINJA_COMPDB_EXPAND') else ''
+ )
+ },
+ )
+
+ ninja.build(
+ "compiledb", rule="phony", implicit=["compile_commands.json"],
+ )
+
# Look in SCons's list of DEFAULT_TARGETS, find the ones that
# we generated a ninja build rule for.
scons_default_targets = [
@@ -1336,9 +1254,7 @@ def register_custom_rule_mapping(env, pre_subst_string, rule):
__NINJA_RULE_MAPPING[pre_subst_string] = rule
-def register_custom_rule(env, rule, command, description="", deps=None, pool=None,
- use_depfile=False, use_response_file=False, response_file_content="$rspc",
- restat=False):
+def register_custom_rule(env, rule, command, description="", deps=None, pool=None, use_depfile=False, use_response_file=False, response_file_content="$rspc"):
"""Allows specification of Ninja rules from inside SCons files."""
rule_obj = {
"command": command,
@@ -1356,15 +1272,8 @@ def register_custom_rule(env, rule, command, description="", deps=None, pool=Non
if use_response_file:
rule_obj["rspfile"] = "$out.rsp"
- if rule_obj["rspfile"] not in command:
- raise Exception(
- f'Bad Ninja Custom Rule: response file requested, but {rule_obj["rspfile"]} not in in command: {command}'
- )
rule_obj["rspfile_content"] = response_file_content
- if restat:
- rule_obj["restat"] = 1
-
env[NINJA_RULES][rule] = rule_obj
@@ -1541,6 +1450,7 @@ def generate(env):
ninja_file_name = env.subst("${NINJA_PREFIX}.${NINJA_SUFFIX}")
ninja_file = env.Ninja(target=ninja_file_name, source=[])
env.AlwaysBuild(ninja_file)
+ env.Alias("$NINJA_ALIAS_NAME", ninja_file)
# TODO: API for getting the SConscripts programmatically
# exists upstream: https://github.com/SCons/scons/issues/3625
@@ -1566,15 +1476,6 @@ def generate(env):
env.AddMethod(gen_get_response_file_command, "NinjaGenResponseFileProvider")
env.AddMethod(set_build_node_callback, "NinjaSetBuildNodeCallback")
- # Expose ninja node path converstion functions to make writing
- # custom function action handlers easier.
- env.AddMethod(lambda _env, node: get_outputs(node), "NinjaGetOutputs")
- env.AddMethod(lambda _env, node, skip_unknown_types=False: get_inputs(node, skip_unknown_types),
- "NinjaGetInputs")
- env.AddMethod(lambda _env, node, skip_sources=False: get_dependencies(node),
- "NinjaGetDependencies")
- env.AddMethod(lambda _env, node: get_order_only(node), "NinjaGetOrderOnly")
-
# Provides a way for users to handle custom FunctionActions they
# want to translate to Ninja.
env[NINJA_CUSTOM_HANDLERS] = {}
@@ -1669,12 +1570,6 @@ def generate(env):
if not exists(env):
return
- # There is a target called generate-ninja which needs to be included
- # with the --ninja flag in order to generate the ninja file. Because the --ninja
- # flag is ONLY used with generate-ninja, we have combined the two by making the --ninja flag
- # implicitly build the generate-ninja target.
- SCons.Script.BUILD_TARGETS = SCons.Script.TargetList(env.Alias("$NINJA_ALIAS_NAME", ninja_file))
-
# Set a known variable that other tools can query so they can
# behave correctly during ninja generation.
env["GENERATING_NINJA"] = True
diff --git a/site_scons/site_tools/oom_auto_retry.py b/site_scons/site_tools/oom_auto_retry.py
deleted file mode 100644
index 7ff457d2798..00000000000
--- a/site_scons/site_tools/oom_auto_retry.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2023 MongoDB Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
-# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
-# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-
-import SCons
-
-import functools
-import subprocess
-import sys
-import time
-import random
-import os
-import re
-
-from typing import Callable, List, Dict
-
-
-def command_spawn_func(sh: str, escape: Callable[[str], str], cmd: str, args: List, env: Dict,
- target: List, source: List):
- retries = 0
- success = False
-
- build_env = target[0].get_build_env()
- oom_messages = [
- re.compile(msg, re.MULTILINE | re.DOTALL)
- for msg in build_env.get('OOM_RETRY_MESSAGES', [])
- ]
- oom_returncodes = [int(returncode) for returncode in build_env.get('OOM_RETRY_RETURNCODES', [])]
- max_retries = build_env.get('OOM_RETRY_ATTEMPTS', 10)
- oom_max_retry_delay = build_env.get('OOM_RETRY_MAX_DELAY_SECONDS', 120)
-
- while not success and retries <= max_retries:
-
- try:
- start_time = time.time()
- if sys.platform[:3] == 'win':
- # have to use shell=True for windows because of https://github.com/python/cpython/issues/53908
- proc = subprocess.run(' '.join(args), env=env, close_fds=True, shell=True,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
- check=True)
- else:
- proc = subprocess.run([sh, '-c', ' '.join(args)], env=env, close_fds=True,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
- check=True)
- except subprocess.CalledProcessError as exc:
- print(f"{os.path.basename(__file__)} captured error:")
- print(exc.stdout)
- if any([re.findall(oom_message, exc.stdout) for oom_message in oom_messages]) or any(
- [oom_returncode == exc.returncode for oom_returncode in oom_returncodes]):
- retries += 1
- retry_delay = int((time.time() - start_time) +
- oom_max_retry_delay * random.random())
- print(f"Ran out of memory while trying to build {target[0]}", )
- if retries <= max_retries:
- print(f"trying again in {retry_delay} seconds with retry attempt {retries}")
- time.sleep(retry_delay)
- continue
-
- # There was no OOM error or no more OOM retries left
- return exc.returncode
- else:
- if proc.stdout:
- print(proc.stdout)
- return proc.returncode
-
-
-def generate(env):
-
- original_command_execute = SCons.Action.CommandAction.execute
-
- def oom_retry_execute(command_action_instance, target, source, env, executor=None):
-
- if 'conftest' not in str(target[0]) and target[0].has_builder() and target[0].get_builder(
- ).get_name(env) in [
- 'Object', 'SharedObject', 'StaticObject', 'Program', 'StaticLibrary',
- 'SharedLibrary'
- ]:
-
- original_spawn = env['SPAWN']
-
- env['SPAWN'] = functools.partial(command_spawn_func, target=target, source=source)
- result = original_command_execute(command_action_instance, target, source, env,
- executor)
- env['SPAWN'] = original_spawn
-
- else:
- result = original_command_execute(command_action_instance, target, source, env,
- executor)
- return result
-
- SCons.Action.CommandAction.execute = oom_retry_execute
-
-
-def exists(env):
- return True