diff options
Diffstat (limited to 'buildscripts/resmokelib/logging')
| -rw-r--r-- | buildscripts/resmokelib/logging/buildlogger.py | 8 | ||||
| -rw-r--r-- | buildscripts/resmokelib/logging/formatters.py | 14 | ||||
| -rw-r--r-- | buildscripts/resmokelib/logging/handlers.py | 23 | ||||
| -rw-r--r-- | buildscripts/resmokelib/logging/loggers.py | 66 |
4 files changed, 0 insertions, 111 deletions
diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index 8aa6b6bd983..6325ecabb89 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -14,7 +14,6 @@ CREATE_BUILD_ENDPOINT = "/build" APPEND_GLOBAL_LOGS_ENDPOINT = "/build/%(build_id)s" CREATE_TEST_ENDPOINT = "/build/%(build_id)s/test" APPEND_TEST_LOGS_ENDPOINT = "/build/%(build_id)s/test/%(test_id)s" -PARSLEY_LOGS_URL = "https://parsley.mongodb.com/resmoke/%(build_id)s/test/%(test_id)s" _BUILDLOGGER_CONFIG = os.getenv("BUILDLOGGER_CREDENTIALS", "mci.buildlogger") @@ -299,7 +298,6 @@ class BuildloggerServer(object): "builder": builder, "buildnum": build_num, "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -318,7 +316,6 @@ class BuildloggerServer(object): "command": test_command, "phase": self.config.get("build_phase", "unknown"), "task_id": _config.EVERGREEN_TASK_ID, - "execution": _config.EVERGREEN_EXECUTION, }) return response["id"] @@ -344,8 +341,3 @@ class BuildloggerServer(object): base_url = _config.BUILDLOGGER_URL.rstrip("/") endpoint = APPEND_TEST_LOGS_ENDPOINT % {"build_id": build_id, "test_id": test_id} return "%s/%s" % (base_url, endpoint.strip("/")) - - @staticmethod - def get_parsley_log_url(build_id, test_id): - """Return the parsley log URL.""" - return PARSLEY_LOGS_URL % {"build_id": build_id, "test_id": test_id} diff --git a/buildscripts/resmokelib/logging/formatters.py b/buildscripts/resmokelib/logging/formatters.py index b4ee2abd4b5..2998756a27e 100644 --- a/buildscripts/resmokelib/logging/formatters.py +++ b/buildscripts/resmokelib/logging/formatters.py @@ -19,17 +19,3 @@ class TimestampFormatter(logging.Formatter): formatted_time = time.strftime("%H:%M:%S", converted_time) return "%s.%03dZ" % (formatted_time, record.msecs) - - -class EvergreenLogFormatter(logging.Formatter): - """Log line formatter for Evergreen log messages. - - See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` - for more info. - """ - - def format(self, record): - """Return formatted line.""" - ts = int(record.created * 1e9) - - return "\n".join([f"{ts} {line}" for line in super().format(record).split("\n")]) diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index 07bc214517a..692b4532c47 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -161,29 +161,6 @@ class BufferedHandler(logging.Handler): logging.Handler.close(self) -class BufferedFileHandler(BufferedHandler): - """File handler with in-memory buffering.""" - - def __init__(self, filename, capacity=2000, interval_secs=600): - """Initialize the handler with the filename and buffer capacity and flush interval.""" - super().__init__(capacity, interval_secs) - self.file = open(filename, "a", encoding="utf-8") - - def process_record(self, record): - """Return the formatted record message appended with a newline.""" - return self.format(record) + "\n" - - def _flush_buffer_with_lock(self, buf, close_called): - """Write the buffered log lines to the destination file.""" - self.file.writelines(buf) - - def close(self): - """Close the handler and the file descriptor.""" - super().close() - - self.file.close() - - class HTTPHandler(object): """A class which sends data to a web server using POST requests.""" diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index 8775c830669..6d5fee06912 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -1,19 +1,16 @@ """Module to hold the logger instances themselves.""" import logging -import os import re import shutil import subprocess import sys -import yaml from buildscripts.resmokelib import config from buildscripts.resmokelib import errors from buildscripts.resmokelib.core import redirect as redirect_lib from buildscripts.resmokelib.logging import buildlogger from buildscripts.resmokelib.logging import formatters -from buildscripts.resmokelib.logging.handlers import BufferedFileHandler _DEFAULT_FORMAT = "[%(name)s] %(message)s" @@ -97,13 +94,6 @@ def configure_loggers(): global ROOT_EXECUTOR_LOGGER # pylint: disable=global-statement ROOT_EXECUTOR_LOGGER = new_root_logger(EXECUTOR_LOGGER_NAME) - _write_evergreen_log_spec() - - -def get_evergreen_log_name(job_num, test_id=None): - """Return the log name, relative to the reserved test log directory, on the Evergreen task host.""" - return f"job{job_num}/" + (f"{test_id}.log" if test_id else "global.log") - def new_root_logger(name): """ @@ -180,7 +170,6 @@ def new_fixture_logger(fixture_class, job_num): logger = FixtureLogger(_shorten(full_name), full_name) logger.parent = ROOT_FIXTURE_LOGGER _add_build_logger_handler(logger, job_num) - _add_evergreen_handler(logger, job_num) _FIXTURE_LOGGER_REGISTRY[job_num] = logger return logger @@ -210,7 +199,6 @@ def new_test_logger(test_shortname, test_basename, command, parent, job_num, tes name = "%s:%s" % (parent.name, test_shortname) logger = logging.Logger(name) logger.parent = parent - _add_evergreen_handler(logger, job_num, test_id) def _get_test_endpoint(job_num, test_basename, command, meta_logger): """Get a new test endpoint for the buildlogger server.""" @@ -268,8 +256,6 @@ def _add_handler(logger, handler_info, formatter): handler = logging.StreamHandler(sys.stdout) elif handler_class == "buildlogger": return # Buildlogger handlers are applied when creating specific child loggers - elif handler_class == "evergreen": - return # Evergreen handlers are applied when creating specific child loggers else: raise ValueError("Unknown handler class '%s'" % handler_class) handler.setFormatter(formatter) @@ -339,55 +325,3 @@ def _shorten(logger_name): logger_name = re.sub(r"(^[:_]+|[:_]+$)", "", logger_name) return logger_name - - -# Utility functions for Evergreen file system logging. -# See `https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Task-Output-Directory#test-logs` -# for more information. - - -def _write_evergreen_log_spec(): - """Configure file system logging for Evergreen tasks.""" - if not config.EVERGREEN_WORK_DIR: - return - - fp = f"{_get_evergreen_log_dirname()}/log_spec.yaml" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - ROOT_EXECUTOR_LOGGER.info("Writing Evergreen test log spec to %s.", fp) - - log_spec = { - "version": 0, - "format": "text-timestamp", - } - with open(fp, "w") as fd: - yaml.dump(log_spec, fd) - - -def _add_evergreen_handler(logger, job_num, test_id=None): - """Add a new evergreen handler to a logger.""" - logger_info = config.LOGGING_CONFIG[TESTS_LOGGER_NAME] - evergreen_handler_info = None - for handler_info in logger_info["handlers"]: - if handler_info["class"] == "evergreen": - evergreen_handler_info = handler_info - break - - if evergreen_handler_info: - fp = f"{_get_evergreen_log_dirname()}/{get_evergreen_log_name(job_num, test_id)}" - os.makedirs(os.path.dirname(fp), exist_ok=True) - - handler = BufferedFileHandler(fp) - handler.setFormatter( - formatters.EvergreenLogFormatter(fmt=logger_info.get("format", _DEFAULT_FORMAT))) - logger.addHandler(handler) - - if test_id: - ROOT_EXECUTOR_LOGGER.info("Writing output of %s to %s.", test_id, fp) - else: - ROOT_EXECUTOR_LOGGER.info("Writing output of job #%d to %s.", job_num, fp) - - -def _get_evergreen_log_dirname(): - """Return the reserved directory for test logs on the Evergreen task host.""" - return f"{config.EVERGREEN_WORK_DIR}/build/TestLogs" |
