Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 70 insertions, 168 deletions

diff --git a/buildscripts/debugsymb_mapper.py b/buildscripts/debugsymb_mapper.py
index 0f79b4349fe..977716a7c0f 100644
--- a/buildscripts/debugsymb_mapper.py
+++ b/buildscripts/debugsymb_mapper.py
@@ -4,13 +4,11 @@ import json
 import logging
 import os
 import pathlib
-import re
 import shutil
 import subprocess
 import sys
 import time
-from json import JSONDecoder
-from typing import Optional, Tuple, Generator, Dict, List, NamedTuple
+import typing
 
 import requests
 
@@ -22,95 +20,30 @@ from buildscripts.util.oauth import get_client_cred_oauth_credentials, Configs
 from buildscripts.resmokelib.setup_multiversion.setup_multiversion import SetupMultiversion, download
 from buildscripts.build_system_options import PathOptions
 
-BUILD_INFO_RE = re.compile(r"Build Info: ({(\n.*)*})")
-MONGOD = "mongod"
 
+class LinuxBuildIDExtractor:
+    """Parse readlef command output & extract Build ID."""
 
-class CmdClient:
-    """Client to run commands."""
+    default_executable_path = "readelf"
 
-    @staticmethod
-    def run(args: List[str]) -> str:
-        """
-        Run command with args.
-
-        :param args: Argument list.
-        :return: Command output.
-        """
-
-        out = subprocess.run(args, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
-                             check=False)
-        return out.stdout.strip().decode()
-
-
-class BuildIdOutput(NamedTuple):
-    """
-    Build ID and command output.
-
-    * build_id: Build ID or None.
-    * cmd_output: Command output.
-    """
-
-    build_id: Optional[str]
-    cmd_output: str
-
-
-class BinVersionOutput(NamedTuple):
-    """
-    Mongodb bin version and command output.
-
-    * mongodb_version: Bin version.
-    * cmd_output: Command output.
-    """
-
-    mongodb_version: Optional[str]
-    cmd_output: str
-
-
-class CmdOutputExtractor:
-    """Data extractor from command output."""
-
-    def __init__(self, cmd_client: Optional[CmdClient] = None,
-                 json_decoder: Optional[JSONDecoder] = None) -> None:
-        """
-        Initialize.
-
-        :param cmd_client: Client to run commands.
-        :param json_decoder: JSONDecoder object.
-        """
-        self.cmd_client = cmd_client if cmd_client is not None else CmdClient()
-        self.json_decoder = json_decoder if json_decoder is not None else JSONDecoder()
-
-    def get_build_id(self, bin_path: str) -> BuildIdOutput:
-        """
-        Get build ID from readelf command.
+    def __init__(self, executable_path: str = None):
+        """Initialize instance."""
 
-        :param bin_path: Path to binary of the build.
-        :return: Build ID or None and command output.
-        """
-        out = self.cmd_client.run(["readelf", "-n", bin_path])
-        build_id = self._extract_build_id(out)
-        return BuildIdOutput(build_id, out)
+        self.executable_path = executable_path or self.default_executable_path
 
-    def get_bin_version(self, bin_path: str) -> BinVersionOutput:
-        """
-        Get mongodb bin version from `{bin} --version` command.
+    def callreadelf(self, binary_path: str) -> str:
+        """Call readelf command for given binary & return string output."""
 
-        :param bin_path: Path to mongodb binary.
-        :return: Bin version or None and command output.
-        """
-        out = self.cmd_client.run([os.path.abspath(bin_path), "--version"])
-        mongodb_version = self._get_mongodb_version(out)
-        return BinVersionOutput(mongodb_version, out)
+        args = [self.executable_path, "-n", binary_path]
+        process = subprocess.Popen(args=args, close_fds=True, stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE)
+        process.wait()
+        return process.stdout.read().decode()
 
     @staticmethod
-    def _extract_build_id(out: str) -> Optional[str]:
-        """
-        Parse readelf output and extract Build ID from it.
+    def extractbuildid(out: str) -> typing.Optional[str]:
+        """Parse readelf output and extract Build ID from it."""
 
-        :param out: readelf command output.
-        :return: Build ID on None.
-        """
         build_id = None
         for line in out.splitlines():
             line = line.strip()
@@ -120,21 +53,13 @@ class CmdOutputExtractor:
                 build_id = line.split(': ')[1]
         return build_id
 
-    def _get_mongodb_version(self, out: str) -> Optional[str]:
-        """
-        Parse version command output and extract mongodb version.
-
-        :param out: Version command output.
-        :return: Version or None.
-        """
-        mongodb_version = None
+    def run(self, binary_path: str) -> typing.Tuple[str, str]:
+        """Perform all necessary actions to get Build ID."""
 
-        search = BUILD_INFO_RE.search(out)
-        if search:
-            build_info = self.json_decoder.decode(search.group(1))
-            mongodb_version = build_info.get("version")
+        readelfout = self.callreadelf(binary_path)
+        buildid = self.extractbuildid(readelfout)
 
-        return mongodb_version
+        return buildid, readelfout
 
 
 class DownloadOptions(object):
@@ -158,29 +83,24 @@ class Mapper:
 
     default_web_service_base_url: str = "https://symbolizer-service.server-tig.prod.corp.mongodb.com"
     default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache')
-    selected_binaries = ('mongos', 'mongod', 'mongo')
+    selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug')
     default_client_credentials_scope = "servertig-symbolizer-fullaccess"
     default_client_credentials_user_name = "client-user"
     default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
 
-    def __init__(self, evg_version: str, evg_variant: str, is_san_variant: bool, client_id: str,
-                 client_secret: str, cache_dir: str = None, web_service_base_url: str = None,
+    def __init__(self, version: str, client_id: str, client_secret: str, variant: str,
+                 cache_dir: str = None, web_service_base_url: str = None,
                  logger: logging.Logger = None):
         """
         Initialize instance.
 
-        :param evg_version: Evergreen version ID.
-        :param evg_variant: Evergreen build variant name.
-        :param is_san_variant: Whether build variant is sanitizer build.
-        :param client_id: Client id for Okta Oauth.
-        :param client_secret: Secret key for Okta Oauth.
-        :param cache_dir: Full path to cache directory as a string.
-        :param web_service_base_url: URL of symbolizer web service.
-        :param logger: Debug symbols mapper logger.
+        :param version: version string
+        :param variant: build variant string
+        :param cache_dir: full path to cache directory as a string
+        :param web_service_base_url: URL of symbolizer web service
         """
-        self.evg_version = evg_version
-        self.evg_variant = evg_variant
-        self.is_san_variant = is_san_variant
+        self.version = version
+        self.variant = variant
         self.cache_dir = cache_dir or self.default_cache_dir
         self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
 
@@ -193,8 +113,8 @@ class Mapper:
         self.http_client = requests.Session()
 
         self.multiversion_setup = SetupMultiversion(
-            DownloadOptions(download_symbols=True, download_binaries=True),
-            variant=self.evg_variant, ignore_failed_push=True)
+            DownloadOptions(download_symbols=True, download_binaries=True), variant=self.variant,
+            ignore_failed_push=True)
         self.debug_symbols_url = None
         self.url = None
         self.configs = Configs(
@@ -219,7 +139,7 @@ class Mapper:
                 data = json.loads(cfile.read())
                 access_token, expire_time = data.get("access_token"), data.get("expire_time")
                 if time.time() < expire_time:
-                    # credentials haven't expired yet
+                    # credentials hasn't expired yet
                     self.http_client.headers.update({"Authorization": f"Bearer {access_token}"})
                     return
 
@@ -227,7 +147,7 @@ class Mapper:
                                                         configs=self.configs)
         self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"})
 
-        # write credentials to local file for further usage
+        # write credentials to local file for further useage
         with open(self.default_creds_file_path, "w") as cfile:
             cfile.write(
                 json.dumps({
@@ -264,19 +184,17 @@ class Mapper:
     def setup_urls(self):
         """Set up URLs using multiversion."""
 
-        urlinfo = self.multiversion_setup.get_urls(self.evg_version, self.evg_variant)
+        urlinfo = self.multiversion_setup.get_urls(self.version, self.variant)
 
+        download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None)
         binaries_url = urlinfo.urls.get("Binaries", "")
-        if self.is_san_variant:
-            # Sanitizer builds are not stripped and contain debug symbols
-            download_symbols_url = binaries_url
-        else:
-            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz") or urlinfo.urls.get(
-                "mongo-debugsymbols.zip")
+
+        if not download_symbols_url:
+            download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None)
 
         if not download_symbols_url:
             self.logger.error("Couldn't find URL for debug symbols. Version: %s, URLs dict: %s",
-                              self.evg_version, urlinfo.urls)
+                              self.version, urlinfo.urls)
             raise ValueError(f"Debug symbols URL not found. URLs dict: {urlinfo.urls}")
 
         self.debug_symbols_url = download_symbols_url
@@ -315,64 +233,54 @@ class Mapper:
         tarball_full_path = download.download_from_s3(url)
         return tarball_full_path
 
-    def generate_build_id_mapping(self) -> Generator[Dict[str, str], None, None]:
+    def generate_build_id_mapping(self) -> typing.Generator[typing.Dict[str, str], None, None]:
         """
         Extract build id from binaries and creates new dict using them.
 
         :return: mapped data as dict
         """
 
-        extractor = CmdOutputExtractor()
+        readelf_extractor = LinuxBuildIDExtractor()
+
+        debug_symbols_path = self.download(self.debug_symbols_url)
+        debug_symbols_unpacked_path = self.unpack(debug_symbols_path)
 
         binaries_path = self.download(self.url)
         binaries_unpacked_path = self.unpack(binaries_path)
 
-        # we need to analyze two directories: main binary folder and
+        # we need to analyze two directories: main binary folder inside debug-symbols and
         # shared libraries folder inside binaries.
         # main binary folder holds main binaries, like mongos, mongod, mongo ...
         # shared libraries folder holds shared libraries, tons of them.
         # some build variants do not contain shared libraries.
 
+        debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test')
         binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test')
 
+        self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s",
+                         os.listdir(debug_symbols_unpacked_path))
         self.logger.info("INSIDE unpacked binaries/dist-test: %s",
                          os.listdir(binaries_unpacked_path))
 
-        mongod_bin = os.path.join(binaries_unpacked_path, self.path_options.main_binary_folder_name,
-                                  MONGOD)
-        bin_version_output = extractor.get_bin_version(mongod_bin)
-
-        if bin_version_output.mongodb_version is None:
-            self.logger.error("mongodb version could not be extracted. \n`%s --version` output: %s",
-                              mongod_bin, bin_version_output.cmd_output)
-            return
-        else:
-            self.logger.info("Extracted mongodb version: %s", bin_version_output.mongodb_version)
-
         # start with main binary folder
         for binary in self.selected_binaries:
-            full_bin_path = os.path.join(binaries_unpacked_path,
+            full_bin_path = os.path.join(debug_symbols_unpacked_path,
                                          self.path_options.main_binary_folder_name, binary)
 
             if not os.path.exists(full_bin_path):
                 self.logger.error("Could not find binary at %s", full_bin_path)
-                continue
+                return
 
-            build_id_output = extractor.get_build_id(full_bin_path)
+            build_id, readelf_out = readelf_extractor.run(full_bin_path)
 
-            if not build_id_output.build_id:
+            if not build_id:
                 self.logger.error("Build ID couldn't be extracted. \nReadELF output %s",
-                                  build_id_output.cmd_output)
-                continue
-            else:
-                self.logger.info("Extracted build ID: %s", build_id_output.build_id)
+                                  readelf_out)
+                return
 
             yield {
-                'url': self.url,
-                'debug_symbols_url': self.debug_symbols_url,
-                'build_id': build_id_output.build_id,
-                'file_name': binary,
-                'version': bin_version_output.mongodb_version,
+                'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id,
+                'file_name': binary, 'version': self.version
             }
 
         # move to shared libraries folder.
@@ -396,23 +304,20 @@ class Mapper:
 
             if not os.path.exists(sofile_path):
                 self.logger.error("Could not find binary at %s", sofile_path)
-                continue
+                return
 
-            build_id_output = extractor.get_build_id(sofile_path)
+            build_id, readelf_out = readelf_extractor.run(sofile_path)
 
-            if not build_id_output.build_id:
-                self.logger.error("Build ID couldn't be extracted. \nReadELF out %s",
-                                  build_id_output.cmd_output)
-                continue
-            else:
-                self.logger.info("Extracted build ID: %s", build_id_output.build_id)
+            if not build_id:
+                self.logger.error("Build ID couldn't be extracted. \nReadELF out %s", readelf_out)
+                return
 
             yield {
                 'url': self.url,
                 'debug_symbols_url': self.debug_symbols_url,
-                'build_id': build_id_output.build_id,
+                'build_id': build_id,
                 'file_name': sofile,
-                'version': bin_version_output.mongodb_version,
+                'version': self.version,
             }
 
     def run(self):
@@ -425,7 +330,6 @@ class Mapper:
 
         # mappings is a generator, we iterate over to generate mappings on the go
         for mapping in mappings:
-            self.logger.info("Creating mapping %s", mapping)
             response = self.http_client.post('/'.join((self.web_service_base_url, 'add')),
                                              json=mapping)
             if response.status_code != 200:
@@ -440,20 +344,18 @@ def make_argument_parser(parser=None, **kwargs):
     if parser is None:
         parser = argparse.ArgumentParser(**kwargs)
 
-    parser.add_argument("--version")
-    parser.add_argument("--client-id")
-    parser.add_argument("--client-secret")
-    parser.add_argument("--variant")
-    parser.add_argument("--is-san-variant", action="store_true")
-    parser.add_argument("--web-service-base-url", default="")
+    parser.add_argument('--version')
+    parser.add_argument('--client-id')
+    parser.add_argument('--client-secret')
+    parser.add_argument('--variant')
+    parser.add_argument('--web-service-base-url', default="")
     return parser
 
 
 def main(options):
     """Execute mapper here. Main entry point."""
 
-    mapper = Mapper(evg_version=options.version, evg_variant=options.variant,
-                    is_san_variant=options.is_san_variant, client_id=options.client_id,
+    mapper = Mapper(version=options.version, variant=options.variant, client_id=options.client_id,
                     client_secret=options.client_secret,
                     web_service_base_url=options.web_service_base_url)