Update upstream source from tag 'upstream/6.0.0'master

Update to upstream version '6.0.0'
with Debian dir 5604a80ec1c96ca76f25f40d78e6ef855abec322

1 files changed, 57 insertions, 164 deletions

diff --git a/buildscripts/mongosymb.py b/buildscripts/mongosymb.py
index 035b00f3a07..7dc529e39d0 100755
--- a/buildscripts/mongosymb.py
+++ b/buildscripts/mongosymb.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """Script and library for symbolizing MongoDB stack traces.
 
 To use as a script, paste the JSON object on the line after ----- BEGIN BACKTRACE ----- into the
@@ -23,42 +24,20 @@ import signal
 import subprocess
 import sys
 import time
-from abc import ABC, abstractmethod
 from collections import OrderedDict
-from datetime import timedelta
 from pathlib import Path
-from typing import Dict, List, Any, Union, Optional
+from typing import Dict
 
 import requests
 
 # pylint: disable=wrong-import-position
 # pylint: disable=too-many-branches
-from tenacity import wait_fixed, stop_after_delay, retry_if_result, Retrying
-
 sys.path.append(str(Path(os.getcwd(), __file__).parent.parent))
-from buildscripts.util.oauth import Configs, get_oauth_credentials, get_client_cred_oauth_credentials
+from buildscripts.util.oauth import Configs, get_oauth_credentials
 from buildscripts.build_system_options import PathOptions
 
-SYMBOLIZER_PATH_ENV = "MONGOSYMB_SYMBOLIZER_PATH"
-# since older versions may have issues with symbolizing, we are setting the toolchain version to v4
-DEFAULT_SYMBOLIZER_PATH = "/opt/mongodbtoolchain/v4/bin/llvm-symbolizer"
-
-
-class DbgFileResolver(ABC):
-    """Base gdb path resolver class."""
-
-    @abstractmethod
-    def get_dbg_file(self, soinfo: Dict[str, Any]) -> Union[str, None]:
-        """
-        To get path for given build info.
-
-        :param soinfo: soinfo as dict
-        :return: path as string or None (if path not found)
-        """
-        raise NotImplementedError
-
 
-class PathDbgFileResolver(DbgFileResolver):
+class PathDbgFileResolver(object):
     """PathDbgFileResolver class."""
 
     def __init__(self, bin_path_guess):
@@ -75,7 +54,7 @@ class PathDbgFileResolver(DbgFileResolver):
         return path if path else self._bin_path_guess
 
 
-class S3BuildidDbgFileResolver(DbgFileResolver):
+class S3BuildidDbgFileResolver(object):
     """S3BuildidDbgFileResolver class."""
 
     def __init__(self, cache_dir, s3_bucket):
@@ -94,7 +73,7 @@ class S3BuildidDbgFileResolver(DbgFileResolver):
         if not os.path.exists(build_id_path):
             try:
                 self._get_from_s3(build_id)
-            except Exception:  # noqa pylint: disable=broad-except
+            except Exception:  # pylint: disable=broad-except
                 ex = sys.exc_info()[0]
                 sys.stderr.write("Failed to find debug symbols for {} in s3: {}\n".format(
                     build_id, ex))
@@ -168,7 +147,7 @@ class CachedResults(object):
         return self._cached_results.get(key)
 
 
-class PathResolver(DbgFileResolver):
+class PathResolver(object):
     """
     Class to find path for given buildId.
 
@@ -184,17 +163,16 @@ class PathResolver(DbgFileResolver):
     # This amount of attributes are necessary.
 
     # the main (API) sever that we'll be sending requests to
-    default_host = "https://symbolizer-service.server-tig.prod.corp.mongodb.com"
-    default_cache_dir = os.path.join(os.getcwd(), "build", "symbolizer_downloads_cache")
-    default_creds_file_path = os.path.join(os.getcwd(), ".symbolizer_credentials.json")
+    default_host = 'https://symbolizer-service.server-tig.prod.corp.mongodb.com'
+    default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbolizer_downloads_cache')
+    default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json')
     default_client_credentials_scope = "servertig-symbolizer-fullaccess"
     default_client_credentials_user_name = "client-user"
-    download_timeout_secs = timedelta(minutes=4).total_seconds()
 
     def __init__(self, host: str = None, cache_size: int = 0, cache_dir: str = None,
                  client_credentials_scope: str = None, client_credentials_user_name: str = None,
-                 client_id: str = None, client_secret: str = None, redirect_port: int = None,
-                 scope: str = None, auth_domain: str = None):
+                 client_id: str = None, redirect_port: int = None, scope: str = None,
+                 auth_domain: str = None):
         """
         Initialize instance.
 
@@ -209,7 +187,6 @@ class PathResolver(DbgFileResolver):
         self.client_credentials_scope = client_credentials_scope or self.default_client_credentials_scope
         self.client_credentials_user_name = client_credentials_user_name or self.default_client_credentials_user_name
         self.client_id = client_id
-        self.client_secret = client_secret
         self.redirect_port = redirect_port
         self.scope = scope
         self.auth_domain = auth_domain
@@ -235,17 +212,11 @@ class PathResolver(DbgFileResolver):
                 data = json.loads(cfile.read())
                 access_token, expire_time = data.get("access_token"), data.get("expire_time")
                 if time.time() < expire_time:
-                    # credentials not expired yet
+                    # credentials hasn't expired yet
                     self.http_client.headers.update({"Authorization": f"Bearer {access_token}"})
                     return
 
-        if self.client_id and self.client_secret:
-            # auth using secrets
-            credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret,
-                                                            self.configs)
-        else:
-            # since we don't have access to secrets, ask user to auth manually
-            credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True)
+        credentials = get_oauth_credentials(configs=self.configs, print_auth_url=True)
         self.http_client.headers.update({"Authorization": f"Bearer {credentials.access_token}"})
 
         # write credentials to local file for further useage
@@ -294,7 +265,7 @@ class PathResolver(DbgFileResolver):
         :param url: download URL
         :return: full name for local file
         """
-        return url.split("/")[-1]
+        return url.split('/')[-1]
 
     @staticmethod
     def unpack(path: str) -> str:
@@ -302,9 +273,9 @@ class PathResolver(DbgFileResolver):
         Use to utar/unzip files.
 
         :param path: full path of file
-        :return: full path to directory of unpacked file
+        :return: full path of directory of unpacked file
         """
-        out_dir = path.replace(".tgz", "", 1)
+        out_dir = path.replace('.tgz', '', 1)
         if not os.path.exists(out_dir):
             os.mkdir(out_dir)
 
@@ -325,25 +296,12 @@ class PathResolver(DbgFileResolver):
         filename = self.url_to_filename(url)
         path = os.path.join(self.cache_dir, filename)
         if not os.path.exists(path):
-            print("Downloading the file...")
-            self.get_file_from_service(url, path)
+            subprocess.check_call(['wget', url], cwd=self.cache_dir)
         else:
-            print("File already exists in cache")
+            print('File aready exists in cache')
             exists_locally = True
         return path, exists_locally
 
-    def get_file_from_service(self, url: str, local_path: str) -> None:
-        """
-        Get file from URL and write to a local file.
-
-        :param url: URL string
-        :param local_path: full name for local file
-        """
-        with requests.get(url, stream=True, timeout=self.download_timeout_secs) as response:
-            with open(local_path, "wb") as file:
-                for chunk in response.iter_content(chunk_size=2 * 1024 * 1024):
-                    file.write(chunk)
-
     def get_dbg_file(self, soinfo: dict) -> str or None:
         """
         To get path for given buildId.
@@ -352,26 +310,32 @@ class PathResolver(DbgFileResolver):
         :return: path as string or None (if path not found)
         """
         build_id = soinfo.get("buildId", "").lower()
-        version = soinfo.get("version")
-        binary_name = "mongo"
+        binary_name = 'mongo'
         # search from cached results
         path = self.get_from_cache(build_id)
         if not path:
             # path does not exist in cache, so we send request to server
             try:
-                search_parameters = {"build_id": build_id}
-                if version:
-                    search_parameters["version"] = version
-                print(f"Getting data from service... Search parameters: {search_parameters}")
-                response = self.http_client.get(f"{self.host}/find_by_id", params=search_parameters)
+                response = self.http_client.get(f'{self.host}/find_by_id',
+                                                params={'build_id': build_id})
                 if response.status_code != 200:
+                    # if we could not find the path of binary, that might be system library.
+                    # we can try using frame's own `path` data.
+                    # symbolization can succeed only if that binary exists on local
+                    # machine (more specifically: in the given path).
+                    system_path = soinfo.get('path')
+                    if system_path:
+                        sys.stdout.write(
+                            f"Could not find path of binary from symbolizer web service. Trying to use the "
+                            f"provided path: {system_path}\n")
+                        return system_path
                     sys.stderr.write(
                         f"Server returned unsuccessful status: {response.status_code}, "
                         f"response body: {response.text}\n")
                     return None
                 else:
-                    data = response.json().get("data", {})
-                    path, binary_name = data.get("debug_symbols_url"), data.get("file_name")
+                    data = response.json().get('data', {})
+                    path, binary_name = data.get('debug_symbols_url'), data.get('file_name')
             except Exception as err:  # noqa pylint: disable=broad-except
                 sys.stderr.write(f"Error occurred while trying to get response from server "
                                  f"for buildId({build_id}): {err}\n")
@@ -388,7 +352,7 @@ class PathResolver(DbgFileResolver):
         try:
             dl_path, exists_locally = self.download(path)
             if exists_locally:
-                path = dl_path.replace(".tgz", "", 1)
+                path = dl_path.replace('.tgz', '', 1)
             else:
                 print("Downloaded, now unpacking...")
                 path = self.unpack(dl_path)
@@ -398,8 +362,8 @@ class PathResolver(DbgFileResolver):
         # if file has extension, it is good. if not, we should append .debug, because those without extension are
         # from release builds, and their debug symbol files contain .debug extension.
         # we need to map those 2 different file names ('<name>' becomes '<name>.debug').
-        if not binary_name.endswith(".debug"):
-            binary_name = f"{binary_name}.debug"
+        if not binary_name.endswith('.debug') and not binary_name.endswith('.so'):
+            binary_name = f'{binary_name}.debug'
 
         inner_folder_name = self.path_options.get_binary_folder_name(binary_name)
 
@@ -417,7 +381,6 @@ def parse_input(trace_doc, dbg_path_resolver):
         return {so_entry["b"]: so_entry for so_entry in somap_list if "b" in so_entry}
 
     base_addr_map = make_base_addr_map(trace_doc["processInfo"]["somap"])
-    version = get_version(trace_doc)
 
     frames = []
     for frame in trace_doc["backtrace"]:
@@ -427,8 +390,6 @@ def parse_input(trace_doc, dbg_path_resolver):
             )
             continue
         soinfo = base_addr_map.get(frame["b"], {})
-        if version:
-            soinfo["version"] = version
         elf_type = soinfo.get("elfType", 0)
         if elf_type == 3:
             addr_base = "0"
@@ -438,7 +399,7 @@ def parse_input(trace_doc, dbg_path_resolver):
             addr_base = soinfo.get("vmaddr", "0")
         addr = int(addr_base, 16) + int(frame["o"], 16)
         # addr currently points to the return address which is the one *after* the call. x86 is
-        # variable length so going backwards is difficult. However, llvm-symbolizer seems to do the
+        # variable length so going backwards is difficult. However llvm-symbolizer seems to do the
         # right thing if we just subtract 1 byte here. This has the downside of also adjusting the
         # address of instructions that cause signals (such as segfaults and divide-by-zero) which
         # are already correct, but there doesn't seem to be a reliable way to detect that case.
@@ -450,16 +411,6 @@ def parse_input(trace_doc, dbg_path_resolver):
     return frames
 
 
-def get_version(trace_doc: Dict[str, Any]) -> Optional[str]:
-    """
-    Get version from trace doc.
-
-    :param trace_doc: Traceback dict.
-    :return: Version string or None.
-    """
-    return trace_doc.get("processInfo", {}).get("mongodbVersion")
-
-
 def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, input_format,
                      **kwargs):
     """Return a list of symbolized stack frames from a trace_doc in MongoDB stack dump format."""
@@ -467,23 +418,24 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
     # Keep frames in kwargs to avoid changing the function signature.
     frames = kwargs.get("frames")
     if frames is None:
-        total_seconds_for_retries = kwargs.get("total_seconds_for_retries", 0)
-        frames = preprocess_frames_with_retries(dbg_path_resolver, trace_doc, input_format,
-                                                total_seconds_for_retries)
+        frames = preprocess_frames(dbg_path_resolver, trace_doc, input_format)
 
     if not symbolizer_path:
-        symbolizer_path = os.environ.get(SYMBOLIZER_PATH_ENV)
+        symbolizer_path_env = "MONGOSYMB_SYMBOLIZER_PATH"
+        default_symbolizer_path = "llvm-symbolizer"
+        symbolizer_path = os.environ.get(symbolizer_path_env)
         if not symbolizer_path:
-            print(f"Env value for '{SYMBOLIZER_PATH_ENV}' not found, using"
-                  f" '{DEFAULT_SYMBOLIZER_PATH}' as a default executable path.")
-            symbolizer_path = DEFAULT_SYMBOLIZER_PATH
+            print(
+                f"Env value for '{symbolizer_path_env}' not found, using '{default_symbolizer_path}' "
+                f"as a defualt executable path.")
+            symbolizer_path = default_symbolizer_path
 
     symbolizer_args = [symbolizer_path]
     for dh in dsym_hint:
         symbolizer_args.append("-dsym-hint={}".format(dh))
     symbolizer_process = subprocess.Popen(args=symbolizer_args, close_fds=True,
                                           stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                                          stderr=sys.stdout)
+                                          stderr=open("/dev/null"))
 
     def extract_symbols(stdin):
         """Extract symbol information from the output of llvm-symbolizer.
@@ -515,7 +467,7 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
 
     for frame in frames:
         if frame["path"] is None:
-            print(f"Path not found in frame: {frame}")
+            print("Path not found in frame:", frame)
             continue
         symbol_line = "CODE {path:} {addr:}\n".format(**frame)
         symbolizer_process.stdin.write(symbol_line.encode())
@@ -526,17 +478,8 @@ def symbolize_frames(trace_doc, dbg_path_resolver, symbolizer_path, dsym_hint, i
     return frames
 
 
-def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any],
-                      input_format: str) -> List[Dict[str, Any]]:
-    """
-    Process the paths in frame objects.
-
-    :param dbg_path_resolver: debug symbols file path resolver
-    :param trace_doc: traceback object
-    :param input_format: format of input
-    :return: the list of traceback frames
-    """
-
+def preprocess_frames(dbg_path_resolver, trace_doc, input_format):
+    """Process the paths in frame objects."""
     if input_format == "classic":
         frames = parse_input(trace_doc, dbg_path_resolver)
     elif input_format == "thin":
@@ -545,43 +488,9 @@ def preprocess_frames(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, A
             frame["path"] = dbg_path_resolver.get_dbg_file(frame)
     else:
         raise ValueError('Unknown input format "{}"'.format(input_format))
-
     return frames
 
 
-def has_high_not_found_paths_ratio(frames: List[Dict[str, Any]]) -> bool:
-    """
-    Check whether not found paths in frames ratio is higher than 0.5.
-
-    :param frames: the list of traceback frames
-    :return: True if ratio is higher than 0.5
-    """
-    not_found = [1 for f in frames if f.get("path") is None]
-    not_found_ratio = len(not_found) / (len(frames) or 1)
-    return not_found_ratio >= 0.5
-
-
-def preprocess_frames_with_retries(dbg_path_resolver: DbgFileResolver, trace_doc: Dict[str, Any],
-                                   input_format: str,
-                                   total_seconds_for_retries: int = 0) -> List[Dict[str, Any]]:
-    """
-    Process the paths in frame objects.
-
-    :param dbg_path_resolver: debug symbols file path resolver
-    :param trace_doc: traceback object
-    :param input_format: format of input
-    :param total_seconds_for_retries: max wait time for retries in seconds
-    :return: the list of traceback frames
-    """
-
-    retrying = Retrying(
-        retry=retry_if_result(has_high_not_found_paths_ratio), wait=wait_fixed(60),
-        stop=stop_after_delay(total_seconds_for_retries),
-        retry_error_callback=lambda retry_state: retry_state.outcome.result())
-
-    return retrying(preprocess_frames, dbg_path_resolver, trace_doc, input_format)
-
-
 def classic_output(frames, outfile, **kwargs):  # pylint: disable=unused-argument
     """Provide classic output."""
     for frame in frames:
@@ -607,12 +516,6 @@ def make_argument_parser(parser=None, **kwargs):
     parser.add_argument('--debug-file-resolver', choices=['path', 's3', 'pr'], default='pr')
     parser.add_argument('--src-dir-to-move', action="store", type=str, default=None,
                         help="Specify a src dir to move to /data/mci/{original_buildid}/src")
-    parser.add_argument(
-        '--total-seconds-for-retries', default=0, type=int,
-        help="If web service fails to find path for given build id, it could be because mapping "
-        "process was not finished yet. We can wait for it to finish and retry again. Each retry"
-        " adds 2 minutes to previous wait time. It is guaranteed that total wait time does not exceed this "
-        "specified amount.")
 
     parser.add_argument('--live', action='store_true')
     s3_group = parser.add_argument_group(
@@ -628,8 +531,6 @@ def make_argument_parser(parser=None, **kwargs):
                           help='URL of web service running the API to get debug symbol URL')
     pr_group.add_argument('--pr-cache-dir', default='',
                           help='Full path to a directory to store cache/files')
-    pr_group.add_argument('--client-secret', default='', help='Secret key for Okta Oauth')
-    pr_group.add_argument('--client-id', default='', help='Client id for Okta Oauth')
     # caching mechanism is currently not fully developed and needs more advanced cleaning techniques, we add an option
     # to enable it after completing the implementation
 
@@ -664,17 +565,11 @@ def substitute_stdin(options, resolver):
             if not trace_doc["backtrace"]:
                 print("Trace is empty, skipping...")
                 continue
-            frames = symbolize_frames(
-                trace_doc,
-                resolver,
-                options.symbolizer_path,
-                [],
-                options.output_format,
-            )
+            frames = symbolize_frames(trace_doc, resolver, options.symbolizer_path, [],
+                                      options.output_format)
             print(prefix)
             print("Symbolizing...")
             classic_output(frames, sys.stdout, indent=2)
-            print("Completed, waiting for input...")
         else:
             print(line)
 
@@ -688,8 +583,7 @@ def main(options):
     elif options.debug_file_resolver == 's3':
         resolver = S3BuildidDbgFileResolver(options.s3_cache_dir, options.s3_bucket)
     elif options.debug_file_resolver == 'pr':
-        resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir,
-                                client_secret=options.client_secret, client_id=options.client_id)
+        resolver = PathResolver(host=options.pr_host, cache_dir=options.pr_cache_dir)
 
     if options.live:
         print("Entering live mode")
@@ -703,7 +597,7 @@ def main(options):
 
     if not trace_doc or not trace_doc.strip():
         print("Please provide the backtrace through stdin for symbolization;"
-              " e.g. `your/symbolization/command < /file/with/stacktrace`")
+              "e.g. `your/symbolization/command < /file/with/stacktrace`")
 
     # Search the trace_doc for an object having "backtrace" and "processInfo" keys.
     def bt_search(obj):
@@ -730,8 +624,8 @@ def main(options):
         except json.JSONDecodeError:
             pass
     else:
-        sys.stderr.write("could not find json backtrace object in input\n")
-        sys.exit(1)
+        print("could not find json backtrace object in input", file=sys.stderr)
+        exit(1)
 
     output_fn = None
     if options.output_format == 'json':
@@ -739,8 +633,7 @@ def main(options):
     if options.output_format == 'classic':
         output_fn = classic_output
 
-    frames = preprocess_frames_with_retries(resolver, trace_doc, options.input_format,
-                                            options.total_seconds_for_retries)
+    frames = preprocess_frames(resolver, trace_doc, options.input_format)
 
     if options.src_dir_to_move and resolver.mci_build_dir is not None:
         try: