diff options
Diffstat (limited to 'buildscripts')
| -rw-r--r-- | buildscripts/__init__.py | 8 | ||||
| -rw-r--r-- | buildscripts/bb.py | 2 | ||||
| -rw-r--r-- | buildscripts/bcp.py | 40 | ||||
| -rw-r--r-- | buildscripts/benchmark_tools.py | 62 | ||||
| -rwxr-xr-x | buildscripts/build_and_test_client.py | 75 | ||||
| -rw-r--r-- | buildscripts/buildlogger.py | 480 | ||||
| -rw-r--r-- | buildscripts/cleanbb.py | 15 | ||||
| -rw-r--r-- | buildscripts/confluence_export.py | 103 | ||||
| -rw-r--r-- | buildscripts/distmirror.py | 139 | ||||
| -rw-r--r-- | buildscripts/emr/FileLock.java | 107 | ||||
| -rw-r--r-- | buildscripts/emr/IOUtil.java | 156 | ||||
| -rw-r--r-- | buildscripts/emr/MANIFEST.MF | 2 | ||||
| -rw-r--r-- | buildscripts/emr/emr.java | 380 | ||||
| -rw-r--r-- | buildscripts/emr/emr.py | 385 | ||||
| -rw-r--r-- | buildscripts/emr/emrnodesetup.sh | 7 | ||||
| -rwxr-xr-x | buildscripts/errorcodes.py | 76 | ||||
| -rw-r--r-- | buildscripts/hacks_mandriva.py | 9 | ||||
| -rw-r--r-- | buildscripts/hacks_ubuntu.py | 54 | ||||
| -rwxr-xr-x | buildscripts/make_archive.py | 116 | ||||
| -rw-r--r-- | buildscripts/moduleconfig.py | 131 | ||||
| -rw-r--r-- | buildscripts/packager.py | 37 | ||||
| -rw-r--r-- | buildscripts/s3md5.py | 4 | ||||
| -rwxr-xr-x | buildscripts/smoke.py | 512 | ||||
| -rw-r--r-- | buildscripts/utils.py | 83 |
24 files changed, 2468 insertions, 515 deletions
diff --git a/buildscripts/__init__.py b/buildscripts/__init__.py index 839da9dd919..442c233b031 100644 --- a/buildscripts/__init__.py +++ b/buildscripts/__init__.py @@ -1,13 +1,5 @@ -import hacks_mandriva -import hacks_ubuntu import os; def findHacks( un ): - if un[0] == 'Linux' and (os.path.exists("/etc/debian_version") or - un[3].find("Ubuntu") >= 0): - return hacks_ubuntu - if un[0] == 'Linux' and (os.path.exists("/etc/mandriva-release") or - un[3].find("mnb") >= 0): - return hacks_mandriva return None diff --git a/buildscripts/bb.py b/buildscripts/bb.py index 1e878283fed..d2f3819b1de 100644 --- a/buildscripts/bb.py +++ b/buildscripts/bb.py @@ -13,7 +13,7 @@ def checkOk(): m = "v" + m[0] print( m ) - print( "excpted version [" + m + "]" ) + print( "expected version [" + m + "]" ) from subprocess import Popen, PIPE diff = Popen( [ "git", "diff", "origin/v1.2" ], stdout=PIPE ).communicate()[ 0 ] diff --git a/buildscripts/bcp.py b/buildscripts/bcp.py new file mode 100644 index 00000000000..dda20924e79 --- /dev/null +++ b/buildscripts/bcp.py @@ -0,0 +1,40 @@ + +import utils +import os +import shutil +import sys + +def go( boost_root ): + + OUTPUT = "src/third_party/boost" + if os.path.exists( OUTPUT ): + shutil.rmtree( OUTPUT ) + + cmd = [ "bcp" , "--scan" , "--boost=%s" % boost_root ] + + src = utils.getAllSourceFiles() + + cmd += src + cmd.append( OUTPUT ) + + if not os.path.exists( OUTPUT ): + os.makedirs( OUTPUT ) + + res = utils.execsys( cmd ) + + out = open( OUTPUT + "/bcp-out.txt" , 'w' ) + out.write( res[0] ) + out.close() + + out = open( OUTPUT + "/notes.txt" , 'w' ) + out.write( "command: " + " ".join( cmd ) ) + out.close() + + print( res[1] ) + +if __name__ == "__main__": + if len(sys.argv) == 1: + print( "usage: python %s <boost root directory>" % sys.argv[0] ) + sys.exit(1) + go( sys.argv[1] ) + diff --git a/buildscripts/benchmark_tools.py b/buildscripts/benchmark_tools.py deleted file mode 100644 index 54a86b019d5..00000000000 --- a/buildscripts/benchmark_tools.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import urllib -import urllib2 -import sys - -try: - import json -except: - import simplejson as json # need simplejson for python < 2.6 - -sys.path.append( "." ) -sys.path.append( ".." ) -sys.path.append( "../../" ) -sys.path.append( "../../../" ) - - -import settings - -def machine_info(extra_info=""): - """Get a dict representing the "machine" section of a benchmark result. - - ie: - { - "os_name": "OS X", - "os_version": "10.5", - "processor": "2.4 GHz Intel Core 2 Duo", - "memory": "3 GB 667 MHz DDR2 SDRAM", - "extra_info": "Python 2.6" - } - - Must have a settings.py file on sys.path that defines "processor" and "memory" - variables. - """ - machine = {} - (machine["os_name"], _, machine["os_version"], _, _) = os.uname() - machine["processor"] = settings.processor - machine["memory"] = settings.memory - machine["extra_info"] = extra_info - return machine - -def post_data(data, machine_extra_info="", post_url="http://mongo-db.appspot.com/benchmark"): - """Post a benchmark data point. - - data should be a Python dict that looks like: - { - "benchmark": { - "project": "http://github.com/mongodb/mongo-python-driver", - "name": "insert test", - "description": "test inserting 10000 documents with the C extension enabled", - "tags": ["insert", "python"] - }, - "trial": { - "server_hash": "4f5a8d52f47507a70b6c625dfb5dbfc87ba5656a", - "client_hash": "8bf2ad3d397cbde745fd92ad41c5b13976fac2b5", - "result": 67.5, - "extra_info": "some logs or something" - } - } - """ - data["machine"] = machine_info(machine_extra_info) - urllib2.urlopen(post_url, urllib.urlencode({"payload": json.dumps(data)})) - return data diff --git a/buildscripts/build_and_test_client.py b/buildscripts/build_and_test_client.py new file mode 100755 index 00000000000..1b97f623cc5 --- /dev/null +++ b/buildscripts/build_and_test_client.py @@ -0,0 +1,75 @@ +#!/usr/bin/python + +'''Script to attempt an isolated build of the C++ driver and its examples. + +Working directory must be the repository root. + +Usage: + +./buildscripts/build_and_test_client.py <mongo client archive file> [optional scons arguments] + +The client is built in a temporary directory, and the sample programs are run against a mongod +instance found in the current working directory. The temporary directory and its contents are +destroyed at the end of execution. +''' + +import os +import shutil +import subprocess +import sys +import tempfile +import tarfile +import zipfile + +import utils + +def main(args): + archive_file = args[1] + scons_args = args[2:] + build_and_test(archive_file, scons_args) + +def build_and_test(archive_name, scons_args): + work_dir = tempfile.mkdtemp() + try: + archive = open_archive(archive_name) + extracted_root = extract_archive(work_dir, archive) + run_scons(extracted_root, scons_args) + smoke_client(extracted_root) + finally: + shutil.rmtree(work_dir) + +def open_tar(archive_name): + return tarfile.open(archive_name, 'r') + +def open_zip(archive_name): + class ZipWrapper(zipfile.ZipFile): + def getnames(self): + return self.namelist() + return ZipWrapper(archive_name, 'r') + +def open_archive(archive_name): + try: + return open_tar(archive_name) + except: + return open_zip(archive_name) + +def extract_archive(work_dir, archive_file): + archive_file.extractall(path=work_dir) + return os.path.join( + work_dir, + os.path.dirname([n for n in archive_file.getnames() if n.endswith('SConstruct')][0]) + ) + +def run_scons(extracted_root, scons_args): + rc = subprocess.call(['scons', '-C', extracted_root, ] + scons_args + ['clientTests']) + if rc is not 0: + sys.exit(rc) + +def smoke_client(extracted_root): + rc = subprocess.call(utils.smoke_command("--test-path", extracted_root, "client")) + if rc is not 0: + sys.exit(rc) + +if __name__ == '__main__': + main(sys.argv) + sys.exit(0) diff --git a/buildscripts/buildlogger.py b/buildscripts/buildlogger.py new file mode 100644 index 00000000000..9f3feacbe55 --- /dev/null +++ b/buildscripts/buildlogger.py @@ -0,0 +1,480 @@ +""" +buildlogger.py + +Wrap a command (specified on the command line invocation of buildlogger.py) +and send output in batches to the buildlogs web application via HTTP POST. + +The script configures itself from environment variables: + + required env vars: + MONGO_BUILDER_NAME (e.g. "Nightly Linux 64-bit") + MONGO_BUILD_NUMBER (an integer) + MONGO_TEST_FILENAME (not required when invoked with -g) + + optional env vars: + MONGO_PHASE (e.g. "core", "slow nightly", etc) + MONGO_* (any other environment vars are passed to the web app) + BUILDLOGGER_CREDENTIALS (see below) + +This script has two modes: a "test" mode, intended to wrap the invocation of +an individual test file, and a "global" mode, intended to wrap the mongod +instances that run throughout the duration of a mongo test phase (the logs +from "global" invocations are displayed interspersed with the logs of each +test, in order to let the buildlogs web app display the full output sensibly.) + +If the BUILDLOGGER_CREDENTIALS environment variable is set, it should be a +path to a valid Python file containing "username" and "password" variables, +which should be valid credentials for authenticating to the buildlogger web +app. For example: + + username = "hello" + password = "world" + +If BUILDLOGGER_CREDENTIALS is a relative path, then the working directory +and the directories one, two, and three levels up, are searched, in that +order. +""" + +import functools +import os +import os.path +import re +import signal +import socket +import subprocess +import sys +import time +import traceback +import urllib2 +import utils + +# suppress deprecation warnings that happen when +# we import the 'buildbot.tac' file below +import warnings +warnings.simplefilter('ignore', DeprecationWarning) + +try: + import json +except: + try: + import simplejson as json + except: + json = None + +# try to load the shared secret from settings.py +# which will be one, two, or three directories up +# from this file's location +credentials_file = os.environ.get('BUILDLOGGER_CREDENTIALS', 'buildbot.tac') +credentials_loc, credentials_name = os.path.split(credentials_file) +if not credentials_loc: + here = os.path.abspath(os.path.dirname(__file__)) + possible_paths = [ + os.path.abspath(os.path.join(here, '..')), + os.path.abspath(os.path.join(here, '..', '..')), + os.path.abspath(os.path.join(here, '..', '..', '..')), + ] +else: + possible_paths = [credentials_loc] + +username, password = None, None +for path in possible_paths: + credentials_path = os.path.join(path, credentials_name) + if os.path.isfile(credentials_path): + credentials = {} + try: + execfile(credentials_path, credentials, credentials) + username = credentials.get('slavename', credentials.get('username')) + password = credentials.get('passwd', credentials.get('password')) + break + except: + pass + + +URL_ROOT = 'http://buildlogs.mongodb.org/' +TIMEOUT_SECONDS = 10 +socket.setdefaulttimeout(TIMEOUT_SECONDS) + +digest_handler = urllib2.HTTPDigestAuthHandler() +digest_handler.add_password( + realm='buildlogs', + uri=URL_ROOT, + user=username, + passwd=password) + +# This version of HTTPErrorProcessor is copied from +# Python 2.7, and allows REST response codes (e.g. +# "201 Created") which are treated as errors by +# older versions. +class HTTPErrorProcessor(urllib2.HTTPErrorProcessor): + def http_response(self, request, response): + code, msg, hdrs = response.code, response.msg, response.info() + + # According to RFC 2616, "2xx" code indicates that the client's + # request was successfully received, understood, and accepted. + if not (200 <= code < 300): + response = self.parent.error( + 'http', request, response, code, msg, hdrs) + + return response + +url_opener = urllib2.build_opener(digest_handler, HTTPErrorProcessor()) + +def url(endpoint): + if not endpoint.endswith('/'): + endpoint = '%s/' % endpoint + + return '%s/%s' % (URL_ROOT.rstrip('/'), endpoint) + +def post(endpoint, data, headers=None): + data = json.dumps(data, encoding='utf-8') + + headers = headers or {} + headers.update({'Content-Type': 'application/json; charset=utf-8'}) + + req = urllib2.Request(url=url(endpoint), data=data, headers=headers) + try: + response = url_opener.open(req) + except urllib2.URLError: + import traceback + traceback.print_exc(file=sys.stderr) + sys.stderr.flush() + # indicate that the request did not succeed + return None + + response_headers = dict(response.info()) + + # eg "Content-Type: application/json; charset=utf-8" + content_type = response_headers.get('content-type') + match = re.match(r'(?P<mimetype>[^;]+).*(?:charset=(?P<charset>[^ ]+))?$', content_type) + if match and match.group('mimetype') == 'application/json': + encoding = match.group('charset') or 'utf-8' + return json.load(response, encoding=encoding) + + return response.read() + +def traceback_to_stderr(func): + """ + decorator which logs any exceptions encountered to stderr + and returns none. + """ + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except urllib2.HTTPError, err: + sys.stderr.write('error: HTTP code %d\n----\n' % err.code) + if hasattr(err, 'hdrs'): + for k, v in err.hdrs.items(): + sys.stderr.write("%s: %s\n" % (k, v)) + sys.stderr.write('\n') + sys.stderr.write(err.read()) + sys.stderr.write('\n----\n') + sys.stderr.flush() + except: + sys.stderr.write('Traceback from buildlogger:\n') + traceback.print_exc(file=sys.stderr) + sys.stderr.flush() + return None + return wrapper + + +@traceback_to_stderr +def get_or_create_build(builder, buildnum, extra={}): + data = {'builder': builder, 'buildnum': buildnum} + data.update(extra) + response = post('build', data) + if response is None: + return None + return response['id'] + +@traceback_to_stderr +def create_test(build_id, test_filename, test_command, test_phase): + response = post('build/%s/test' % build_id, { + 'test_filename': test_filename, + 'command': test_command, + 'phase': test_phase, + }) + if response is None: + return None + return response['id'] + +@traceback_to_stderr +def append_test_logs(build_id, test_id, log_lines): + response = post('build/%s/test/%s' % (build_id, test_id), data=log_lines) + if response is None: + return False + return True + +@traceback_to_stderr +def append_global_logs(build_id, log_lines): + """ + "global" logs are for the mongod(s) started by smoke.py + that last the duration of a test phase -- since there + may be output in here that is important but spans individual + tests, the buildlogs webapp handles these logs specially. + """ + response = post('build/%s' % build_id, data=log_lines) + if response is None: + return False + return True + +@traceback_to_stderr +def finish_test(build_id, test_id, failed=False): + response = post('build/%s/test/%s' % (build_id, test_id), data=[], headers={ + 'X-Sendlogs-Test-Done': 'true', + 'X-Sendlogs-Test-Failed': failed and 'true' or 'false', + }) + if response is None: + return False + return True + +def run_and_echo(command): + """ + this just calls the command, and returns its return code, + allowing stdout and stderr to work as normal. it is used + as a fallback when environment variables or python + dependencies cannot be configured, or when the logging + webapp is unavailable, etc + """ + proc = subprocess.Popen(command) + + def handle_sigterm(signum, frame): + try: + proc.send_signal(signum) + except AttributeError: + os.kill(proc.pid, signum) + orig_handler = signal.signal(signal.SIGTERM, handle_sigterm) + + proc.wait() + + signal.signal(signal.SIGTERM, orig_handler) + return proc.returncode + +class LogAppender(object): + def __init__(self, callback, args, send_after_lines=200, send_after_seconds=2): + self.callback = callback + self.callback_args = args + + self.send_after_lines = send_after_lines + self.send_after_seconds = send_after_seconds + + self.buf = [] + self.retrybuf = [] + self.last_sent = time.time() + + def __call__(self, line): + self.buf.append((time.time(), line)) + + delay = time.time() - self.last_sent + if len(self.buf) >= self.send_after_lines or delay >= self.send_after_seconds: + self.submit() + + # no return value is expected + + def submit(self): + if len(self.buf) + len(self.retrybuf) == 0: + return True + + args = list(self.callback_args) + args.append(list(self.buf) + self.retrybuf) + + self.last_sent = time.time() + + if self.callback(*args): + self.buf = [] + self.retrybuf = [] + return True + else: + self.retrybuf += self.buf + self.buf = [] + return False + + +def wrap_test(command): + """ + call the given command, intercept its stdout and stderr, + and send results in batches of 100 lines or 10s to the + buildlogger webapp + """ + + # get builder name and build number from environment + builder = os.environ.get('MONGO_BUILDER_NAME') + buildnum = os.environ.get('MONGO_BUILD_NUMBER') + + if builder is None or buildnum is None: + return run_and_echo(command) + + try: + buildnum = int(buildnum) + except ValueError: + sys.stderr.write('buildlogger: build number ("%s") was not an int\n' % buildnum) + sys.stderr.flush() + return run_and_echo(command) + + # test takes some extra info + phase = os.environ.get('MONGO_PHASE', 'unknown') + test_filename = os.environ.get('MONGO_TEST_FILENAME', 'unknown') + + build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_')) + build_info.pop('MONGO_BUILDER_NAME', None) + build_info.pop('MONGO_BUILD_NUMBER', None) + build_info.pop('MONGO_PHASE', None) + build_info.pop('MONGO_TEST_FILENAME', None) + + build_id = get_or_create_build(builder, buildnum, extra=build_info) + if not build_id: + return run_and_echo(command) + + test_id = create_test(build_id, test_filename, ' '.join(command), phase) + if not test_id: + return run_and_echo(command) + + # the peculiar formatting here matches what is printed by + # smoke.py when starting tests + output_url = '%s/build/%s/test/%s/' % (URL_ROOT.rstrip('/'), build_id, test_id) + sys.stdout.write(' (output suppressed; see %s)\n' % output_url) + sys.stdout.flush() + + callback = LogAppender(callback=append_test_logs, args=(build_id, test_id)) + returncode = loop_and_callback(command, callback) + failed = bool(returncode != 0) + + # this will append any remaining unsubmitted logs, or + # return True if there are none left to submit + tries = 5 + while not callback.submit() and tries > 0: + sys.stderr.write('failed to finish sending test logs, retrying in 1s\n') + sys.stderr.flush() + time.sleep(1) + tries -= 1 + + tries = 5 + while not finish_test(build_id, test_id, failed) and tries > 5: + sys.stderr.write('failed to mark test finished, retrying in 1s\n') + sys.stderr.flush() + time.sleep(1) + tries -= 1 + + return returncode + +def wrap_global(command): + """ + call the given command, intercept its stdout and stderr, + and send results in batches of 100 lines or 10s to the + buildlogger webapp. see :func:`append_global_logs` for the + difference between "global" and "test" log output. + """ + + # get builder name and build number from environment + builder = os.environ.get('MONGO_BUILDER_NAME') + buildnum = os.environ.get('MONGO_BUILD_NUMBER') + + if builder is None or buildnum is None: + return run_and_echo(command) + + try: + buildnum = int(buildnum) + except ValueError: + sys.stderr.write('int(os.environ["MONGO_BUILD_NUMBER"]):\n') + sys.stderr.write(traceback.format_exc()) + sys.stderr.flush() + return run_and_echo(command) + + build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_')) + build_info.pop('MONGO_BUILDER_NAME', None) + build_info.pop('MONGO_BUILD_NUMBER', None) + + build_id = get_or_create_build(builder, buildnum, extra=build_info) + if not build_id: + return run_and_echo(command) + + callback = LogAppender(callback=append_global_logs, args=(build_id, )) + returncode = loop_and_callback(command, callback) + + # this will append any remaining unsubmitted logs, or + # return True if there are none left to submit + tries = 5 + while not callback.submit() and tries > 0: + sys.stderr.write('failed to finish sending global logs, retrying in 1s\n') + sys.stderr.flush() + time.sleep(1) + tries -= 1 + + return returncode + +def loop_and_callback(command, callback): + """ + run the given command (a sequence of arguments, ordinarily + from sys.argv), and call the given callback with each line + of stdout or stderr encountered. after the command is finished, + callback is called once more with None instead of a string. + """ + proc = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + def handle_sigterm(signum, frame): + try: + proc.send_signal(signum) + except AttributeError: + os.kill(proc.pid, signum) + + # register a handler to delegate SIGTERM + # to the child process + orig_handler = signal.signal(signal.SIGTERM, handle_sigterm) + + while proc.poll() is None: + try: + line = proc.stdout.readline().strip('\r\n') + line = utils.unicode_dammit(line) + callback(line) + except IOError: + # if the signal handler is called while + # we're waiting for readline() to return, + # don't show a traceback + break + + # There may be additional buffered output + for line in proc.stdout.readlines(): + callback(line.strip('\r\n')) + + # restore the original signal handler, if any + signal.signal(signal.SIGTERM, orig_handler) + return proc.returncode + + +if __name__ == '__main__': + # argv[0] is 'buildlogger.py' + del sys.argv[0] + + if sys.argv[0] in ('-g', '--global'): + # then this is wrapping a "global" command, and should + # submit global logs to the build, not test logs to a + # test within the build + del sys.argv[0] + wrapper = wrap_global + + else: + wrapper = wrap_test + + # if we are missing credentials or the json module, then + # we can't use buildlogger; so just echo output, but also + # log why we can't work. + if json is None: + sys.stderr.write('buildlogger: could not import a json module\n') + sys.stderr.flush() + wrapper = run_and_echo + + elif username is None or password is None: + sys.stderr.write('buildlogger: could not find or import %s for authentication\n' % credentials_file) + sys.stderr.flush() + wrapper = run_and_echo + + # otherwise wrap a test command as normal; the + # wrapper functions return the return code of + # the wrapped command, so that should be our + # exit code as well. + sys.exit(wrapper(sys.argv)) + diff --git a/buildscripts/cleanbb.py b/buildscripts/cleanbb.py index bfeafd2d8e9..085eee223fb 100644 --- a/buildscripts/cleanbb.py +++ b/buildscripts/cleanbb.py @@ -14,6 +14,16 @@ if os.path.basename(cwd) == 'buildscripts': print( "cwd [" + cwd + "]" ) def shouldKill( c ): + + if "smoke.py" in c: + return False + + if "emr.py" in c: + return False + + if "java" in c: + return False + if c.find( cwd ) >= 0: return True @@ -25,7 +35,10 @@ def shouldKill( c ): def killprocs( signal="" ): killed = 0 - + + if sys.platform == 'win32': + return killed + l = utils.getprocesslist() print( "num procs:" + str( len( l ) ) ) if len(l) == 0: diff --git a/buildscripts/confluence_export.py b/buildscripts/confluence_export.py deleted file mode 100644 index 29cdde60c7f..00000000000 --- a/buildscripts/confluence_export.py +++ /dev/null @@ -1,103 +0,0 @@ -#! /usr/bin/env python - -# Export the contents on confluence -# -# Dependencies: -# - suds -# -# User: soap, Password: soap -from __future__ import with_statement -import cookielib -import datetime -import os -import shutil -import subprocess -import sys -import urllib2 -sys.path[0:0] = [""] - -import simples3 -from suds.client import Client - -import settings - -HTML_URI = "http://mongodb.onconfluence.com/rpc/soap-axis/confluenceservice-v1?wsdl" -PDF_URI = "http://www.mongodb.org/rpc/soap-axis/pdfexport?wsdl" -USERNAME = "soap" -PASSWORD = "soap" -AUTH_URI = "http://www.mongodb.org/login.action?os_authType=basic" -TMP_DIR = "confluence-tmp" -TMP_FILE = "confluence-tmp.zip" - - -def export_html_and_get_uri(): - client = Client(HTML_URI) - auth = client.service.login(USERNAME, PASSWORD) - return client.service.exportSpace(auth, "DOCS", "TYPE_HTML") - - -def export_pdf_and_get_uri(): - client = Client(PDF_URI) - auth = client.service.login(USERNAME, PASSWORD) - return client.service.exportSpace(auth, "DOCS") - - -def login_and_download(docs): - cookie_jar = cookielib.CookieJar() - cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar) - password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() - password_manager.add_password(None, AUTH_URI, USERNAME, PASSWORD) - auth_handler = urllib2.HTTPBasicAuthHandler(password_manager) - urllib2.build_opener(cookie_handler, auth_handler).open(AUTH_URI) - return urllib2.build_opener(cookie_handler).open(docs) - - -def extract_to_dir(data, dir): - with open(TMP_FILE, "w") as f: - f.write(data.read()) - data.close() - # This is all really annoying but zipfile doesn't do extraction on 2.5 - subprocess.call(["unzip", "-d", dir, TMP_FILE]) - os.unlink(TMP_FILE) - - -def rmdir(dir): - try: - shutil.rmtree(dir) - except: - pass - - -def overwrite(src, dest): - target = "%s/DOCS-%s/" % (dest, datetime.date.today()) - current = "%s/current" % dest - rmdir(target) - shutil.copytree(src, target) - try: - os.unlink(current) - except: - pass - os.symlink(os.path.abspath(target), os.path.abspath(current)) - - -def write_to_s3(pdf): - s3 = simples3.S3Bucket(settings.bucket, settings.id, settings.key) - name = "docs/mongodb-docs-%s.pdf" % datetime.date.today() - s3.put(name, pdf, acl="public-read") - - -def main(dir): - # HTML - rmdir(TMP_DIR) - extract_to_dir(login_and_download(export_html_and_get_uri()), TMP_DIR) - overwrite("%s/DOCS/" % TMP_DIR, dir) - - # PDF - write_to_s3(login_and_download(export_pdf_and_get_uri()).read()) - - -if __name__ == "__main__": - try: - main(sys.argv[1]) - except IndexError: - print "pass outdir as first arg" diff --git a/buildscripts/distmirror.py b/buildscripts/distmirror.py deleted file mode 100644 index 7af1a89f7dc..00000000000 --- a/buildscripts/distmirror.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python - -# Download mongodb stuff (at present builds, sources, docs, but not -# drivers). - -# Usage: <progname> [directory] # directory defaults to cwd. - -# FIXME: this script is fairly sloppy. -import sys -import os -import urllib2 -import time -import hashlib -import warnings - -written_files = [] -def get(url, filename): - # A little safety check. - if filename in written_files: - raise Exception('not overwriting file %s (already written in this session)' % filename) - else: - written_files.append(filename) - print "downloading %s to %s" % (url, filename) - open(filename, 'w').write(urllib2.urlopen(url).read()) - - -def checkmd5(md5str, filename): - m = hashlib.md5() - m.update(open(filename, 'rb').read()) - d = m.hexdigest() - if d != md5str: - warnings.warn("md5sum mismatch for file %s: wanted %s; got %s" % (filename, md5str, d)) - -osarches=(("osx", ("i386", "i386-tiger", "x86_64"), ("tgz", )), - ("linux", ("i686", "x86_64"), ("tgz", )), - ("win32", ("i386", "x86_64"), ("zip", )), - ("sunos5", ("i86pc", "x86_64"), ("tgz", )), - ("src", ("src", ), ("tar.gz", "zip")), ) - -# KLUDGE: this will need constant editing. -versions = ("1.4.2", "1.5.1", "latest") - -url_format = "http://downloads.mongodb.org/%s/mongodb-%s-%s.%s" -filename_format = "mongodb-%s-%s.%s" - -def core_server(): - for version in versions: - for (os, architectures, archives) in osarches: - for architecture in architectures: - for archive in archives: - osarch = os + '-' + architecture if architecture != 'src' else 'src' - # ugh. - if architecture == 'src' and version == 'latest': - if archive == 'tar.gz': - archive2 = 'tarball' - elif archive == 'zip': - archive2 == 'zipball' - url = "http://github.com/mongodb/mongo/"+archive2+"/master" - version2 = "master" - else: - version2 = version if architecture != 'src' else 'r'+version - url = url_format % (os, osarch, version2, archive) - # ugh ugh - md5url = url+'.md5' if architecture != 'src' else None - filename = filename_format % (osarch, version2, archive) - get(url, filename) - if md5url: - print "fetching md5 url " + md5url - md5str = urllib2.urlopen(md5url).read() - checkmd5(md5str, filename) - -def drivers(): - # Drivers... FIXME: drivers. - driver_url_format = "http://github.com/mongodb/mongo-%s-driver/%s/%s" - driver_filename_format = "mongo-%s-driver-%s.%s" - drivers=(("python", ("1.6", "master"), ("zipball", "tarball"), None), - ("ruby", ("0.20", "master"), ("zipball", "tarball"), None), - ("c", ("v0.1", "master"), ("zipball", "tarball"), None), - # FIXME: PHP, Java, and Csharp also have zips and jars of - # precompiled relesaes. - ("php", ("1.0.6", "master"), ("zipball", "tarball"), None), - ("java", ("r1.4", "r2.0rc1", "master"), ("zipball", "tarball"), None), - # And Csharp is in a different github place, too. - ("csharp", ("0.82.2", "master"), ("zipball", "tarball"), - "http://github.com/samus/mongodb-%s/%s/%s"), - ) - - for (lang, releases, archives, url_format) in drivers: - for release in releases: - for archive in archives: - url = (url_format if url_format else driver_url_format) % (lang, archive, release) - if archive == 'zipball': - extension = 'zip' - elif archive == 'tarball': - extension = 'tgz' - else: - raise Exception('unknown archive format %s' % archive) - filename = driver_filename_format % (lang, release, extension) - get(url, filename) - # ugh ugh ugh - if lang == 'csharp' and release != 'master': - url = 'http://github.com/downloads/samus/mongodb-csharp/MongoDBDriver-Release-%.zip' % (release) - filename = 'MongoDBDriver-Release-%.zip' % (release) - get(url, filename) - if lang == 'java' and release != 'master': - get('http://github.com/downloads/mongodb/mongo-java-driver/mongo-%s.jar' % (release), 'mongo-%s.jar' % (release)) - # I have no idea what's going on with the PHP zipfiles. - if lang == 'php' and release == '1.0.6': - get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.2-osx.zip', 'mongo-1.0.6-php5.2-osx.zip') - get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.3-osx.zip', 'mongo-1.0.6-php5.3-osx.zip') - -def docs(): - # FIXME: in principle, the doc PDFs could be out of date. - docs_url = time.strftime("http://downloads.mongodb.org/docs/mongodb-docs-%Y-%m-%d.pdf") - docs_filename = time.strftime("mongodb-docs-%Y-%m-%d.pdf") - get(docs_url, docs_filename) - -def extras(): - # Extras - extras = ("http://media.mongodb.org/zips.json", ) - for extra in extras: - if extra.rfind('/') > -1: - filename = extra[extra.rfind('/')+1:] - else: - raise Exception('URL %s lacks a slash?' % extra) - get(extra, filename) - -if len(sys.argv) > 1: - dir=sys.argv[1] - os.makedirs(dir) - os.chdir(dir) - -print """NOTE: the md5sums for all the -latest tarballs are out of -date. You will probably see warnings as this script runs. (If you -don't, feel free to delete this note.)""" -core_server() -drivers() -docs() -extras() diff --git a/buildscripts/emr/FileLock.java b/buildscripts/emr/FileLock.java new file mode 100644 index 00000000000..52ed4c083b4 --- /dev/null +++ b/buildscripts/emr/FileLock.java @@ -0,0 +1,107 @@ +// FileLock.java + +import java.io.*; +import java.util.*; +import java.util.concurrent.*; + +/** + * "locks" a resource by using the file system as storage + * file has 1 line + * <incarnation> <last ping time in millis> + */ +public class FileLock { + + public FileLock( String logicalName ) + throws IOException { + + _file = new File( "/tmp/java-fileLock-" + logicalName ); + _incarnation = "xxx" + Math.random() + "yyy"; + + if ( ! _file.exists() ) { + FileOutputStream fout = new FileOutputStream( _file ); + fout.write( "\n".getBytes() ); + fout.close(); + } + + } + + /** + * takes lock + * if someone else has it, blocks until the other one finishes + */ + public void lock() + throws IOException { + if ( _lock != null ) + throw new IllegalStateException( "can't lock when you're locked" ); + + try { + _semaphore.acquire(); + } + catch ( InterruptedException ie ) { + throw new RuntimeException( "sad" , ie ); + } + + _raf = new RandomAccessFile( _file , "rw" ); + _lock = _raf.getChannel().lock(); + } + + public void unlock() + throws IOException { + + if ( _lock == null ) + throw new IllegalStateException( "can't unlock when you're not locked" ); + + _lock.release(); + _semaphore.release(); + + _locked = false; + } + + final File _file; + final String _incarnation; + + private RandomAccessFile _raf; + private java.nio.channels.FileLock _lock; + + private boolean _locked; + + private static Semaphore _semaphore = new Semaphore(1); + + + public static void main( final String[] args ) + throws Exception { + + List<Thread> threads = new ArrayList<Thread>(); + + for ( int i=0; i<3; i++ ) { + + threads.add( new Thread() { + public void run() { + try { + FileLock lock = new FileLock( args[0] ); + + long start = System.currentTimeMillis(); + + lock.lock(); + System.out.println( "time to lock:\t" + (System.currentTimeMillis()-start) ); + Thread.sleep( Integer.parseInt( args[1] ) ); + lock.unlock(); + System.out.println( "total time:\t" + (System.currentTimeMillis()-start) ); + } + catch ( Exception e ) { + e.printStackTrace(); + } + } + } ); + } + + for ( Thread t : threads ) { + t.start(); + } + + for ( Thread t : threads ) { + t.join(); + } + + } +} diff --git a/buildscripts/emr/IOUtil.java b/buildscripts/emr/IOUtil.java new file mode 100644 index 00000000000..8ee105c5155 --- /dev/null +++ b/buildscripts/emr/IOUtil.java @@ -0,0 +1,156 @@ +// IOUtil.java + +import java.io.*; +import java.net.*; +import java.util.*; + +public class IOUtil { + + public static String urlFileName( String url ) { + int idx = url.lastIndexOf( "/" ); + if ( idx < 0 ) + return url; + return url.substring( idx + 1 ); + } + + public static long pipe( InputStream in , OutputStream out ) + throws IOException { + + long bytes = 0; + + byte[] buf = new byte[2048]; + + while ( true ) { + int x = in.read( buf ); + if ( x < 0 ) + break; + + bytes += x; + out.write( buf , 0 , x ); + } + + return bytes; + } + + public static class PipingThread extends Thread { + public PipingThread( InputStream in , OutputStream out ) { + _in = in; + _out = out; + + _wrote = 0; + } + + public void run() { + try { + _wrote = pipe( _in , _out ); + } + catch ( IOException ioe ) { + ioe.printStackTrace(); + _wrote = -1; + } + } + + public long wrote() { + return _wrote; + } + + long _wrote; + + final InputStream _in; + final OutputStream _out; + } + + public static String readStringFully( InputStream in ) + throws IOException { + + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + pipe( in , bout ); + return new String( bout.toByteArray() , "UTF8" ); + + } + + public static Map<String,Object> readPythonSettings( File file ) + throws IOException { + + String all = readStringFully( new FileInputStream( file ) ); + + Map<String,Object> map = new TreeMap<String,Object>(); + + for ( String line : all.split( "\n" ) ) { + line = line.trim(); + if ( line.length() == 0 ) + continue; + + String[] pcs = line.split( "=" ); + if ( pcs.length != 2 ) + continue; + + String name = pcs[0].trim(); + String value = pcs[1].trim(); + + if ( value.startsWith( "\"" ) ) { + map.put( name , value.substring( 1 , value.length() - 1 ) ); + } + else { + map.put( name , Long.parseLong( value ) ); + } + + } + + return map; + } + + public static String[] runCommand( String cmd , File dir ) + throws IOException { + + Process p = Runtime.getRuntime().exec( cmd.split( " +" ) , new String[]{} , dir ); + String[] results = new String[]{ IOUtil.readStringFully( p.getInputStream() ) , IOUtil.readStringFully( p.getErrorStream() ) }; + try { + if ( p.waitFor() != 0 ) + throw new RuntimeException( "command failed [" + cmd + "]\n" + results[0] + "\n" + results[1] ); + } + catch ( InterruptedException ie ) { + throw new RuntimeException( "uh oh" ); + } + return results; + } + + + public static void download( String http , File localDir ) + throws IOException { + + File f = localDir; + f.mkdirs(); + + f = new File( f.toString() + File.separator + urlFileName( http ) ); + + System.out.println( "downloading\n\t" + http + "\n\t" + f ); + + if ( f.exists() ) { + System.out.println( "\t already exists" ); + return; + } + + URL url = new URL( http ); + + InputStream in = url.openConnection().getInputStream(); + OutputStream out = new FileOutputStream( f ); + + pipe( in , out ); + + out.close(); + in.close(); + + } + + public static void main( String[] args ) + throws Exception { + + + byte[] data = new byte[]{ 'e' , 'r' , 'h' , 0 }; + System.out.write( data ); + System.out.println( "yo" ); + + } + +} diff --git a/buildscripts/emr/MANIFEST.MF b/buildscripts/emr/MANIFEST.MF new file mode 100644 index 00000000000..4a5b3f96691 --- /dev/null +++ b/buildscripts/emr/MANIFEST.MF @@ -0,0 +1,2 @@ +Manifest-Version: 1.0 +Main-Class: emr diff --git a/buildscripts/emr/emr.java b/buildscripts/emr/emr.java new file mode 100644 index 00000000000..0f01d6d0e2d --- /dev/null +++ b/buildscripts/emr/emr.java @@ -0,0 +1,380 @@ +// emr.java + +import java.io.*; +import java.util.*; +import java.net.*; + +import org.apache.hadoop.conf.*; +import org.apache.hadoop.io.*; +import org.apache.hadoop.mapred.*; +import org.apache.hadoop.fs.*; + + +public class emr { + + static class MongoSuite { + String mongo; + String code; + String workingDir; + + String suite; + + void copy( MongoSuite c ) { + mongo = c.mongo; + code = c.code; + workingDir = c.workingDir; + + suite = c.suite; + + } + + void downloadTo( File localDir ) + throws IOException { + IOUtil.download( mongo , localDir ); + IOUtil.download( code , localDir ); + } + + boolean runTest() + throws IOException { + + // mkdir + File dir = new File( workingDir , suite ); + dir.mkdirs(); + + // download + System.out.println( "going to download" ); + downloadTo( dir ); + + + // explode + System.out.println( "going to explode" ); + IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( code ) , dir ); + String[] res = IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( mongo ) , dir ); + for ( String x : res[0].split( "\n" ) ) { + if ( x.indexOf( "/bin/" ) < 0 ) + continue; + File f = new File( dir.toString() , x ); + if ( ! f.renameTo( new File( dir , IOUtil.urlFileName( x ) ) ) ) + throw new RuntimeException( "rename failed" ); + } + + List<String> cmd = new ArrayList<String>(); + cmd.add( "/usr/bin/python" ); + cmd.add( "buildscripts/smoke.py" ); + + File log_config = new File( dir , "log_config.py" ); + System.out.println( "log_config: " + log_config.exists() ); + if ( log_config.exists() ) { + + java.util.Map<String,Object> properties = IOUtil.readPythonSettings( log_config ); + + cmd.add( "--buildlogger-builder" ); + cmd.add( properties.get( "name" ).toString() ); + + cmd.add( "--buildlogger-buildnum" ); + cmd.add( properties.get( "number" ).toString() ); + + cmd.add( "--buildlogger-credentials" ); + cmd.add( "log_config.py" ); + + cmd.add( "--buildlogger-phase" ); + { + int idx = suite.lastIndexOf( "/" ); + if ( idx < 0 ) + cmd.add( suite ); + else + cmd.add( suite.substring( 0 , idx ) ); + } + + } + + cmd.add( suite ); + + System.out.println( cmd ); + + Process p = Runtime.getRuntime().exec( cmd.toArray( new String[cmd.size()] ) , new String[]{} , dir ); + + List<Thread> threads = new ArrayList<Thread>(); + threads.add( new IOUtil.PipingThread( p.getInputStream() , System.out ) ); + threads.add( new IOUtil.PipingThread( p.getErrorStream() , System.out ) ); + + for ( Thread t : threads ) + t.start(); + + try { + for ( Thread t : threads ) { + t.join(); + } + int rc = p.waitFor(); + return rc == 0; + } + catch ( InterruptedException ie ) { + ie.printStackTrace(); + throw new RuntimeException( "sad" , ie ); + } + + } + + public void readFields( DataInput in ) + throws IOException { + mongo = in.readUTF(); + code = in.readUTF(); + workingDir = in.readUTF(); + + suite = in.readUTF(); + } + + public void write( final DataOutput out ) + throws IOException { + out.writeUTF( mongo ); + out.writeUTF( code ); + out.writeUTF( workingDir ); + + out.writeUTF( suite ); + } + + public String toString() { + return "mongo: " + mongo + " code: " + code + " suite: " + suite + " workingDir: " + workingDir; + } + } + + public static class Map implements Mapper<Text, MongoSuite, Text, IntWritable> { + + public void map( Text key, MongoSuite value, OutputCollector<Text,IntWritable> output, Reporter reporter ) + throws IOException { + + FileLock lock = new FileLock( "mapper" ); + try { + lock.lock(); + + System.out.println( "key: " + key ); + System.out.println( "value: " + value ); + + long start = System.currentTimeMillis(); + boolean passed = value.runTest(); + long end = System.currentTimeMillis(); + + output.collect( new Text( passed ? "passed" : "failed" ) , new IntWritable( 1 ) ); + output.collect( new Text( key.toString() + "-time-seconds" ) , new IntWritable( (int)((end-start)/(1000)) ) ); + output.collect( new Text( key.toString() + "-passed" ) , new IntWritable( passed ? 1 : 0 ) ); + + String ip = IOUtil.readStringFully( new URL( "http://myip.10gen.com/" ).openConnection().getInputStream() ); + ip = ip.substring( ip.indexOf( ":" ) + 1 ).trim(); + output.collect( new Text( ip ) , new IntWritable(1) ); + } + catch ( RuntimeException re ) { + re.printStackTrace(); + throw re; + } + catch ( IOException ioe ) { + ioe.printStackTrace(); + throw ioe; + } + finally { + lock.unlock(); + } + + } + + public void configure(JobConf job) {} + public void close(){} + } + + public static class Reduce implements Reducer<Text, IntWritable, Text, IntWritable> { + + public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text,IntWritable> output , Reporter reporter ) + throws IOException { + + int sum = 0; + while ( values.hasNext() ) { + sum += values.next().get(); + } + output.collect( key , new IntWritable( sum ) ); + } + + public void configure(JobConf job) {} + public void close(){} + } + + public static class MySplit implements InputSplit , Writable { + + public MySplit(){ + } + + MySplit( MongoSuite config , int length ) { + _config = config; + _length = length; + } + + public long getLength() { + return _length; + } + + public String[] getLocations() { + return new String[0]; + } + + public void readFields( DataInput in ) + throws IOException { + _config = new MongoSuite(); + _config.readFields( in ); + _length = in.readInt(); + } + + public void write( final DataOutput out ) + throws IOException { + _config.write( out ); + out.writeInt( _length ); + } + + MongoSuite _config; + int _length; + } + + public static class InputMagic implements InputFormat<Text,MongoSuite> { + + public RecordReader<Text,MongoSuite> getRecordReader( InputSplit split, JobConf job , Reporter reporter ){ + final MySplit s = (MySplit)split; + return new RecordReader<Text,MongoSuite>() { + + public void close(){} + + public Text createKey() { + return new Text(); + } + + public MongoSuite createValue() { + return new MongoSuite(); + } + + public long getPos() { + return _seen ? 1 : 0; + } + + public float getProgress() { + return getPos(); + } + + public boolean next( Text key , MongoSuite value ) { + key.set( s._config.suite ); + value.copy( s._config ); + + + boolean x = _seen; + _seen = true; + return !x; + } + + boolean _seen = false; + }; + } + + public InputSplit[] getSplits( JobConf job , int numSplits ){ + String[] pcs = job.get( "suites" ).split(","); + InputSplit[] splits = new InputSplit[pcs.length]; + for ( int i=0; i<splits.length; i++ ) { + MongoSuite c = new MongoSuite(); + c.suite = pcs[i]; + + c.mongo = job.get( "mongo" ); + c.code = job.get( "code" ); + c.workingDir = job.get( "workingDir" ); + + splits[i] = new MySplit( c , 100 /* XXX */); + } + return splits; + } + + public void validateInput(JobConf job){} + + + } + + /** + * args + * mongo tgz + * code tgz + * output path + * tests to run ? + */ + + public static void main( String[] args ) throws Exception{ + + JobConf conf = new JobConf(); + conf.setJarByClass(emr.class); + + String workingDir = "/data/db/emr/"; + + + // parse args + + int pos = 0; + for ( ; pos < args.length; pos++ ) { + if ( ! args[pos].startsWith( "--" ) ) + break; + + String arg = args[pos].substring(2); + if ( arg.equals( "workingDir" ) ) { + workingDir = args[++pos]; + } + else { + System.err.println( "unknown arg: " + arg ); + throw new RuntimeException( "unknown arg: " + arg ); + } + } + + String mongo = args[pos++]; + String code = args[pos++]; + String output = args[pos++]; + + String suites = ""; + for ( ; pos < args.length; pos++ ) { + if ( suites.length() > 0 ) + suites += ","; + suites += args[pos]; + } + + if ( suites.length() == 0 ) + throw new RuntimeException( "no suites" ); + + System.out.println( "workingDir:\t" + workingDir ); + System.out.println( "mongo:\t" + mongo ); + System.out.println( "code:\t " + code ); + System.out.println( "output\t: " + output ); + System.out.println( "suites\t: " + suites ); + + if ( false ) { + MongoSuite s = new MongoSuite(); + s.mongo = mongo; + s.code = code; + s.workingDir = workingDir; + s.suite = suites; + s.runTest(); + return; + } + + // main hadoop set + conf.set( "mongo" , mongo ); + conf.set( "code" , code ); + conf.set( "workingDir" , workingDir ); + conf.set( "suites" , suites ); + + conf.set( "mapred.map.tasks" , "1" ); + conf.setLong( "mapred.task.timeout" , 4 * 3600 * 1000 /* 4 hours */); + + conf.setOutputKeyClass(Text.class); + conf.setOutputValueClass(IntWritable.class); + + conf.setMapperClass(Map.class); + conf.setReducerClass(Reduce.class); + + conf.setInputFormat(InputMagic.class); + conf.setOutputFormat(TextOutputFormat.class); + + FileOutputFormat.setOutputPath(conf, new Path(output) ); + + // actually run + + JobClient.runJob( conf ); + } +} diff --git a/buildscripts/emr/emr.py b/buildscripts/emr/emr.py new file mode 100644 index 00000000000..e060779cef1 --- /dev/null +++ b/buildscripts/emr/emr.py @@ -0,0 +1,385 @@ + +import os +import sys +import shutil +import datetime +import time +import subprocess +import urllib +import urllib2 +import json +import pprint + +import boto +import simples3 + +import pymongo + +def findSettingsSetup(): + sys.path.append( "./" ) + sys.path.append( "../" ) + sys.path.append( "../../" ) + sys.path.append( "../../../" ) + +findSettingsSetup() +import settings +import buildscripts.utils as utils +import buildscripts.smoke as smoke + +bucket = simples3.S3Bucket( settings.emr_bucket , settings.emr_id , settings.emr_key ) + +def _get_status(): + + def gh( cmds ): + txt = "" + for cmd in cmds: + res = utils.execsys( "git " + cmd ) + txt = txt + res[0] + res[1] + return utils.md5string( txt ) + + return "%s-%s" % ( utils.execsys( "git describe" )[0].strip(), gh( [ "diff" , "status" ] ) ) + +def _get_most_recent_tgz( prefix ): + # this is icky, but works for now + all = [] + for x in os.listdir( "." ): + if not x.startswith( prefix ) or not x.endswith( ".tgz" ): + continue + all.append( ( x , os.stat(x).st_mtime ) ) + + if len(all) == 0: + raise Exception( "can't find file with prefix: " + prefix ) + + all.sort( lambda x,y: int(y[1] - x[1]) ) + + return all[0][0] + +def get_build_info(): + return ( os.environ.get('MONGO_BUILDER_NAME') , os.environ.get('MONGO_BUILD_NUMBER') ) + +def make_tarball(): + + m = _get_most_recent_tgz( "mongodb-" ) + + c = "test-code-emr.tgz" + tar = "tar zcf %s src jstests buildscripts" % c + + log_config = "log_config.py" + if os.path.exists( log_config ): + os.unlink( log_config ) + + credentials = do_credentials() + if credentials: + + builder , buildnum = get_build_info() + + if builder and buildnum: + + file = open( log_config , "wb" ) + file.write( 'username="%s"\npassword="%s"\n' % credentials ) + file.write( 'name="%s"\nnumber=%s\n'% ( builder , buildnum ) ) + + file.close() + + tar = tar + " " + log_config + + utils.execsys( tar ) + return ( m , c ) + +def _put_ine( bucket , local , remote ): + print( "going to put\n\t%s\n\thttp://%s.s3.amazonaws.com/%s" % ( local , settings.emr_bucket , remote ) ) + + for x in bucket.listdir( prefix=remote ): + print( "\talready existed" ) + return remote + + bucket.put( remote , open( local , "rb" ).read() , acl="public-read" ) + return remote + +def build_jar(): + root = "build/emrjar" + src = "buildscripts/emr" + + if os.path.exists( root ): + shutil.rmtree( root ) + os.makedirs( root ) + + for x in os.listdir( src ): + if not x.endswith( ".java" ): + continue + shutil.copyfile( src + "/" + x , root + "/" + x ) + shutil.copyfile( src + "/MANIFEST.MF" , root + "/MANIFEST.FM" ) + + classpath = os.listdir( src + "/lib" ) + for x in classpath: + shutil.copyfile( src + "/lib/" + x , root + "/" + x ) + classpath.append( "." ) + classpath = ":".join(classpath) + + for x in os.listdir( root ): + if x.endswith( ".java" ): + if subprocess.call( [ "javac" , "-cp" , classpath , x ] , cwd=root) != 0: + raise Exception( "compiled failed" ) + + args = [ "jar" , "-cfm" , "emr.jar" , "MANIFEST.FM" ] + for x in os.listdir( root ): + if x.endswith( ".class" ): + args.append( x ) + subprocess.call( args , cwd=root ) + + shutil.copyfile( root + "/emr.jar" , "emr.jar" ) + + return "emr.jar" + +def push(): + mongo , test_code = make_tarball() + print( mongo ) + print( test_code ) + + root = "emr/%s/%s" % ( datetime.date.today().strftime("%Y-%m-%d") , os.uname()[0].lower() ) + + def make_long_name(local,hash): + pcs = local.rpartition( "." ) + h = _get_status() + if hash: + h = utils.md5sum( local ) + return "%s/%s-%s.%s" % ( root , pcs[0] , h , pcs[2] ) + + mongo = _put_ine( bucket , mongo , make_long_name( mongo , False ) ) + test_code = _put_ine( bucket , test_code , make_long_name( test_code , True ) ) + + jar = build_jar() + jar = _put_ine( bucket , jar , make_long_name( jar , False ) ) + + setup = "buildscripts/emr/emrnodesetup.sh" + setup = _put_ine( bucket , setup , make_long_name( setup , True ) ) + + return mongo , test_code , jar , setup + +def run_tests( things , tests ): + if len(tests) == 0: + raise Exception( "no tests" ) + oldNum = len(tests) + tests = fix_suites( tests ) + print( "tests expanded from %d to %d" % ( oldNum , len(tests) ) ) + + print( "things:%s\ntests:%s\n" % ( things , tests ) ) + + emr = boto.connect_emr( settings.emr_id , settings.emr_key ) + + def http(path): + return "http://%s.s3.amazonaws.com/%s" % ( settings.emr_bucket , path ) + + run_s3_path = "emr/%s/%s/%s/" % ( os.getenv( "USER" ) , + os.getenv( "HOST" ) , + datetime.datetime.today().strftime( "%Y%m%d-%H%M" ) ) + + run_s3_root = "s3n://%s/%s/" % ( settings.emr_bucket , run_s3_path ) + + out = run_s3_root + "out" + logs = run_s3_root + "logs" + + jar="s3n://%s/%s" % ( settings.emr_bucket , things[2] ) + step_args=[ http(things[0]) , http(things[1]) , out , ",".join(tests) ] + + step = boto.emr.step.JarStep( "emr main" , jar=jar,step_args=step_args ) + print( "jar:%s\nargs:%s" % ( jar , step_args ) ) + + setup = boto.emr.BootstrapAction( "setup" , "s3n://%s/%s" % ( settings.emr_bucket , things[3] ) , [] ) + + jobid = emr.run_jobflow( name = "Mongo EMR for %s from %s" % ( os.getenv( "USER" ) , os.getenv( "HOST" ) ) , + ec2_keyname = "emr1" , + slave_instance_type = "m1.large" , + ami_version = "latest" , + num_instances=5 , + log_uri = logs , + bootstrap_actions = [ setup ] , + steps = [ step ] ) + + + print( "%s jobid: %s" % ( datetime.datetime.today() , jobid ) ) + + while ( True ): + flow = emr.describe_jobflow( jobid ) + print( "%s status: %s" % ( datetime.datetime.today() , flow.state ) ) + if flow.state == "COMPLETED" or flow.state == "FAILED": + break + time.sleep(30) + + syncdir = "build/emrout/" + jobid + "/" + sync_s3( run_s3_path , syncdir ) + + final_out = "build/emrout/" + jobid + "/" + + print("output in: " + final_out ) + do_output( final_out ) + +def sync_s3( remote_dir , local_dir ): + for x in bucket.listdir( remote_dir ): + out = local_dir + "/" + x[0] + + if os.path.exists( out ) and x[2].find( utils.md5sum( out ) ) >= 0: + continue + + dir = out.rpartition( "/" )[0] + if not os.path.exists( dir ): + os.makedirs( dir ) + + thing = bucket.get( x[0] ) + open( out , "wb" ).write( thing.read() ) + +def fix_suites( suites ): + fixed = [] + for name,x in smoke.expand_suites( suites , False ): + idx = name.find( "/jstests" ) + if idx >= 0: + name = name[idx+1:] + fixed.append( name ) + return fixed + +def do_credentials(): + root = "buildbot.tac" + + while len(root) < 40 : + if os.path.exists( root ): + break + root = "../" + root + + if not os.path.exists( root ): + return None + + credentials = {} + execfile(root, credentials, credentials) + + if "slavename" not in credentials: + return None + + if "passwd" not in credentials: + return None + + return ( credentials["slavename"] , credentials["passwd"] ) + + +def do_output( dir ): + + def go_down( start ): + lst = os.listdir(dir) + if len(lst) != 1: + raise Exception( "sad: " + start ) + return start + "/" + lst[0] + + while "out" not in os.listdir( dir ): + dir = go_down( dir ) + + dir = dir + "/out" + + pieces = os.listdir(dir) + pieces.sort() + + passed = [] + failed = [] + times = {} + + for x in pieces: + if not x.startswith( "part" ): + continue + full = dir + "/" + x + + for line in open( full , "rb" ): + if line.find( "-passed" ) >= 0: + passed.append( line.partition( "-passed" )[0] ) + continue + + if line.find( "-failed" ) >= 0: + failed.append( line.partition( "-failed" )[0] ) + continue + + if line.find( "-time-seconds" ) >= 0: + p = line.partition( "-time-seconds" ) + times[p[0]] = p[2].strip() + continue + + print( "\t" + line.strip() ) + + def print_list(name,lst): + print( name ) + for x in lst: + print( "\t%s\t%s" % ( x , times[x] ) ) + + print_list( "passed" , passed ) + print_list( "failed" , failed ) + + if do_credentials(): + builder , buildnum = get_build_info() + if builder and buildnum: + conn = pymongo.Connection( "bbout1.10gen.cc" ) + db = conn.buildlogs + q = { "builder" : builder , "buildnum" : int(buildnum) } + doc = db.builds.find_one( q ) + + if doc: + print( "\nhttp://buildlogs.mongodb.org/build/%s" % doc["_id"] ) + + +if __name__ == "__main__": + if len(sys.argv) == 1: + print( "need an arg" ) + + elif sys.argv[1] == "tarball": + make_tarball() + elif sys.argv[1] == "jar": + build_jar() + elif sys.argv[1] == "push": + print( push() ) + + elif sys.argv[1] == "sync": + sync_s3( sys.argv[2] , sys.argv[3] ) + + elif sys.argv[1] == "fix_suites": + for x in fix_suites( sys.argv[2:] ): + print(x) + + elif sys.argv[1] == "credentials": + print( do_credentials() ) + + elif sys.argv[1] == "test": + m , c = make_tarball() + build_jar() + cmd = [ "java" , "-cp" , os.environ.get( "CLASSPATH" , "." ) + ":emr.jar" , "emr" ] + + workingDir = "/data/emr/test" + cmd.append( "--workingDir" ) + cmd.append( workingDir ) + if os.path.exists( workingDir ): + shutil.rmtree( workingDir ) + + cmd.append( "file://" + os.getcwd() + "/" + m ) + cmd.append( "file://" + os.getcwd() + "/" + c ) + + out = "/tmp/emrresults" + cmd.append( out ) + if os.path.exists( out ): + shutil.rmtree( out ) + + cmd.append( "jstests/basic1.js" ) + + subprocess.call( cmd ) + + for x in os.listdir( out ): + if x.startswith( "." ): + continue + print( x ) + for z in open( out + "/" + x ): + print( "\t" + z.strip() ) + + elif sys.argv[1] == "output": + do_output( sys.argv[2] ) + + elif sys.argv[1] == "full": + things = push() + run_tests( things , sys.argv[2:] ) + + else: + things = push() + run_tests( things , sys.argv[1:] ) + diff --git a/buildscripts/emr/emrnodesetup.sh b/buildscripts/emr/emrnodesetup.sh new file mode 100644 index 00000000000..546becf3e27 --- /dev/null +++ b/buildscripts/emr/emrnodesetup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +sudo mkdir /mnt/data +sudo ln -s /mnt/data /data +sudo chown hadoop /mnt/data + +sudo easy_install pymongo diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py index dec1030ddad..ef5e1a88877 100755 --- a/buildscripts/errorcodes.py +++ b/buildscripts/errorcodes.py @@ -6,7 +6,7 @@ import re import utils -assertNames = [ "uassert" , "massert" ] +assertNames = [ "uassert" , "massert", "fassert", "fassertFailed" ] def assignErrorCodes(): cur = 10000 @@ -32,10 +32,15 @@ def assignErrorCodes(): codes = [] def readErrorCodes( callback, replaceZero = False ): - ps = [ re.compile( "(([umsg]asser(t|ted))) *\(( *)(\d+)" ) , - re.compile( "((User|Msg|MsgAssertion)Exceptio(n))\(( *)(\d+)" ) , - re.compile( "(((verify))) *\(( *)(\d+)" ) + + quick = [ "assert" , "Exception"] + + ps = [ re.compile( "(([umsgf]asser(t|ted))) *\(( *)(\d+)" ) , + re.compile( "((User|Msg|MsgAssertion)Exceptio(n))\(( *)(\d+)" ), + re.compile( "((fassertFailed)()) *\(( *)(\d+)" ) ] + + bad = [ re.compile( "\sassert *\(" ) ] for x in utils.getAllSourceFiles(): @@ -45,30 +50,46 @@ def readErrorCodes( callback, replaceZero = False ): lineNum = 1 for line in open( x ): - - for p in ps: - - def repl( m ): - m = m.groups() - - start = m[0] - spaces = m[3] - code = m[4] - if code == '0' and replaceZero : - code = getNextCode( lastCodes ) - lastCodes.append( code ) - code = str( code ) - needReplace[0] = True - - print( "Adding code " + code + " to line " + x + ":" + str( lineNum ) ) - - else : - codes.append( ( x , lineNum , line , code ) ) - callback( x , lineNum , line , code ) - - return start + "(" + spaces + code + + found = False + for zz in quick: + if line.find( zz ) >= 0: + found = True + break + + if found: - line = re.sub( p, repl, line ) + if x.find( "src/mongo/" ) >= 0: + for b in bad: + if len(b.findall( line )) > 0: + print( x ) + print( line ) + raise Exception( "you can't use a bare assert" ) + + for p in ps: + + def repl( m ): + m = m.groups() + + start = m[0] + spaces = m[3] + code = m[4] + if code == '0' and replaceZero : + code = getNextCode( lastCodes ) + lastCodes.append( code ) + code = str( code ) + needReplace[0] = True + + print( "Adding code " + code + " to line " + x + ":" + str( lineNum ) ) + + else : + codes.append( ( x , lineNum , line , code ) ) + callback( x , lineNum , line , code ) + + return start + "(" + spaces + code + + line = re.sub( p, repl, line ) + # end if ps loop if replaceZero : lines.append( line ) lineNum = lineNum + 1 @@ -78,6 +99,7 @@ def readErrorCodes( callback, replaceZero = False ): of = open( x + ".tmp", 'w' ) of.write( "".join( lines ) ) of.close() + os.remove(x) os.rename( x + ".tmp", x ) diff --git a/buildscripts/hacks_mandriva.py b/buildscripts/hacks_mandriva.py deleted file mode 100644 index d46170960cc..00000000000 --- a/buildscripts/hacks_mandriva.py +++ /dev/null @@ -1,9 +0,0 @@ - -import os -import glob - -def insert( env , options ): - jslibPaths = glob.glob('/usr/include/js-*/') - if len(jslibPaths) >= 1: - jslibPath = jslibPaths.pop() - env.Append( CPPPATH=[ jslibPath ] )
\ No newline at end of file diff --git a/buildscripts/hacks_ubuntu.py b/buildscripts/hacks_ubuntu.py deleted file mode 100644 index 3de1a6f0b7e..00000000000 --- a/buildscripts/hacks_ubuntu.py +++ /dev/null @@ -1,54 +0,0 @@ - -import os - -def insert( env , options ): - - # now that sm is in the source tree, don't need this - # if not foundxulrunner( env , options ): - # if os.path.exists( "usr/include/mozjs/" ): - # env.Append( CPPDEFINES=[ "MOZJS" ] ) - - return - -def foundxulrunner( env , options ): - best = None - - for x in os.listdir( "/usr/include" ): - if x.find( "xulrunner" ) != 0: - continue - if x == "xulrunner": - best = x - break - best = x - - - if best is None: - print( "warning: using ubuntu without xulrunner-dev. we recommend installing it" ) - return False - - incroot = "/usr/include/" + best + "/" - libroot = "/usr/lib" - if options["linux64"] and os.path.exists("/usr/lib64"): - libroot += "64"; - libroot += "/" + best - - - if not os.path.exists( libroot ): - print( "warning: found xulrunner include but not lib for: " + best ) - return False - - env.Prepend( LIBPATH=[ libroot ] ) - env.Prepend( RPATH=[ libroot ] ) - - env.Prepend( CPPPATH=[ incroot + "stable/" , - incroot + "unstable/" , - incroot ] ) - env.Prepend( CPPPATH=[ "/usr/include/nspr/" ] ) - - env.Append( CPPDEFINES=[ "XULRUNNER" , "OLDJS" ] ) - if best.find( "1.9.0" ) >= 0 or best.endswith("1.9"): - if best.endswith( "1.9.1.9" ): - pass - else: - env.Append( CPPDEFINES=[ "XULRUNNER190" ] ) - return True diff --git a/buildscripts/make_archive.py b/buildscripts/make_archive.py new file mode 100755 index 00000000000..4c12e901a64 --- /dev/null +++ b/buildscripts/make_archive.py @@ -0,0 +1,116 @@ +#!/usr/bin/python + +'''Helper script for constructing an archive (zip or tar) from a list of files. + +The output format (tar, tgz, zip) is determined from the file name, unless the user specifies +--format on the command line. + +This script simplifies the specification of filename transformations, so that, e.g., +src/mongo/foo.cpp and build/linux2/normal/buildinfo.cpp can get put into the same +directory in the archive, perhaps mongodb-2.0.2/src/mongo. + +Usage: + +make_archive.py -o <output-file> [--format (tar|tgz|zip)] \ + [--transform match1=replacement1 [--transform match2=replacement2 [...]]] \ + <input file 1> [...] + +If the input file names start with "@", the file is expected to contain a list of +whitespace-separated file names to include in the archive. This helps get around the Windows +command line length limit. + +Transformations are processed in command-line order and are short-circuiting. So, if a file matches +match1, it is never compared against match2 or later. Matches are just python startswith() +comparisons. + +For a detailed usage example, see src/SConscript.client or src/mongo/SConscript. +''' + +import optparse +import os +import sys + +def main(argv): + opts = parse_options(argv[1:]) + archive = open_archive_for_write(opts.output_filename, opts.archive_format) + try: + for input_filename in opts.input_filenames: + archive.add(input_filename, arcname=get_preferred_filename(input_filename, + opts.transformations)) + finally: + archive.close() + +def parse_options(args): + parser = optparse.OptionParser() + parser.add_option('-o', dest='output_filename', default=None, + help='Name of the archive to output.', metavar='FILE') + parser.add_option('--format', dest='archive_format', default=None, + choices=('zip', 'tar', 'tgz'), + help='Format of archive to create. ' + 'If omitted, use the suffix of the output filename to decide.') + parser.add_option('--transform', action='append', dest='transformations', default=[]) + + (opts, input_filenames) = parser.parse_args(args) + opts.input_filenames = [] + + for input_filename in input_filenames: + if input_filename.startswith('@'): + opts.input_filenames.extend(open(input_filename[1:], 'r').read().split()) + else: + opts.input_filenames.append(input_filename) + + if opts.output_filename is None: + parser.error('-o switch is required') + + if opts.archive_format is None: + if opts.output_filename.endswith('.zip'): + opts.archive_format = 'zip' + elif opts.output_filename.endswith('tar.gz') or opts.output_filename.endswith('.tgz'): + opts.archive_format = 'tgz' + elif opts.output_filename.endswith('.tar'): + opts.archive_format = 'tar' + else: + parser.error('Could not deduce archive format from output filename "%s"' % + opts.output_filename) + + try: + opts.transformations = [ + xform.replace(os.path.altsep or os.path.sep, os.path.sep).split('=', 1) + for xform in opts.transformations] + except Exception, e: + parser.error(e) + + return opts + +def open_archive_for_write(filename, archive_format): + '''Open a tar or zip archive for write, with the given format, and return it. + + The type of archive is determined by the "archive_format" parameter, which should be + "tar", "tgz" (for gzipped tar) or "zip". + ''' + + if archive_format in ('tar', 'tgz'): + import tarfile + mode = 'w' + if archive_format is 'tgz': + mode += '|gz' + return tarfile.open(filename, mode) + if archive_format is 'zip': + import zipfile + # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical, + # for our purposes. WrappedZipFile is a minimal adapter class. + class WrappedZipFile(zipfile.ZipFile): + def add(self, filename, arcname): + return self.write(filename, arcname) + return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED) + raise ValueError('Unsupported archive format "%s"' % archive_format) + +def get_preferred_filename(input_filename, transformations): + for match, replace in transformations: + if input_filename.startswith(match): + return replace + input_filename[len(match):] + return input_filename + +if __name__ == '__main__': + main(sys.argv) + sys.exit(0) diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py new file mode 100644 index 00000000000..42fc52094ca --- /dev/null +++ b/buildscripts/moduleconfig.py @@ -0,0 +1,131 @@ +"""Utility functions for SCons to discover and configure +MongoDB modules (sub-trees of db/modules/). This file exports +two functions: + + discover_modules, which returns a dictionary of module name + to the imported python module object for the module's + build.py file + + configure_modules, which runs per-module configuration, and + is given the SCons environment, its own path, etc + +Each module must have a "build.py" script, which is expected to +have a "configure" function, and optionally a "test" function +if the module exposes per-module tests. +""" + +__all__ = ('discover_modules', 'configure_modules', 'register_module_test') + +import imp +from os import listdir, makedirs +from os.path import abspath, dirname, join, isdir, isfile + +def discover_modules(mongo_root): + """Scan <mongo_root>/db/modules/ for directories that + look like MongoDB modules (i.e. they contain a "build.py" + file), and return a dictionary of module name (the directory + name) to build.py python modules. + """ + found_modules = {} + + module_root = abspath(join(mongo_root, 'db', 'modules')) + if not isdir(module_root): + return found_modules + + for name in listdir(module_root): + root = join(module_root, name) + if '.' in name or not isdir(root): + continue + + build_py = join(root, 'build.py') + module = None + + if isfile(build_py): + print "adding module: %s" % name + fp = open(build_py, "r") + module = imp.load_module("module_" + name, fp, build_py, (".py", "r", imp.PY_SOURCE)) + found_modules[name] = module + fp.close() + + return found_modules + +def configure_modules(modules, conf, env): + """ + Run the configure() function in the build.py python modules + for each module listed in the modules dictionary (as created + by discover_modules). The configure() function should use the + prepare the Mongo build system for building the module. + + build.py files may specify a "customIncludes" flag, which, if + True, causes configure() to be called with three arguments: + the SCons Configure() object, the SCons environment, and an + empty list which should be modified in-place by the configure() + function; if false, configure() is called with only the first + two arguments, and the source files are discovered with a + glob against the <module_root>/src/*.cpp. + + Returns a dictionary mapping module name to a list of source + files to be compiled for the module. + """ + source_map = {} + + for name, module in modules.items(): + print "configuring module: %s" % name + + root = dirname(module.__file__) + module_sources = [] + + if getattr(module, "customIncludes", False): + # then the module configures itself and its + # configure() takes 3 args + module.configure(conf, env, module_sources) + else: + # else we glob the files in the module's src/ + # subdirectory, and its configure() takes 2 args + module.configure(conf, env) + module_sources.extend(env.Glob(join(root, "src/*.cpp"))) + + if not module_sources: + print "WARNING: no source files for module %s, module will not be built." % name + else: + source_map[name] = module_sources + + _setup_module_tests_file(str(env.File(env['MODULETEST_LIST']))) + + return source_map + +module_tests = [] +def register_module_test(*command): + """Modules can register tests as part of their configure(), which + are commands whose exit status indicates the success or failure of + the test. + + Use this function from configure() like: + + register_module_test('/usr/bin/python', '/path/to/module/tests/foo.py') + register_module_test('/bin/bash', '/path/to/module/tests/bar.sh') + + The registered test commands can be run with "scons smokeModuleTests" + """ + command = ' '.join(command) + module_tests.append(command) + +def _setup_module_tests_file(test_file): + """Modules' configure() functions may have called register_module_test, + in which case, we need to record the registered tests' commands into + a text file which smoke.py and SCons know how to work with. + """ + if not module_tests: + return + + folder = dirname(test_file) + if not isdir(folder): + makedirs(folder) + + fp = file(test_file, 'w') + for test in module_tests: + fp.write(test) + fp.write('\n') + fp.close() + print "Generated %s" % test_file + diff --git a/buildscripts/packager.py b/buildscripts/packager.py index 400239cfd54..d0dd492981a 100644 --- a/buildscripts/packager.py +++ b/buildscripts/packager.py @@ -376,7 +376,7 @@ def make_deb(distro, arch, spec, srcdir): oldcwd=os.getcwd() try: os.chdir(sdir) - sysassert(["dpkg-buildpackage", "-a"+distro_arch]) + sysassert(["dpkg-buildpackage", "-a"+distro_arch, "-k Richard Kreuter <richard@10gen.com>"]) finally: os.chdir(oldcwd) r=distro.repodir(arch) @@ -576,7 +576,7 @@ Description: An object/document-oriented database """ s=re.sub("@@PACKAGE_BASENAME@@", "mongodb%s" % spec.suffix(), s) conflict_suffixes=["", "-stable", "-unstable", "-nightly", "-10gen", "-10gen-unstable"] - conflict_suffixes.remove(spec.suffix()) + conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()] s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongodb"+suffix for suffix in conflict_suffixes]), s) f=open(path, 'w') try: @@ -686,7 +686,8 @@ binary-arch: build install #\tdh_installinfo \tdh_installman \tdh_link -\tdh_strip +# Appears to be broken on Ubuntu 11.10...? +#\tdh_strip \tdh_compress \tdh_fixperms \tdh_installdeb @@ -901,8 +902,10 @@ fi %{_bindir}/mongo %{_bindir}/mongodump %{_bindir}/mongoexport -%{_bindir}/mongofiles +#@@VERSION!=2.1.0@@%{_bindir}/mongofiles %{_bindir}/mongoimport +#@@VERSION>=2.1.0@@%{_bindir}/mongooplog +#@@VERSION>=2.1.0@@%{_bindir}/mongoperf %{_bindir}/mongorestore #@@VERSION>1.9@@%{_bindir}/mongotop %{_bindir}/mongostat @@ -952,9 +955,9 @@ fi s=re.sub("@@PACKAGE_REVISION@@", str(int(spec.param("revision"))+1) if spec.param("revision") else "1", s) s=re.sub("@@BINARYDIR@@", BINARYDIR, s) conflict_suffixes=["", "-10gen", "-10gen-unstable"] - conflict_suffixes.remove(suffix) + conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()] s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongo"+_ for _ in conflict_suffixes]), s) - if suffix == "-10gen": + if suffix.endswith("-10gen"): s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-stable", s) s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-stable", s) elif suffix == "-10gen-unstable": @@ -965,9 +968,25 @@ fi lines=[] for line in s.split("\n"): - m = re.search("@@VERSION>(.*)@@(.*)", line) - if m and spec.version_better_than(m.group(1)): - lines.append(m.group(2)) + m = re.search("@@VERSION(>|>=|!=)(\d.*)@@(.*)", line) + if m: + op = m.group(1) + ver = m.group(2) + fn = m.group(3) + if op == '>': + if spec.version_better_than(ver): + lines.append(fn) + elif op == '>=': + if spec.version() == ver or spec.version_better_than(ver): + lines.append(fn) + elif op == '!=': + if spec.version() != ver: + lines.append(fn) + else: + # Since we're inventing our own template system for RPM + # specfiles here, we oughtn't use template syntax we don't + # support. + raise Exception("BUG: probable bug in packager script: %s, %s, %s" % (m.group(1), m.group(2), m.group(3))) else: lines.append(line) s="\n".join(lines) diff --git a/buildscripts/s3md5.py b/buildscripts/s3md5.py index 89800cd6898..3b0d0d82917 100644 --- a/buildscripts/s3md5.py +++ b/buildscripts/s3md5.py @@ -18,7 +18,7 @@ def check_dir( bucket , prefix ): zips = {} md5s = {} for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ): - if key.endswith( ".tgz" ) or key.endswith( ".zip" ): + if key.endswith( ".tgz" ) or key.endswith( ".zip" ) or key.endswith( ".tar.gz" ): zips[key] = etag.replace( '"' , '' ) elif key.endswith( ".md5" ): md5s[key] = True @@ -40,7 +40,7 @@ def run(): bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key ) - for x in [ "osx" , "linux" , "win32" , "sunos5" ]: + for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]: check_dir( bucket , x ) diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py index c46b5d1879d..7740756b42f 100755 --- a/buildscripts/smoke.py +++ b/buildscripts/smoke.py @@ -33,14 +33,14 @@ # off all mongods on a box, which means you can't run two smoke.py # jobs on the same host at once. So something's gotta change. -from __future__ import with_statement - +from datetime import datetime import glob from optparse import OptionParser import os import parser import re import shutil +import shlex import socket from subprocess import (Popen, PIPE, @@ -49,20 +49,29 @@ import sys import time from pymongo import Connection +from pymongo.errors import OperationFailure import utils +try: + import cPickle as pickle +except ImportError: + import pickle + # TODO clean this up so we don't need globals... mongo_repo = os.getcwd() #'./' +failfile = os.path.join(mongo_repo, 'failfile.smoke') test_path = None mongod_executable = None mongod_port = None shell_executable = None continue_on_failure = None +file_of_commands_mode = False tests = [] winners = [] losers = {} +fails = [] # like losers but in format of tests # For replication hash checking replicated_collections = [] @@ -72,6 +81,7 @@ screwy_in_slave = {} smoke_db_prefix = '' small_oplog = False +small_oplog_rs = False # This class just implements the with statement API, for a sneaky # purpose below. @@ -81,10 +91,24 @@ class Nothing(object): def __exit__(self, type, value, traceback): return not isinstance(value, Exception) +def buildlogger(cmd, is_global=False): + # if the environment variable MONGO_USE_BUILDLOGGER + # is set to 'true', then wrap the command with a call + # to buildlogger.py, which sends output to the buidlogger + # machine; otherwise, return as usual. + if os.environ.get('MONGO_USE_BUILDLOGGER', '').lower().strip() == 'true': + if is_global: + return [utils.find_python(), 'buildscripts/buildlogger.py', '-g'] + cmd + else: + return [utils.find_python(), 'buildscripts/buildlogger.py'] + cmd + return cmd + + class mongod(object): def __init__(self, **kwargs): self.kwargs = kwargs self.proc = None + self.auth = False def __enter__(self): self.start() @@ -122,6 +146,15 @@ class mongod(object): print >> sys.stderr, "timeout starting mongod" return False + def setup_admin_user(self, port=mongod_port): + try: + Connection( "localhost" , int(port) ).admin.add_user("admin","password") + except OperationFailure, e: + if e.message == 'need to login': + pass # SERVER-4225 + else: + raise e + def start(self): global mongod_port global mongod @@ -139,25 +172,34 @@ class mongod(object): self.slave = True if os.path.exists(dir_name): if 'slave' in self.kwargs: - argv = ["python", "buildscripts/cleanbb.py", '--nokill', dir_name] + argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name] else: - argv = ["python", "buildscripts/cleanbb.py", dir_name] + argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name] call(argv) utils.ensureDir(dir_name) argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name] if self.kwargs.get('small_oplog'): - argv += ["--master", "--oplogSize", "256"] + argv += ["--master", "--oplogSize", "511"] + if self.kwargs.get('small_oplog_rs'): + argv += ["--replSet", "foo", "--oplogSize", "511"] if self.slave: argv += ['--slave', '--source', 'localhost:' + str(srcport)] if self.kwargs.get('no_journal'): argv += ['--nojournal'] if self.kwargs.get('no_preallocj'): argv += ['--nopreallocj'] + if self.kwargs.get('auth'): + argv += ['--auth'] + self.auth = True print "running " + " ".join(argv) - self.proc = Popen(argv) + self.proc = self._start(buildlogger(argv, is_global=True)) + if not self.did_mongod_start(self.port): raise Exception("Failed to start mongod") + if self.auth: + self.setup_admin_user(self.port) + if self.slave: local = Connection(port=self.port, slave_okay=True).local synced = False @@ -166,20 +208,53 @@ class mongod(object): for source in local.sources.find(fields=["syncedTo"]): synced = synced and "syncedTo" in source and source["syncedTo"] + def _start(self, argv): + """In most cases, just call subprocess.Popen(). On windows, + add the started process to a new Job Object, so that any + child processes of this process can be killed with a single + call to TerminateJobObject (see self.stop()). + """ + proc = Popen(argv) + + if os.sys.platform == "win32": + # Create a job object with the "kill on job close" + # flag; this is inherited by child processes (ie + # the mongod started on our behalf by buildlogger) + # and lets us terminate the whole tree of processes + # rather than orphaning the mongod. + import win32job + + self.job_object = win32job.CreateJobObject(None, '') + + job_info = win32job.QueryInformationJobObject( + self.job_object, win32job.JobObjectExtendedLimitInformation) + job_info['BasicLimitInformation']['LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE + win32job.SetInformationJobObject( + self.job_object, + win32job.JobObjectExtendedLimitInformation, + job_info) + + win32job.AssignProcessToJobObject(self.job_object, proc._handle) + + return proc + def stop(self): if not self.proc: print >> sys.stderr, "probable bug: self.proc unset in stop()" return try: - # This function not available in Python 2.5 - self.proc.terminate() - except AttributeError: if os.sys.platform == "win32": - import win32process - win32process.TerminateProcess(self.proc._handle, -1) + import win32job + win32job.TerminateJobObject(self.job_object, -1) + import time + # Windows doesn't seem to kill the process immediately, so give it some time to die + time.sleep(5) else: - from os import kill - kill(self.proc.pid, 15) + # This function not available in Python 2.5 + self.proc.terminate() + except AttributeError: + from os import kill + kill(self.proc.pid, 15) self.proc.wait() sys.stderr.flush() sys.stdout.flush() @@ -241,12 +316,31 @@ def check_db_hashes(master, slave): lost_in_master.append(db) +def ternary( b , l="true", r="false" ): + if b: + return l + return r + # Blech. def skipTest(path): - if small_oplog: - if os.path.basename(path) in ["cursor8.js", "indexh.js", "dropdb.js"]: + basename = os.path.basename(path) + parentDir = os.path.basename(os.path.dirname(path)) + if small_oplog: # For tests running in parallel + if basename in ["cursor8.js", "indexh.js", "dropdb.js"]: + return True + if auth or keyFile: # For tests running with auth + # Skip any tests that run with auth explicitly + if parentDir == "auth" or "auth" in basename or parentDir == "tool": # SERVER-6368 + return True + # These tests don't pass with authentication due to limitations of the test infrastructure, + # not due to actual bugs. + if os.path.join(parentDir,basename) in ["sharding/sync3.js", "sharding/sync6.js", "sharding/parallel.js", "jstests/bench_test1.js", "jstests/bench_test2.js", "jstests/bench_test3.js"]: + return True + # These tests fail due to bugs + if os.path.join(parentDir,basename) in ["sharding/sync_conn_cmd.js"]: return True + return False def runTest(test): @@ -259,11 +353,22 @@ def runTest(test): if skipTest(path): print "skipping " + path return - if ext == ".js": + if file_of_commands_mode: + # smoke.py was invoked like "--mode files --from-file foo", + # so don't try to interpret the test path too much + if os.sys.platform == "win32": + argv = [path] + else: + argv = shlex.split(path) + path = argv[0] + # if the command is a python script, use the script name + if os.path.basename(path) in ('python', 'python.exe'): + path = argv[1] + elif ext == ".js": argv = [shell_executable, "--port", mongod_port] if not usedb: argv += ["--nodb"] - if small_oplog: + if small_oplog or small_oplog_rs: argv += ["--eval", 'testingReplication = true;'] argv += [path] elif ext in ["", ".exe"]: @@ -271,36 +376,68 @@ def runTest(test): if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]: argv = [path] # more blech - elif os.path.basename(path) == 'mongos': + elif os.path.basename(path) in ['mongos', 'mongos.exe']: argv = [path, "--test"] else: argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path, "--port", mongod_port] else: raise Bug("fell off in extenstion case: %s" % path) - sys.stderr.write( "starting test : %s \n" % os.path.basename(path) ) - sys.stderr.flush() - print " *******************************************" - print " Test : " + os.path.basename(path) + " ..." - t1 = time.time() + + if keyFile: + f = open(keyFile, 'r') + keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace + f.close() + else: + keyFileData = None + + + # sys.stdout.write() is more atomic than print, so using it prevents + # lines being interrupted by, e.g., child processes + sys.stdout.write(" *******************************************\n") + sys.stdout.write(" Test : %s ...\n" % os.path.basename(path)) + sys.stdout.flush() + # FIXME: we don't handle the case where the subprocess # hangs... that's bad. - if argv[0].endswith( 'mongo' ) and not '--eval' in argv : - argv = argv + [ '--eval', 'TestData = new Object();' + - 'TestData.testPath = "' + path + '";' + - 'TestData.testFile = "' + os.path.basename( path ) + '";' + - 'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + - 'TestData.noJournal = ' + ( 'true' if no_journal else 'false' ) + ";" + - 'TestData.noJournalPrealloc = ' + ( 'true' if no_preallocj else 'false' ) + ";" ] + if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv : + evalString = 'TestData = new Object();' + \ + 'TestData.testPath = "' + path + '";' + \ + 'TestData.testFile = "' + os.path.basename( path ) + '";' + \ + 'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \ + 'TestData.noJournal = ' + ternary( no_journal ) + ";" + \ + 'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \ + 'TestData.auth = ' + ternary( auth ) + ";" + \ + 'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \ + 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";" + if os.sys.platform == "win32": + # double quotes in the evalString on windows; this + # prevents the backslashes from being removed when + # the shell (i.e. bash) evaluates this string. yuck. + evalString = evalString.replace('\\', '\\\\') + + if auth and usedb: + evalString += 'jsTest.authenticate(db.getMongo());' + + argv = argv + [ '--eval', evalString] if argv[0].endswith( 'test' ) and no_preallocj : argv = argv + [ '--nopreallocj' ] - print argv - r = call(argv, cwd=test_path) + sys.stdout.write(" Command : %s\n" % ' '.join(argv)) + sys.stdout.write(" Date : %s\n" % datetime.now().ctime()) + sys.stdout.flush() + + os.environ['MONGO_TEST_FILENAME'] = os.path.basename(path) + t1 = time.time() + r = call(buildlogger(argv), cwd=test_path) t2 = time.time() - print " " + str((t2 - t1) * 1000) + "ms" + del os.environ['MONGO_TEST_FILENAME'] + + sys.stdout.write(" %fms\n" % ((t2 - t1) * 1000)) + sys.stdout.flush() + if r != 0: raise TestExitFailure(path, r) @@ -317,17 +454,49 @@ def run_tests(tests): # dbpath, etc., and so long as we shut ours down properly, # starting this mongod shouldn't break anything, though.) - # The reason we use with is so that we get __exit__ semantics + # The reason we want to use "with" is so that we get __exit__ semantics + # but "with" is only supported on Python 2.5+ - with mongod(small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj) as master: - with mongod(slave=True) if small_oplog else Nothing() as slave: - if small_oplog: + master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__() + try: + if small_oplog: + slave = mongod(slave=True).__enter__() + elif small_oplog_rs: + slave = mongod(slave=True,small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__() + primary = Connection(port=master.port, slave_okay=True); + + primary.admin.command({'replSetInitiate' : {'_id' : 'foo', 'members' : [ + {'_id': 0, 'host':'localhost:%s' % master.port}, + {'_id': 1, 'host':'localhost:%s' % slave.port,'priority':0}]}}) + + ismaster = False + while not ismaster: + result = primary.admin.command("ismaster"); + ismaster = result["ismaster"] + time.sleep(1) + else: + slave = Nothing() + + try: + if small_oplog or small_oplog_rs: master.wait_for_repl() - for test in tests: + tests_run = 0 + for tests_run, test in enumerate(tests): try: + fails.append(test) runTest(test) + fails.pop() winners.append(test) + + if small_oplog or small_oplog_rs: + master.wait_for_repl() + elif test[1]: # reach inside test and see if startmongod is true + if (tests_run+1) % 20 == 0: + # restart mongo every 20 times, for our 32-bit machines + master.__exit__(None, None, None) + master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__() + except TestFailure, f: try: print f @@ -341,12 +510,15 @@ def run_tests(tests): return 1 if isinstance(slave, mongod): check_db_hashes(master, slave) - + finally: + slave.__exit__(None, None, None) + finally: + master.__exit__(None, None, None) return 0 def report(): - print "%d test%s succeeded" % (len(winners), '' if len(winners) == 1 else 's') + print "%d tests succeeded" % len(winners) num_missed = len(tests) - (len(winners) + len(losers.keys())) if num_missed: print "%d tests didn't get run" % num_missed @@ -368,18 +540,35 @@ at the end of testing:""" % (src, dst) at the end of testing:""" for db in screwy_in_slave.keys(): print "%s\t %s" % (db, screwy_in_slave[db]) - if small_oplog and not (lost_in_master or lost_in_slave or screwy_in_slave): + if (small_oplog or small_oplog_rs) and not (lost_in_master or lost_in_slave or screwy_in_slave): print "replication ok for %d collections" % (len(replicated_collections)) if losers or lost_in_slave or lost_in_master or screwy_in_slave: raise Exception("Test failures") - -def expand_suites(suites): +suiteGlobalConfig = {"js": ("[!_]*.js", True), + "quota": ("quota/*.js", True), + "jsPerf": ("perf/*.js", True), + "disk": ("disk/*.js", True), + "jsSlowNightly": ("slowNightly/*.js", True), + "jsSlowWeekly": ("slowWeekly/*.js", False), + "parallel": ("parallel/*.js", True), + "clone": ("clone/*.js", False), + "repl": ("repl/*.js", False), + "replSets": ("replsets/*.js", False), + "dur": ("dur/*.js", False), + "auth": ("auth/*.js", False), + "sharding": ("sharding/*.js", False), + "tool": ("tool/*.js", False), + "aggregation": ("aggregation/*.js", True), + "multiVersion": ("multiVersion/*.js", True ) + } + +def expand_suites(suites,expandUseDB=True): globstr = None tests = [] for suite in suites: if suite == 'all': - return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'parallel', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool']) + return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'],expandUseDB=expandUseDB) if suite == 'test': if os.sys.platform == "win32": program = 'test.exe' @@ -405,31 +594,31 @@ def expand_suites(suites): program = 'mongos' tests += [(os.path.join(mongo_repo, program), False)] elif os.path.exists( suite ): - tests += [ ( os.path.join( mongo_repo , suite ) , True ) ] + usedb = True + for name in suiteGlobalConfig: + if suite in glob.glob( "jstests/" + suiteGlobalConfig[name][0] ): + usedb = suiteGlobalConfig[name][1] + break + tests += [ ( os.path.join( mongo_repo , suite ) , usedb ) ] else: try: - globstr, usedb = {"js": ("[!_]*.js", True), - "quota": ("quota/*.js", True), - "jsPerf": ("perf/*.js", True), - "disk": ("disk/*.js", True), - "jsSlowNightly": ("slowNightly/*.js", True), - "jsSlowWeekly": ("slowWeekly/*.js", True), - "parallel": ("parallel/*.js", True), - "clone": ("clone/*.js", False), - "repl": ("repl/*.js", False), - "replSets": ("replsets/*.js", False), - "dur": ("dur/*.js", False), - "auth": ("auth/*.js", False), - "sharding": ("sharding/*.js", False), - "tool": ("tool/*.js", False)}[suite] + globstr, usedb = suiteGlobalConfig[suite] except KeyError: raise Exception('unknown test suite %s' % suite) if globstr: - globstr = os.path.join(mongo_repo, (os.path.join(('jstests/' if globstr.endswith('.js') else ''), globstr))) - paths = glob.glob(globstr) - paths.sort() - tests += [(path, usedb) for path in paths] + if usedb and not expandUseDB: + tests += [ (suite,False) ] + else: + if globstr.endswith('.js'): + loc = 'jstests/' + else: + loc = '' + globstr = os.path.join(mongo_repo, (os.path.join(loc, globstr))) + globstr = os.path.normpath(globstr) + paths = glob.glob(globstr) + paths.sort() + tests += [(path, usedb) for path in paths] return tests @@ -438,8 +627,118 @@ def add_exe(e): e += ".exe" return e +def set_globals(options, tests): + global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path + global file_of_commands_mode + #Careful, this can be called multiple times + test_path = options.test_path + + mongod_executable = add_exe(options.mongod_executable) + if not os.path.exists(mongod_executable): + raise Exception("no mongod found in this directory.") + + mongod_port = options.mongod_port + + shell_executable = add_exe( options.shell_executable ) + if not os.path.exists(shell_executable): + raise Exception("no mongo shell found in this directory.") + + continue_on_failure = options.continue_on_failure + smoke_db_prefix = options.smoke_db_prefix + small_oplog = options.small_oplog + if hasattr(options, "small_oplog_rs"): + small_oplog_rs = options.small_oplog_rs + no_journal = options.no_journal + no_preallocj = options.no_preallocj + if options.mode == 'suite' and tests == ['client']: + # The client suite doesn't work with authentication + if options.auth: + print "Not running client suite with auth even though --auth was provided" + auth = False; + keyFile = False; + else: + auth = options.auth + keyFile = options.keyFile + + if auth and not keyFile: + # if only --auth was given to smoke.py, load the + # default keyFile from jstests/libs/authTestsKey + keyFile = os.path.join(mongo_repo, 'jstests', 'libs', 'authTestsKey') + + # if smoke.py is running a list of commands read from a + # file (or stdin) rather than running a suite of js tests + file_of_commands_mode = options.File and options.mode == 'files' + +def clear_failfile(): + if os.path.exists(failfile): + os.remove(failfile) + +def run_old_fails(): + global tests + + try: + f = open(failfile, 'r') + testsAndOptions = pickle.load(f) + f.close() + except Exception: + try: + f.close() + except: + pass + clear_failfile() + return # This counts as passing so we will run all tests + + tests = [x[0] for x in testsAndOptions] + passed = [] + try: + for (i, (test, options)) in enumerate(testsAndOptions): + # SERVER-5102: until we can figure out a better way to manage + # dependencies of the --only-old-fails build phase, just skip + # tests which we can't safely run at this point + path, usedb = test + + if not os.path.exists(path): + passed.append(i) + winners.append(test) + continue + + filename = os.path.basename(path) + if filename in ('test', 'test.exe') or filename.endswith('.js'): + set_globals(options, [filename]) + oldWinners = len(winners) + run_tests([test]) + if len(winners) != oldWinners: # can't use return value due to continue_on_failure + passed.append(i) + finally: + for offset, i in enumerate(passed): + testsAndOptions.pop(i - offset) + + if testsAndOptions: + f = open(failfile, 'w') + pickle.dump(testsAndOptions, f) + else: + clear_failfile() + + report() # exits with failure code if there is an error + +def add_to_failfile(tests, options): + try: + f = open(failfile, 'r') + testsAndOptions = pickle.load(f) + except Exception: + testsAndOptions = [] + + for test in tests: + if (test, options) not in testsAndOptions: + testsAndOptions.append( (test, options) ) + + f = open(failfile, 'w') + pickle.dump(testsAndOptions, f) + + + def main(): - global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, smoke_db_prefix, test_path + global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*") parser.add_option('--mode', dest='mode', default='suite', help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)') @@ -451,7 +750,7 @@ def main(): "currently only used for 'client' (%default)") parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'), help='Path to mongod to run (%default)') - parser.add_option('--port', dest='mongod_port', default="32000", + parser.add_option('--port', dest='mongod_port', default="27999", help='Port the mongod will bind to (%default)') parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'), help='Path to mongo, for .js test files (%default)') @@ -465,55 +764,96 @@ def main(): parser.add_option('--small-oplog', dest='small_oplog', default=False, action="store_true", help='Run tests with master/slave replication & use a small oplog') + parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False, + action="store_true", + help='Run tests with replica set replication & use a small oplog') parser.add_option('--nojournal', dest='no_journal', default=False, action="store_true", help='Do not turn on journaling in tests') parser.add_option('--nopreallocj', dest='no_preallocj', default=False, action="store_true", help='Do not preallocate journal files in tests') + parser.add_option('--auth', dest='auth', default=False, + action="store_true", + help='Run standalone mongods in tests with authentication enabled') + parser.add_option('--keyFile', dest='keyFile', default=None, + help='Path to keyFile to use to run replSet and sharding tests with authentication enabled') + parser.add_option('--ignore', dest='ignore_files', default=None, + help='Pattern of files to ignore in tests') + parser.add_option('--only-old-fails', dest='only_old_fails', default=False, + action="store_true", + help='Check the failfile and only run all tests that failed last time') + parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False, + action="store_true", + help='Clear the failfile. Do this if all tests pass') + parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False, + action="store_true", + help='Clear database files from previous smoke.py runs') + + # Buildlogger invocation from command line + parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None, + action="store", help='Set the "builder name" for buildlogger') + parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None, + action="store", help='Set the "build number" for buildlogger') + parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None, + action="store", help='Path to Python file containing buildlogger credentials') + parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None, + action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)') + global tests (options, tests) = parser.parse_args() - print tests - - test_path = options.test_path - - mongod_executable = add_exe(options.mongod_executable) - if not os.path.exists(mongod_executable): - raise Exception("no mongod found in this directory.") - - mongod_port = options.mongod_port - - shell_executable = add_exe( options.shell_executable ) - if not os.path.exists(shell_executable): - raise Exception("no mongo shell found in this directory.") + set_globals(options, tests) - continue_on_failure = options.continue_on_failure - smoke_db_prefix = options.smoke_db_prefix - small_oplog = options.small_oplog - no_journal = options.no_journal - no_preallocj = options.no_preallocj + buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials) + if all(buildlogger_opts): + os.environ['MONGO_USE_BUILDLOGGER'] = 'true' + os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder + os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum + os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials + if options.buildlogger_phase: + os.environ['MONGO_PHASE'] = options.buildlogger_phase + elif any(buildlogger_opts): + # some but not all of the required options were sete + raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials") if options.File: if options.File == '-': tests = sys.stdin.readlines() else: - with open(options.File) as f: - tests = f.readlines() + f = open(options.File) + tests = f.readlines() tests = [t.rstrip('\n') for t in tests] + if options.only_old_fails: + run_old_fails() + return + elif options.reset_old_fails: + clear_failfile() + return + # If we're in suite mode, tests is a list of names of sets of tests. if options.mode == 'suite': tests = expand_suites(tests) elif options.mode == 'files': tests = [(os.path.abspath(test), True) for test in tests] + if options.ignore_files != None : + ignore_patt = re.compile( options.ignore_files ) + tests = filter( lambda x : ignore_patt.search( x[0] ) == None, tests ) + if not tests: - raise Exception( "no tests specified" ) + print "warning: no tests specified" + return + + if options.with_cleanbb: + dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db') + call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot]) try: run_tests(tests) finally: + add_to_failfile(fails, options) report() diff --git a/buildscripts/utils.py b/buildscripts/utils.py index bde2b083612..1f04d7b6af4 100644 --- a/buildscripts/utils.py +++ b/buildscripts/utils.py @@ -1,9 +1,14 @@ +import codecs import re import socket import time import os +import os.path +import itertools +import subprocess import sys +import hashlib # various utilities that are handy @@ -136,34 +141,84 @@ def didMongodStart( port=27017 , timeout=20 ): timeout = timeout - 1 return False -def smoke_python_name(): - # if this script is being run by py2.5 or greater, - # then we assume that "python" points to a 2.5 or - # greater python VM. otherwise, explicitly use 2.5 - # which we assume to be installed. - min_version_tuple = (2, 5) +def which(executable): + if sys.platform == 'win32': + paths = os.environ.get('Path', '').split(';') + else: + paths = os.environ.get('PATH', '').split(':') + + for path in paths: + path = os.path.expandvars(path) + path = os.path.expanduser(path) + path = os.path.abspath(path) + executable_path = os.path.join(path, executable) + if os.path.exists(executable_path): + return executable_path + + return executable + +def md5sum( file ): + #TODO error handling, etc.. + return execsys( "md5sum " + file )[0].partition(" ")[0] + +def md5string( a_string ): + return hashlib.md5(a_string).hexdigest() + +def find_python(min_version=(2, 5)): try: - if sys.version_info >= min_version_tuple: + if sys.version_info >= min_version: return sys.executable except AttributeError: # In case the version of Python is somehow missing sys.version_info or sys.executable. pass - import subprocess version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE) - binaries = ['python2.5', 'python2.6', 'python2.7', 'python25', 'python26', 'python27', 'python'] + binaries = ('python27', 'python2.7', 'python26', 'python2.6', 'python25', 'python2.5', 'python') for binary in binaries: try: - # py-2.4 compatible replacement for shell backticks out, err = subprocess.Popen([binary, '-V'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() for stream in (out, err): match = version.search(stream) if match: versiontuple = tuple(map(int, match.group(1).split('.'))) - if versiontuple >= min_version_tuple: - return binary + if versiontuple >= min_version: + return which(binary) except: pass - # if that all fails, fall back to "python" - return "python" + + raise Exception('could not find suitable Python (version >= %s)' % '.'.join(str(v) for v in min_version)) + +def smoke_command(*args): + # return a list of arguments that comprises a complete + # invocation of smoke.py + here = os.path.dirname(__file__) + smoke_py = os.path.abspath(os.path.join(here, 'smoke.py')) + # the --with-cleanbb argument causes smoke.py to run + # buildscripts/cleanbb.py before each test phase; this + # prevents us from running out of disk space on slaves + return [find_python(), smoke_py, '--with-cleanbb'] + list(args) + +def run_smoke_command(*args): + # to run a command line script from a scons Alias (or any + # Action), the command sequence must be enclosed in a list, + # otherwise SCons treats it as a list of dependencies. + return [smoke_command(*args)] + +# unicode is a pain. some strings cannot be unicode()'d +# but we want to just preserve the bytes in a human-readable +# fashion. this codec error handler will substitute the +# repr() of the offending bytes into the decoded string +# at the position they occurred +def replace_with_repr(unicode_error): + offender = unicode_error.object[unicode_error.start:unicode_error.end] + return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end) + +codecs.register_error('repr', replace_with_repr) + +def unicode_dammit(string, encoding='utf8'): + # convert a string to a unicode, using the Python + # representation of non-ascii bytes when necessary + # + # name inpsired by BeautifulSoup's "UnicodeDammit" + return string.decode(encoding, 'repr') |