Imported Upstream version 2.2.0upstream/2.2.0

24 files changed, 2468 insertions, 515 deletions

diff --git a/buildscripts/__init__.py b/buildscripts/__init__.py
index 839da9dd919..442c233b031 100644
--- a/buildscripts/__init__.py
+++ b/buildscripts/__init__.py
@@ -1,13 +1,5 @@
 
-import hacks_mandriva
-import hacks_ubuntu
 import os;
 
 def findHacks( un ):
-    if un[0] == 'Linux' and (os.path.exists("/etc/debian_version") or
-                             un[3].find("Ubuntu") >= 0):
-        return hacks_ubuntu
-    if un[0] == 'Linux' and (os.path.exists("/etc/mandriva-release") or
-                            un[3].find("mnb") >= 0):
-        return hacks_mandriva
     return None
diff --git a/buildscripts/bb.py b/buildscripts/bb.py
index 1e878283fed..d2f3819b1de 100644
--- a/buildscripts/bb.py
+++ b/buildscripts/bb.py
@@ -13,7 +13,7 @@ def checkOk():
     
     m = "v" + m[0]
     print( m )
-    print( "excpted version [" + m + "]" )
+    print( "expected version [" + m + "]" )
 
     from subprocess import Popen, PIPE
     diff = Popen( [ "git", "diff", "origin/v1.2" ], stdout=PIPE ).communicate()[ 0 ]
diff --git a/buildscripts/bcp.py b/buildscripts/bcp.py
new file mode 100644
index 00000000000..dda20924e79
--- /dev/null
+++ b/buildscripts/bcp.py
@@ -0,0 +1,40 @@
+
+import utils
+import os
+import shutil
+import sys
+
+def go( boost_root ):
+    
+    OUTPUT = "src/third_party/boost"
+    if os.path.exists( OUTPUT ):
+        shutil.rmtree( OUTPUT )
+
+    cmd = [ "bcp" , "--scan" , "--boost=%s" % boost_root ]
+    
+    src = utils.getAllSourceFiles()
+    
+    cmd += src
+    cmd.append( OUTPUT )
+
+    if not os.path.exists( OUTPUT ):
+        os.makedirs( OUTPUT )
+
+    res = utils.execsys( cmd )
+    
+    out = open( OUTPUT + "/bcp-out.txt" , 'w' )
+    out.write( res[0] )
+    out.close()
+    
+    out = open( OUTPUT + "/notes.txt" , 'w' )
+    out.write( "command: " + " ".join( cmd ) )
+    out.close()
+
+    print( res[1] )
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        print( "usage: python %s <boost root directory>" % sys.argv[0] )
+        sys.exit(1)
+    go( sys.argv[1] )
+
diff --git a/buildscripts/benchmark_tools.py b/buildscripts/benchmark_tools.py
deleted file mode 100644
index 54a86b019d5..00000000000
--- a/buildscripts/benchmark_tools.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import os
-import urllib
-import urllib2
-import sys
-
-try:
-    import json
-except:
-    import simplejson as json # need simplejson for python < 2.6
-
-sys.path.append( "." )
-sys.path.append( ".." )
-sys.path.append( "../../" )
-sys.path.append( "../../../" )
-
-
-import settings
-
-def machine_info(extra_info=""):
-    """Get a dict representing the "machine" section of a benchmark result.
-
-    ie:
-    {
-        "os_name": "OS X",
-        "os_version": "10.5",
-        "processor": "2.4 GHz Intel Core 2 Duo",
-        "memory": "3 GB 667 MHz DDR2 SDRAM",
-        "extra_info": "Python 2.6"
-    }
-
-    Must have a settings.py file on sys.path that defines "processor" and "memory"
-    variables.
-    """
-    machine = {}
-    (machine["os_name"], _, machine["os_version"], _, _) = os.uname()
-    machine["processor"] = settings.processor
-    machine["memory"] = settings.memory
-    machine["extra_info"] = extra_info
-    return machine
-
-def post_data(data, machine_extra_info="", post_url="http://mongo-db.appspot.com/benchmark"):
-    """Post a benchmark data point.
-
-    data should be a Python dict that looks like:
-        {
-          "benchmark": {
-            "project": "http://github.com/mongodb/mongo-python-driver",
-            "name": "insert test",
-            "description": "test inserting 10000 documents with the C extension enabled",
-            "tags": ["insert", "python"]
-          },
-          "trial": {
-            "server_hash": "4f5a8d52f47507a70b6c625dfb5dbfc87ba5656a",
-            "client_hash": "8bf2ad3d397cbde745fd92ad41c5b13976fac2b5",
-            "result": 67.5,
-            "extra_info": "some logs or something"
-          }
-        }
-    """
-    data["machine"] = machine_info(machine_extra_info)
-    urllib2.urlopen(post_url, urllib.urlencode({"payload": json.dumps(data)}))
-    return data
diff --git a/buildscripts/build_and_test_client.py b/buildscripts/build_and_test_client.py
new file mode 100755
index 00000000000..1b97f623cc5
--- /dev/null
+++ b/buildscripts/build_and_test_client.py
@@ -0,0 +1,75 @@
+#!/usr/bin/python
+
+'''Script to attempt an isolated build of the C++ driver and its examples.
+
+Working directory must be the repository root.
+
+Usage:
+
+./buildscripts/build_and_test_client.py <mongo client archive file> [optional scons arguments]
+
+The client is built in a temporary directory, and the sample programs are run against a mongod
+instance found in the current working directory.  The temporary directory and its contents are
+destroyed at the end of execution.
+'''
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import tarfile
+import zipfile
+
+import utils
+
+def main(args):
+    archive_file = args[1]
+    scons_args = args[2:]
+    build_and_test(archive_file, scons_args)
+
+def build_and_test(archive_name, scons_args):
+    work_dir = tempfile.mkdtemp()
+    try:
+        archive = open_archive(archive_name)
+        extracted_root = extract_archive(work_dir, archive)
+        run_scons(extracted_root, scons_args)
+        smoke_client(extracted_root)
+    finally:
+        shutil.rmtree(work_dir)
+
+def open_tar(archive_name):
+    return tarfile.open(archive_name, 'r')
+
+def open_zip(archive_name):
+    class ZipWrapper(zipfile.ZipFile):
+        def getnames(self):
+            return self.namelist()
+    return ZipWrapper(archive_name, 'r')
+
+def open_archive(archive_name):
+    try:
+        return open_tar(archive_name)
+    except:
+        return open_zip(archive_name)
+
+def extract_archive(work_dir, archive_file):
+    archive_file.extractall(path=work_dir)
+    return os.path.join(
+        work_dir,
+        os.path.dirname([n for n in archive_file.getnames() if n.endswith('SConstruct')][0])
+        )
+
+def run_scons(extracted_root, scons_args):
+    rc = subprocess.call(['scons', '-C', extracted_root, ] + scons_args + ['clientTests'])
+    if rc is not 0:
+        sys.exit(rc)
+
+def smoke_client(extracted_root):
+    rc = subprocess.call(utils.smoke_command("--test-path", extracted_root, "client"))
+    if rc is not 0:
+        sys.exit(rc)
+
+if __name__ == '__main__':
+    main(sys.argv)
+    sys.exit(0)
diff --git a/buildscripts/buildlogger.py b/buildscripts/buildlogger.py
new file mode 100644
index 00000000000..9f3feacbe55
--- /dev/null
+++ b/buildscripts/buildlogger.py
@@ -0,0 +1,480 @@
+"""
+buildlogger.py
+
+Wrap a command (specified on the command line invocation of buildlogger.py)
+and send output in batches to the buildlogs web application via HTTP POST.
+
+The script configures itself from environment variables:
+
+  required env vars:
+    MONGO_BUILDER_NAME (e.g. "Nightly Linux 64-bit")
+    MONGO_BUILD_NUMBER (an integer)
+    MONGO_TEST_FILENAME (not required when invoked with -g)
+
+  optional env vars:
+    MONGO_PHASE (e.g. "core", "slow nightly", etc)
+    MONGO_* (any other environment vars are passed to the web app)
+    BUILDLOGGER_CREDENTIALS (see below)
+
+This script has two modes: a "test" mode, intended to wrap the invocation of
+an individual test file, and a "global" mode, intended to wrap the mongod
+instances that run throughout the duration of a mongo test phase (the logs
+from "global" invocations are displayed interspersed with the logs of each
+test, in order to let the buildlogs web app display the full output sensibly.)
+
+If the BUILDLOGGER_CREDENTIALS environment variable is set, it should be a
+path to a valid Python file containing "username" and "password" variables,
+which should be valid credentials for authenticating to the buildlogger web
+app. For example:
+
+    username = "hello"
+    password = "world"
+
+If BUILDLOGGER_CREDENTIALS is a relative path, then the working directory
+and the directories one, two, and three levels up, are searched, in that
+order.
+"""
+
+import functools
+import os
+import os.path
+import re
+import signal
+import socket
+import subprocess
+import sys
+import time
+import traceback
+import urllib2
+import utils
+
+# suppress deprecation warnings that happen when
+# we import the 'buildbot.tac' file below
+import warnings
+warnings.simplefilter('ignore', DeprecationWarning)
+
+try:
+    import json
+except:
+    try:
+        import simplejson as json
+    except:
+        json = None
+
+# try to load the shared secret from settings.py
+# which will be one, two, or three directories up
+# from this file's location
+credentials_file = os.environ.get('BUILDLOGGER_CREDENTIALS', 'buildbot.tac')
+credentials_loc, credentials_name = os.path.split(credentials_file)
+if not credentials_loc:
+    here = os.path.abspath(os.path.dirname(__file__))
+    possible_paths = [
+        os.path.abspath(os.path.join(here, '..')),
+        os.path.abspath(os.path.join(here, '..', '..')),
+        os.path.abspath(os.path.join(here, '..', '..', '..')),
+    ]
+else:
+    possible_paths = [credentials_loc]
+
+username, password = None, None
+for path in possible_paths:
+    credentials_path = os.path.join(path, credentials_name)
+    if os.path.isfile(credentials_path):
+        credentials = {}
+        try:
+            execfile(credentials_path, credentials, credentials)
+            username = credentials.get('slavename', credentials.get('username'))
+            password = credentials.get('passwd', credentials.get('password'))
+            break
+        except:
+            pass
+
+
+URL_ROOT = 'http://buildlogs.mongodb.org/'
+TIMEOUT_SECONDS = 10
+socket.setdefaulttimeout(TIMEOUT_SECONDS)
+
+digest_handler = urllib2.HTTPDigestAuthHandler()
+digest_handler.add_password(
+    realm='buildlogs',
+    uri=URL_ROOT,
+    user=username,
+    passwd=password)
+
+# This version of HTTPErrorProcessor is copied from
+# Python 2.7, and allows REST response codes (e.g.
+# "201 Created") which are treated as errors by
+# older versions.
+class HTTPErrorProcessor(urllib2.HTTPErrorProcessor):
+    def http_response(self, request, response):
+        code, msg, hdrs = response.code, response.msg, response.info()
+
+        # According to RFC 2616, "2xx" code indicates that the client's
+        # request was successfully received, understood, and accepted.
+        if not (200 <= code < 300):
+            response = self.parent.error(
+                'http', request, response, code, msg, hdrs)
+
+        return response
+
+url_opener = urllib2.build_opener(digest_handler, HTTPErrorProcessor())
+
+def url(endpoint):
+    if not endpoint.endswith('/'):
+        endpoint = '%s/' % endpoint
+
+    return '%s/%s' % (URL_ROOT.rstrip('/'), endpoint)
+
+def post(endpoint, data, headers=None):
+    data = json.dumps(data, encoding='utf-8')
+
+    headers = headers or {}
+    headers.update({'Content-Type': 'application/json; charset=utf-8'})
+
+    req = urllib2.Request(url=url(endpoint), data=data, headers=headers)
+    try:
+        response = url_opener.open(req)
+    except urllib2.URLError:
+        import traceback
+        traceback.print_exc(file=sys.stderr)
+        sys.stderr.flush()
+        # indicate that the request did not succeed
+        return None
+
+    response_headers = dict(response.info())
+
+    # eg "Content-Type: application/json; charset=utf-8"
+    content_type = response_headers.get('content-type')
+    match = re.match(r'(?P<mimetype>[^;]+).*(?:charset=(?P<charset>[^ ]+))?$', content_type)
+    if match and match.group('mimetype') == 'application/json':
+        encoding = match.group('charset') or 'utf-8'
+        return json.load(response, encoding=encoding)
+
+    return response.read()
+
+def traceback_to_stderr(func):
+    """
+    decorator which logs any exceptions encountered to stderr
+    and returns none.
+    """
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except urllib2.HTTPError, err:
+            sys.stderr.write('error: HTTP code %d\n----\n' % err.code)
+            if hasattr(err, 'hdrs'):
+                for k, v in err.hdrs.items():
+                    sys.stderr.write("%s: %s\n" % (k, v))
+                sys.stderr.write('\n')
+            sys.stderr.write(err.read())
+            sys.stderr.write('\n----\n')
+            sys.stderr.flush()
+        except:
+            sys.stderr.write('Traceback from buildlogger:\n')
+            traceback.print_exc(file=sys.stderr)
+            sys.stderr.flush()
+        return None
+    return wrapper
+
+
+@traceback_to_stderr
+def get_or_create_build(builder, buildnum, extra={}):
+    data = {'builder': builder, 'buildnum': buildnum}
+    data.update(extra)
+    response = post('build', data)
+    if response is None:
+        return None
+    return response['id']
+
+@traceback_to_stderr
+def create_test(build_id, test_filename, test_command, test_phase):
+    response = post('build/%s/test' % build_id, {
+        'test_filename': test_filename,
+        'command': test_command,
+        'phase': test_phase,
+    })
+    if response is None:
+        return None
+    return response['id']
+
+@traceback_to_stderr
+def append_test_logs(build_id, test_id, log_lines):
+    response = post('build/%s/test/%s' % (build_id, test_id), data=log_lines)
+    if response is None:
+        return False
+    return True
+
+@traceback_to_stderr
+def append_global_logs(build_id, log_lines):
+    """
+    "global" logs are for the mongod(s) started by smoke.py
+    that last the duration of a test phase -- since there
+    may be output in here that is important but spans individual
+    tests, the buildlogs webapp handles these logs specially.
+    """
+    response = post('build/%s' % build_id, data=log_lines)
+    if response is None:
+        return False
+    return True
+
+@traceback_to_stderr
+def finish_test(build_id, test_id, failed=False):
+    response = post('build/%s/test/%s' % (build_id, test_id), data=[], headers={
+        'X-Sendlogs-Test-Done': 'true',
+        'X-Sendlogs-Test-Failed': failed and 'true' or 'false',
+    })
+    if response is None:
+        return False
+    return True
+
+def run_and_echo(command):
+    """
+    this just calls the command, and returns its return code,
+    allowing stdout and stderr to work as normal. it is used
+    as a fallback when environment variables or python
+    dependencies cannot be configured, or when the logging
+    webapp is unavailable, etc
+    """
+    proc = subprocess.Popen(command)
+
+    def handle_sigterm(signum, frame):
+        try:
+            proc.send_signal(signum)
+        except AttributeError:
+            os.kill(proc.pid, signum)
+    orig_handler = signal.signal(signal.SIGTERM, handle_sigterm)
+
+    proc.wait()
+
+    signal.signal(signal.SIGTERM, orig_handler)
+    return proc.returncode
+
+class LogAppender(object):
+    def __init__(self, callback, args, send_after_lines=200, send_after_seconds=2):
+        self.callback = callback
+        self.callback_args = args
+
+        self.send_after_lines = send_after_lines
+        self.send_after_seconds = send_after_seconds
+
+        self.buf = []
+        self.retrybuf = []
+        self.last_sent = time.time()
+
+    def __call__(self, line):
+        self.buf.append((time.time(), line))
+
+        delay = time.time() - self.last_sent
+        if len(self.buf) >= self.send_after_lines or delay >= self.send_after_seconds:
+            self.submit()
+
+        # no return value is expected
+
+    def submit(self):
+        if len(self.buf) + len(self.retrybuf) == 0:
+            return True
+
+        args = list(self.callback_args)
+        args.append(list(self.buf) + self.retrybuf)
+
+        self.last_sent = time.time()
+
+        if self.callback(*args):
+            self.buf = []
+            self.retrybuf = []
+            return True
+        else:
+            self.retrybuf += self.buf
+            self.buf = []
+            return False
+
+
+def wrap_test(command):
+    """
+    call the given command, intercept its stdout and stderr,
+    and send results in batches of 100 lines or 10s to the 
+    buildlogger webapp
+    """
+
+    # get builder name and build number from environment
+    builder = os.environ.get('MONGO_BUILDER_NAME')
+    buildnum = os.environ.get('MONGO_BUILD_NUMBER')
+
+    if builder is None or buildnum is None:
+        return run_and_echo(command)
+
+    try:
+        buildnum = int(buildnum)
+    except ValueError:
+        sys.stderr.write('buildlogger: build number ("%s") was not an int\n' % buildnum)
+        sys.stderr.flush()
+        return run_and_echo(command)
+
+    # test takes some extra info
+    phase = os.environ.get('MONGO_PHASE', 'unknown')
+    test_filename = os.environ.get('MONGO_TEST_FILENAME', 'unknown')
+
+    build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_'))
+    build_info.pop('MONGO_BUILDER_NAME', None)
+    build_info.pop('MONGO_BUILD_NUMBER', None)
+    build_info.pop('MONGO_PHASE', None)
+    build_info.pop('MONGO_TEST_FILENAME', None)
+
+    build_id = get_or_create_build(builder, buildnum, extra=build_info)
+    if not build_id:
+        return run_and_echo(command)
+
+    test_id = create_test(build_id, test_filename, ' '.join(command), phase)
+    if not test_id:
+        return run_and_echo(command)
+
+    # the peculiar formatting here matches what is printed by
+    # smoke.py when starting tests
+    output_url = '%s/build/%s/test/%s/' % (URL_ROOT.rstrip('/'), build_id, test_id)
+    sys.stdout.write('                (output suppressed; see %s)\n' % output_url)
+    sys.stdout.flush()
+
+    callback = LogAppender(callback=append_test_logs, args=(build_id, test_id))
+    returncode = loop_and_callback(command, callback)
+    failed = bool(returncode != 0)
+
+    # this will append any remaining unsubmitted logs, or
+    # return True if there are none left to submit
+    tries = 5
+    while not callback.submit() and tries > 0:
+        sys.stderr.write('failed to finish sending test logs, retrying in 1s\n')
+        sys.stderr.flush()
+        time.sleep(1)
+        tries -= 1
+
+    tries = 5
+    while not finish_test(build_id, test_id, failed) and tries > 5:
+        sys.stderr.write('failed to mark test finished, retrying in 1s\n')
+        sys.stderr.flush()
+        time.sleep(1)
+        tries -= 1
+
+    return returncode
+
+def wrap_global(command):
+    """
+    call the given command, intercept its stdout and stderr,
+    and send results in batches of 100 lines or 10s to the
+    buildlogger webapp. see :func:`append_global_logs` for the
+    difference between "global" and "test" log output.
+    """
+
+    # get builder name and build number from environment
+    builder = os.environ.get('MONGO_BUILDER_NAME')
+    buildnum = os.environ.get('MONGO_BUILD_NUMBER')
+
+    if builder is None or buildnum is None:
+        return run_and_echo(command)
+
+    try:
+        buildnum = int(buildnum)
+    except ValueError:
+        sys.stderr.write('int(os.environ["MONGO_BUILD_NUMBER"]):\n')
+        sys.stderr.write(traceback.format_exc())
+        sys.stderr.flush()
+        return run_and_echo(command)
+
+    build_info = dict((k, v) for k, v in os.environ.items() if k.startswith('MONGO_'))
+    build_info.pop('MONGO_BUILDER_NAME', None)
+    build_info.pop('MONGO_BUILD_NUMBER', None)
+
+    build_id = get_or_create_build(builder, buildnum, extra=build_info)
+    if not build_id:
+        return run_and_echo(command)
+
+    callback = LogAppender(callback=append_global_logs, args=(build_id, ))
+    returncode = loop_and_callback(command, callback)
+
+    # this will append any remaining unsubmitted logs, or
+    # return True if there are none left to submit
+    tries = 5
+    while not callback.submit() and tries > 0:
+        sys.stderr.write('failed to finish sending global logs, retrying in 1s\n')
+        sys.stderr.flush()
+        time.sleep(1)
+        tries -= 1
+
+    return returncode
+
+def loop_and_callback(command, callback):
+    """
+    run the given command (a sequence of arguments, ordinarily
+    from sys.argv), and call the given callback with each line
+    of stdout or stderr encountered. after the command is finished,
+    callback is called once more with None instead of a string.
+    """
+    proc = subprocess.Popen(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+
+    def handle_sigterm(signum, frame):
+        try:
+            proc.send_signal(signum)
+        except AttributeError:
+            os.kill(proc.pid, signum)
+
+    # register a handler to delegate SIGTERM
+    # to the child process
+    orig_handler = signal.signal(signal.SIGTERM, handle_sigterm)
+
+    while proc.poll() is None:
+        try:
+            line = proc.stdout.readline().strip('\r\n')
+            line = utils.unicode_dammit(line)
+            callback(line)
+        except IOError:
+            # if the signal handler is called while
+            # we're waiting for readline() to return,
+            # don't show a traceback
+            break
+
+    # There may be additional buffered output
+    for line in proc.stdout.readlines():
+        callback(line.strip('\r\n'))
+
+    # restore the original signal handler, if any
+    signal.signal(signal.SIGTERM, orig_handler)
+    return proc.returncode
+
+
+if __name__ == '__main__':
+    # argv[0] is 'buildlogger.py'
+    del sys.argv[0]
+
+    if sys.argv[0] in ('-g', '--global'):
+        # then this is wrapping a "global" command, and should
+        # submit global logs to the build, not test logs to a
+        # test within the build
+        del sys.argv[0]
+        wrapper = wrap_global
+
+    else:
+        wrapper = wrap_test
+
+    # if we are missing credentials or the json module, then
+    # we can't use buildlogger; so just echo output, but also
+    # log why we can't work.
+    if json is None:
+        sys.stderr.write('buildlogger: could not import a json module\n')
+        sys.stderr.flush()
+        wrapper = run_and_echo
+
+    elif username is None or password is None:
+        sys.stderr.write('buildlogger: could not find or import %s for authentication\n' % credentials_file)
+        sys.stderr.flush()
+        wrapper = run_and_echo
+
+    # otherwise wrap a test command as normal; the
+    # wrapper functions return the return code of
+    # the wrapped command, so that should be our
+    # exit code as well.
+    sys.exit(wrapper(sys.argv))
+
diff --git a/buildscripts/cleanbb.py b/buildscripts/cleanbb.py
index bfeafd2d8e9..085eee223fb 100644
--- a/buildscripts/cleanbb.py
+++ b/buildscripts/cleanbb.py
@@ -14,6 +14,16 @@ if os.path.basename(cwd) == 'buildscripts':
 print( "cwd [" + cwd + "]" )
 
 def shouldKill( c ):
+    
+    if "smoke.py" in c:
+        return False
+
+    if "emr.py" in c:
+        return False
+
+    if "java" in c:
+        return False
+
     if c.find( cwd ) >= 0:
         return True
 
@@ -25,7 +35,10 @@ def shouldKill( c ):
 def killprocs( signal="" ):
 
     killed = 0
-        
+
+    if sys.platform == 'win32':
+        return killed
+
     l = utils.getprocesslist()
     print( "num procs:" + str( len( l ) ) )
     if len(l) == 0:
diff --git a/buildscripts/confluence_export.py b/buildscripts/confluence_export.py
deleted file mode 100644
index 29cdde60c7f..00000000000
--- a/buildscripts/confluence_export.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#! /usr/bin/env python
-
-# Export the contents on confluence
-#
-# Dependencies:
-#   - suds
-#
-# User: soap, Password: soap
-from __future__ import with_statement
-import cookielib
-import datetime
-import os
-import shutil
-import subprocess
-import sys
-import urllib2
-sys.path[0:0] = [""]
-
-import simples3
-from suds.client import Client
-
-import settings
-
-HTML_URI = "http://mongodb.onconfluence.com/rpc/soap-axis/confluenceservice-v1?wsdl"
-PDF_URI = "http://www.mongodb.org/rpc/soap-axis/pdfexport?wsdl"
-USERNAME = "soap"
-PASSWORD = "soap"
-AUTH_URI = "http://www.mongodb.org/login.action?os_authType=basic"
-TMP_DIR = "confluence-tmp"
-TMP_FILE = "confluence-tmp.zip"
-
-
-def export_html_and_get_uri():
-    client = Client(HTML_URI)
-    auth = client.service.login(USERNAME, PASSWORD)
-    return client.service.exportSpace(auth, "DOCS", "TYPE_HTML")
-
-
-def export_pdf_and_get_uri():
-    client = Client(PDF_URI)
-    auth = client.service.login(USERNAME, PASSWORD)
-    return client.service.exportSpace(auth, "DOCS")
-
-
-def login_and_download(docs):
-    cookie_jar = cookielib.CookieJar()
-    cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)
-    password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
-    password_manager.add_password(None, AUTH_URI, USERNAME, PASSWORD)
-    auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
-    urllib2.build_opener(cookie_handler, auth_handler).open(AUTH_URI)
-    return urllib2.build_opener(cookie_handler).open(docs)
-
-
-def extract_to_dir(data, dir):
-    with open(TMP_FILE, "w") as f:
-        f.write(data.read())
-    data.close()
-    # This is all really annoying but zipfile doesn't do extraction on 2.5
-    subprocess.call(["unzip", "-d", dir, TMP_FILE])
-    os.unlink(TMP_FILE)
-
-
-def rmdir(dir):
-    try:
-        shutil.rmtree(dir)
-    except:
-        pass
-
-
-def overwrite(src, dest):
-    target = "%s/DOCS-%s/" % (dest, datetime.date.today())
-    current = "%s/current" % dest
-    rmdir(target)
-    shutil.copytree(src, target)
-    try:
-        os.unlink(current)
-    except:
-        pass
-    os.symlink(os.path.abspath(target), os.path.abspath(current))
-
-
-def write_to_s3(pdf):
-    s3 = simples3.S3Bucket(settings.bucket, settings.id, settings.key)
-    name = "docs/mongodb-docs-%s.pdf" % datetime.date.today()
-    s3.put(name, pdf, acl="public-read")
-
-
-def main(dir):
-    # HTML
-    rmdir(TMP_DIR)
-    extract_to_dir(login_and_download(export_html_and_get_uri()), TMP_DIR)
-    overwrite("%s/DOCS/" % TMP_DIR, dir)
-
-    # PDF
-    write_to_s3(login_and_download(export_pdf_and_get_uri()).read())
-
-
-if __name__ == "__main__":
-    try:
-        main(sys.argv[1])
-    except IndexError:
-        print "pass outdir as first arg"
diff --git a/buildscripts/distmirror.py b/buildscripts/distmirror.py
deleted file mode 100644
index 7af1a89f7dc..00000000000
--- a/buildscripts/distmirror.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python
-
-# Download mongodb stuff (at present builds, sources, docs, but not
-# drivers).
-
-# Usage: <progname> [directory] # directory defaults to cwd.
-
-# FIXME: this script is fairly sloppy.
-import sys
-import os
-import urllib2
-import time
-import hashlib
-import warnings
-
-written_files = []
-def get(url, filename):
-    # A little safety check.
-    if filename in written_files:
-        raise Exception('not overwriting file %s (already written in this session)' % filename)
-    else:
-        written_files.append(filename)
-    print "downloading %s to %s" % (url, filename)    
-    open(filename, 'w').write(urllib2.urlopen(url).read())
-
-
-def checkmd5(md5str, filename):
-    m = hashlib.md5()
-    m.update(open(filename, 'rb').read())
-    d = m.hexdigest()
-    if d != md5str:
-        warnings.warn("md5sum mismatch for file %s: wanted %s; got %s" % (filename, md5str, d))
-    
-osarches=(("osx", ("i386", "i386-tiger", "x86_64"), ("tgz", )),
-          ("linux", ("i686", "x86_64"), ("tgz", )),
-          ("win32", ("i386", "x86_64"), ("zip", )),
-          ("sunos5", ("i86pc", "x86_64"), ("tgz", )),
-          ("src", ("src", ), ("tar.gz", "zip")), )
-
-# KLUDGE: this will need constant editing.
-versions = ("1.4.2", "1.5.1", "latest")
-
-url_format = "http://downloads.mongodb.org/%s/mongodb-%s-%s.%s"
-filename_format = "mongodb-%s-%s.%s"
-
-def core_server():
-    for version in versions:
-        for (os, architectures, archives) in osarches:
-            for architecture in architectures:
-                for archive in archives:
-                    osarch = os + '-' + architecture if architecture != 'src' else 'src'
-                    # ugh.
-                    if architecture == 'src' and version == 'latest':
-                        if archive == 'tar.gz':
-                            archive2 = 'tarball'
-                        elif archive == 'zip':
-                            archive2 == 'zipball'
-                        url = "http://github.com/mongodb/mongo/"+archive2+"/master"
-                        version2 = "master"
-                    else:
-                        version2 = version if architecture != 'src' else 'r'+version
-                        url = url_format % (os, osarch, version2, archive)
-                    # ugh ugh
-                    md5url = url+'.md5' if architecture != 'src' else None
-                    filename = filename_format % (osarch, version2, archive)
-                    get(url, filename)
-                    if md5url:
-                        print "fetching md5 url " + md5url
-                        md5str = urllib2.urlopen(md5url).read()
-                        checkmd5(md5str, filename)
-
-def drivers():
-    # Drivers... FIXME: drivers.
-    driver_url_format = "http://github.com/mongodb/mongo-%s-driver/%s/%s"
-    driver_filename_format = "mongo-%s-driver-%s.%s"
-    drivers=(("python", ("1.6", "master"), ("zipball", "tarball"), None),
-             ("ruby", ("0.20", "master"), ("zipball", "tarball"), None),
-             ("c", ("v0.1", "master"), ("zipball", "tarball"), None),
-             # FIXME: PHP, Java, and Csharp also have zips and jars of
-             # precompiled relesaes.
-             ("php", ("1.0.6", "master"), ("zipball", "tarball"), None),
-             ("java", ("r1.4", "r2.0rc1", "master"), ("zipball", "tarball"), None),
-             # And Csharp is in a different github place, too.
-             ("csharp", ("0.82.2", "master"), ("zipball", "tarball"),
-              "http://github.com/samus/mongodb-%s/%s/%s"),
-             )
-
-    for (lang, releases, archives, url_format) in drivers:
-        for release in releases:
-            for archive in archives:
-                url = (url_format if url_format else driver_url_format) % (lang, archive, release) 
-                if archive == 'zipball':
-                    extension = 'zip'
-                elif archive == 'tarball':
-                    extension = 'tgz'
-                else:
-                    raise Exception('unknown archive format %s' % archive)
-                filename = driver_filename_format % (lang, release, extension)
-                get(url, filename)
-        # ugh ugh ugh
-        if lang == 'csharp' and release != 'master':
-            url = 'http://github.com/downloads/samus/mongodb-csharp/MongoDBDriver-Release-%.zip' % (release)
-            filename = 'MongoDBDriver-Release-%.zip' % (release)
-            get(url, filename)
-        if lang == 'java' and release != 'master':
-            get('http://github.com/downloads/mongodb/mongo-java-driver/mongo-%s.jar' % (release), 'mongo-%s.jar' % (release))
-        # I have no idea what's going on with the PHP zipfiles.
-        if lang == 'php' and release == '1.0.6':
-            get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.2-osx.zip', 'mongo-1.0.6-php5.2-osx.zip')
-            get('http://github.com/downloads/mongodb/mongo-php-driver/mongo-1.0.6-php5.3-osx.zip', 'mongo-1.0.6-php5.3-osx.zip')
-
-def docs():
-    # FIXME: in principle, the doc PDFs could be out of date.
-    docs_url = time.strftime("http://downloads.mongodb.org/docs/mongodb-docs-%Y-%m-%d.pdf")
-    docs_filename = time.strftime("mongodb-docs-%Y-%m-%d.pdf")
-    get(docs_url, docs_filename)
-
-def extras():
-    # Extras
-    extras = ("http://media.mongodb.org/zips.json", )
-    for extra in extras:
-        if extra.rfind('/') > -1:
-            filename = extra[extra.rfind('/')+1:]
-        else: 
-            raise Exception('URL %s lacks a slash?' % extra)
-        get(extra, filename)
-
-if len(sys.argv) > 1: 
-    dir=sys.argv[1]
-    os.makedirs(dir)
-    os.chdir(dir)
-
-print """NOTE: the md5sums for all the -latest tarballs are out of
-date.  You will probably see warnings as this script runs.  (If you
-don't, feel free to delete this note.)"""
-core_server()
-drivers()
-docs()
-extras()
diff --git a/buildscripts/emr/FileLock.java b/buildscripts/emr/FileLock.java
new file mode 100644
index 00000000000..52ed4c083b4
--- /dev/null
+++ b/buildscripts/emr/FileLock.java
@@ -0,0 +1,107 @@
+// FileLock.java
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
+
+/**
+ * "locks" a resource by using the file system as storage
+ * file has 1 line
+ * <incarnation> <last ping time in millis>
+ */
+public class FileLock {
+    
+    public FileLock( String logicalName ) 
+        throws IOException {
+
+        _file = new File( "/tmp/java-fileLock-" + logicalName );
+        _incarnation = "xxx" + Math.random() + "yyy";
+
+        if ( ! _file.exists() ) {
+            FileOutputStream fout = new FileOutputStream( _file );
+            fout.write( "\n".getBytes() );
+            fout.close();
+        }
+
+    }
+
+    /**
+     * takes lock
+     * if someone else has it, blocks until the other one finishes
+     */
+    public void lock() 
+        throws IOException {
+        if ( _lock != null )
+            throw new IllegalStateException( "can't lock when you're locked" );        
+        
+        try {
+            _semaphore.acquire();
+        }
+        catch ( InterruptedException ie ) {
+            throw new RuntimeException( "sad" , ie );
+        }
+
+        _raf = new RandomAccessFile( _file , "rw" );
+        _lock = _raf.getChannel().lock();
+    }
+
+    public void unlock() 
+        throws IOException {
+
+        if ( _lock == null )
+            throw new IllegalStateException( "can't unlock when you're not locked" );
+
+        _lock.release();
+        _semaphore.release();
+        
+        _locked = false;
+    }
+
+    final File _file;
+    final String _incarnation;
+    
+    private RandomAccessFile _raf;
+    private java.nio.channels.FileLock _lock;
+
+    private boolean _locked;
+
+    private static Semaphore _semaphore = new Semaphore(1);
+
+
+    public static void main( final String[] args ) 
+        throws Exception {
+
+        List<Thread> threads = new ArrayList<Thread>();
+
+        for ( int i=0; i<3; i++ ) {
+
+            threads.add( new Thread() {
+                    public void run() {
+                        try {
+                            FileLock lock = new FileLock( args[0] );
+                            
+                            long start = System.currentTimeMillis();
+                            
+                            lock.lock();
+                            System.out.println( "time to lock:\t" + (System.currentTimeMillis()-start) );
+                            Thread.sleep( Integer.parseInt( args[1] ) );
+                            lock.unlock();
+                            System.out.println( "total time:\t" + (System.currentTimeMillis()-start) );
+                        }
+                        catch ( Exception e ) {
+                            e.printStackTrace();
+                        }
+                    }
+                } );
+        }
+        
+        for ( Thread t : threads ) {
+            t.start();
+        }
+
+        for ( Thread t : threads ) {
+            t.join();
+        }
+        
+    }
+}
diff --git a/buildscripts/emr/IOUtil.java b/buildscripts/emr/IOUtil.java
new file mode 100644
index 00000000000..8ee105c5155
--- /dev/null
+++ b/buildscripts/emr/IOUtil.java
@@ -0,0 +1,156 @@
+// IOUtil.java
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+public class IOUtil {
+
+    public static String urlFileName( String url ) {
+        int idx = url.lastIndexOf( "/" );
+        if ( idx < 0 )
+            return url;
+        return url.substring( idx + 1 );
+    }
+
+    public static long pipe( InputStream in , OutputStream out )
+        throws IOException {
+
+        long bytes = 0;
+
+        byte[] buf = new byte[2048];
+        
+        while ( true ) {
+            int x = in.read( buf );
+            if ( x < 0 ) 
+                break;
+
+            bytes += x;
+            out.write( buf , 0 , x );
+        }
+
+        return bytes;
+    }
+
+    public static class PipingThread extends Thread {
+        public PipingThread( InputStream in , OutputStream out ) {
+            _in = in;
+            _out = out;
+
+            _wrote = 0;
+        }
+        
+        public void run() {
+            try {
+                _wrote = pipe( _in , _out );
+            }
+            catch ( IOException ioe ) {
+                ioe.printStackTrace();
+                _wrote = -1;
+            }
+        }
+        
+        public long wrote() {
+            return _wrote;
+        }
+
+        long _wrote;
+        
+        final InputStream _in;
+        final OutputStream _out;
+    }
+
+    public static String readStringFully( InputStream in ) 
+        throws IOException {
+        
+        ByteArrayOutputStream bout = new ByteArrayOutputStream();
+        pipe( in , bout );
+        return new String( bout.toByteArray() , "UTF8" );
+        
+    }
+
+    public static Map<String,Object> readPythonSettings( File file ) 
+        throws IOException {
+        
+        String all = readStringFully( new FileInputStream( file ) );
+
+        Map<String,Object> map = new TreeMap<String,Object>();
+        
+        for ( String line : all.split( "\n" ) ) {
+            line = line.trim();
+            if ( line.length() == 0 )
+                continue;
+            
+            String[] pcs = line.split( "=" );
+            if ( pcs.length != 2 )
+                continue;
+            
+            String name = pcs[0].trim();
+            String value = pcs[1].trim();
+            
+            if ( value.startsWith( "\"" ) ) {
+                map.put( name , value.substring( 1 , value.length() - 1 ) );
+            }
+            else {
+                map.put( name , Long.parseLong( value ) );
+            }
+            
+        }
+        
+        return map;
+    }
+
+    public static String[] runCommand( String cmd , File dir )
+        throws IOException {
+        
+        Process p = Runtime.getRuntime().exec( cmd.split( " +" ) , new String[]{} , dir );
+        String[] results = new String[]{ IOUtil.readStringFully( p.getInputStream() ) , IOUtil.readStringFully( p.getErrorStream() ) };
+        try {
+            if ( p.waitFor() != 0 )
+                throw new RuntimeException( "command failed [" + cmd + "]\n" + results[0] + "\n" + results[1] );
+        }
+        catch ( InterruptedException ie ) {
+            throw new RuntimeException( "uh oh" );
+        }
+        return results;
+    }
+
+
+    public static void download( String http , File localDir ) 
+        throws IOException {
+        
+        File f = localDir;
+        f.mkdirs();
+        
+        f = new File( f.toString() + File.separator + urlFileName( http ) );
+        
+        System.out.println( "downloading\n\t" + http + "\n\t" + f );
+        
+        if ( f.exists() ) {
+            System.out.println( "\t already exists" );
+            return;
+        }
+        
+        URL url = new URL( http );
+        
+        InputStream in = url.openConnection().getInputStream();
+        OutputStream out = new FileOutputStream( f );
+        
+        pipe( in , out );
+        
+        out.close();
+        in.close();
+        
+    }
+
+    public static void main( String[] args )
+        throws Exception {
+        
+        
+        byte[] data = new byte[]{ 'e' , 'r' , 'h' , 0 };
+        System.out.write( data );
+        System.out.println( "yo" );
+        
+    }
+
+}
diff --git a/buildscripts/emr/MANIFEST.MF b/buildscripts/emr/MANIFEST.MF
new file mode 100644
index 00000000000..4a5b3f96691
--- /dev/null
+++ b/buildscripts/emr/MANIFEST.MF
@@ -0,0 +1,2 @@
+Manifest-Version: 1.0
+Main-Class: emr
diff --git a/buildscripts/emr/emr.java b/buildscripts/emr/emr.java
new file mode 100644
index 00000000000..0f01d6d0e2d
--- /dev/null
+++ b/buildscripts/emr/emr.java
@@ -0,0 +1,380 @@
+// emr.java
+
+import java.io.*;
+import java.util.*;
+import java.net.*;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.mapred.*;
+import org.apache.hadoop.fs.*;
+
+
+public class emr {
+
+    static class MongoSuite {
+        String mongo;
+        String code;
+        String workingDir;
+
+        String suite;
+
+        void copy( MongoSuite c ) {
+            mongo = c.mongo;
+            code = c.code;
+            workingDir = c.workingDir;
+            
+            suite = c.suite;
+            
+        }
+
+        void downloadTo( File localDir ) 
+            throws IOException {
+            IOUtil.download( mongo , localDir );
+            IOUtil.download( code , localDir );
+        }
+
+        boolean runTest() 
+            throws IOException {
+
+            // mkdir
+            File dir = new File( workingDir , suite );
+            dir.mkdirs();
+            
+            // download
+            System.out.println( "going to download" );
+            downloadTo( dir );
+            
+            
+            // explode
+            System.out.println( "going to explode" );
+            IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( code ) , dir );
+            String[] res = IOUtil.runCommand( "tar zxvf " + IOUtil.urlFileName( mongo ) , dir );
+            for ( String x : res[0].split( "\n" ) ) {
+                if ( x.indexOf( "/bin/" ) < 0 )
+                    continue;
+                File f = new File( dir.toString() , x );
+                if ( ! f.renameTo( new File( dir , IOUtil.urlFileName( x ) ) ) )
+                    throw new RuntimeException( "rename failed" );
+            }
+            
+            List<String> cmd = new ArrayList<String>();
+            cmd.add( "/usr/bin/python" );
+            cmd.add( "buildscripts/smoke.py" );
+            
+            File log_config = new File( dir , "log_config.py" );
+            System.out.println( "log_config: " + log_config.exists() );
+            if ( log_config.exists() ) {
+
+                java.util.Map<String,Object> properties = IOUtil.readPythonSettings( log_config );
+
+                cmd.add( "--buildlogger-builder" );
+                cmd.add( properties.get( "name" ).toString() );
+                
+                cmd.add( "--buildlogger-buildnum" );
+                cmd.add( properties.get( "number" ).toString() );
+                
+                cmd.add( "--buildlogger-credentials" );
+                cmd.add( "log_config.py" );
+                
+                cmd.add( "--buildlogger-phase" );
+                {
+                    int idx = suite.lastIndexOf( "/" );
+                    if ( idx < 0 ) 
+                        cmd.add( suite );
+                    else
+                        cmd.add( suite.substring( 0 , idx ) );
+                }
+
+            }
+
+            cmd.add( suite );
+            
+            System.out.println( cmd );
+
+            Process p = Runtime.getRuntime().exec( cmd.toArray( new String[cmd.size()] ) , new String[]{} , dir );
+
+            List<Thread> threads = new ArrayList<Thread>();
+            threads.add( new IOUtil.PipingThread( p.getInputStream() , System.out ) );
+            threads.add( new IOUtil.PipingThread( p.getErrorStream() , System.out ) );
+
+            for ( Thread t : threads ) 
+                t.start();
+
+            try {
+                for ( Thread t : threads ) {
+                    t.join();
+                }
+                int rc = p.waitFor();
+                return rc == 0;
+            }
+            catch ( InterruptedException ie ) {
+                ie.printStackTrace();
+                throw new RuntimeException( "sad" , ie );
+            }
+            
+        }
+
+        public void readFields( DataInput in ) 
+            throws IOException {
+            mongo = in.readUTF();
+            code = in.readUTF();
+            workingDir = in.readUTF();
+            
+            suite = in.readUTF();
+        }
+
+        public void write( final DataOutput out ) 
+            throws IOException {
+            out.writeUTF( mongo );            
+            out.writeUTF( code );
+            out.writeUTF( workingDir );
+
+            out.writeUTF( suite );
+        }
+
+        public String toString() {
+            return "mongo: " + mongo + " code: " + code + " suite: " + suite + " workingDir: " + workingDir;
+        }
+    }
+
+    public static class Map implements Mapper<Text, MongoSuite, Text, IntWritable> {
+
+        public void map( Text key, MongoSuite value, OutputCollector<Text,IntWritable> output, Reporter reporter ) 
+            throws IOException {
+            
+            FileLock lock = new FileLock( "mapper" );
+            try {
+                lock.lock();
+
+                System.out.println( "key: " + key );
+                System.out.println( "value: " + value );
+                
+                long start = System.currentTimeMillis();
+                boolean passed = value.runTest();
+                long end = System.currentTimeMillis();
+                
+                output.collect( new Text( passed ? "passed" : "failed" ) , new IntWritable( 1 ) );
+                output.collect( new Text( key.toString() + "-time-seconds" ) , new IntWritable( (int)((end-start)/(1000)) ) );
+                output.collect( new Text( key.toString() + "-passed" ) , new IntWritable( passed ? 1 : 0 ) );
+                
+                String ip = IOUtil.readStringFully( new URL( "http://myip.10gen.com/" ).openConnection().getInputStream() );
+                ip = ip.substring( ip.indexOf( ":" ) + 1 ).trim();
+                output.collect( new Text( ip ) , new IntWritable(1) );
+            }
+            catch ( RuntimeException re ) {
+                re.printStackTrace();
+                throw re;
+            }
+            catch ( IOException ioe ) {
+                ioe.printStackTrace();
+                throw ioe;
+            }
+            finally {
+                lock.unlock();
+            }
+            
+        }
+
+        public void configure(JobConf job) {}
+        public void close(){}
+    }
+
+    public static class Reduce implements Reducer<Text, IntWritable, Text, IntWritable> {
+
+        public void reduce( Text key, Iterator<IntWritable> values, OutputCollector<Text,IntWritable> output , Reporter reporter ) 
+            throws IOException {
+
+            int sum = 0;
+            while ( values.hasNext() ) {
+                sum += values.next().get();
+            }
+            output.collect( key , new IntWritable( sum ) );
+        }
+
+        public void configure(JobConf job) {}
+        public void close(){}
+    }
+
+    public static class MySplit implements InputSplit , Writable {
+        
+        public MySplit(){
+        }
+        
+        MySplit( MongoSuite config , int length ) {
+            _config = config;
+            _length = length;
+        }
+
+        public long getLength() {
+            return _length;
+        }
+        
+        public String[] getLocations() {
+            return new String[0];
+        }
+
+        public void readFields( DataInput in ) 
+            throws IOException {
+            _config = new MongoSuite();
+            _config.readFields( in );
+            _length = in.readInt();
+        }
+
+        public void write( final DataOutput out ) 
+            throws IOException {
+            _config.write( out );
+            out.writeInt( _length );
+        }
+
+        MongoSuite _config;
+        int _length;
+    }
+
+    public static class InputMagic implements InputFormat<Text,MongoSuite> {
+
+        public RecordReader<Text,MongoSuite> getRecordReader( InputSplit split, JobConf job , Reporter reporter ){
+            final MySplit s = (MySplit)split;
+            return new RecordReader<Text,MongoSuite>() {
+                
+                public void close(){}
+                
+                public Text createKey() {
+                    return new Text();
+                }
+                
+                public MongoSuite createValue() {
+                    return new MongoSuite();
+                }
+
+                public long getPos() {
+                    return _seen ? 1 : 0;
+                }
+
+                public float getProgress() {
+                    return getPos();
+                }
+                
+                public boolean next( Text key , MongoSuite value ) {
+                    key.set( s._config.suite );
+                    value.copy( s._config );
+
+                    
+                    boolean x = _seen;
+                    _seen = true;
+                    return !x;
+                }
+                
+                boolean _seen = false;
+            };
+        }
+        
+        public InputSplit[] getSplits( JobConf job , int numSplits ){
+            String[] pcs = job.get( "suites" ).split(",");
+            InputSplit[] splits = new InputSplit[pcs.length];
+            for ( int i=0; i<splits.length; i++ ) {
+                MongoSuite c = new MongoSuite();
+                c.suite = pcs[i];
+                
+                c.mongo = job.get( "mongo" );
+                c.code = job.get( "code" );
+                c.workingDir = job.get( "workingDir" );
+
+                splits[i] = new MySplit( c , 100 /* XXX */);
+            }
+            return splits;
+        }
+        
+        public void validateInput(JobConf job){}
+
+        
+    }
+
+    /**
+     * args
+     *   mongo tgz
+     *   code tgz
+     *   output path
+     *   tests to run ?
+     */
+
+    public static void main( String[] args ) throws Exception{
+
+        JobConf conf = new JobConf();
+        conf.setJarByClass(emr.class);
+        
+        String workingDir = "/data/db/emr/";
+        
+
+        // parse args
+
+        int pos = 0;
+        for ( ; pos < args.length; pos++ ) {
+            if ( ! args[pos].startsWith( "--" ) )
+                break;
+            
+            String arg = args[pos].substring(2);
+            if ( arg.equals( "workingDir" ) ) {
+                workingDir = args[++pos];
+            }
+            else {
+                System.err.println( "unknown arg: " + arg );
+                throw new RuntimeException( "unknown arg: " + arg );
+            }
+        }
+        
+        String mongo = args[pos++];
+        String code = args[pos++];
+        String output = args[pos++];
+        
+        String suites = "";
+        for ( ; pos < args.length; pos++ ) {
+            if ( suites.length() > 0 )
+                suites += ",";
+            suites += args[pos];
+        }
+        
+        if ( suites.length() == 0 )
+            throw new RuntimeException( "no suites" );
+        
+        System.out.println( "workingDir:\t" + workingDir );
+        System.out.println( "mongo:\t" + mongo );
+        System.out.println( "code:\t " + code );
+        System.out.println( "output\t: " + output );
+        System.out.println( "suites\t: " + suites );
+
+        if ( false ) {
+            MongoSuite s = new MongoSuite();
+            s.mongo = mongo;
+            s.code = code;
+            s.workingDir = workingDir;
+            s.suite = suites;
+            s.runTest();
+            return;
+        }
+
+        // main hadoop set
+        conf.set( "mongo" , mongo );
+        conf.set( "code" , code );
+        conf.set( "workingDir" , workingDir );
+        conf.set( "suites" , suites );
+
+        conf.set( "mapred.map.tasks" , "1" );
+        conf.setLong( "mapred.task.timeout" , 4 * 3600 * 1000 /* 4  hours */);
+
+        conf.setOutputKeyClass(Text.class);
+        conf.setOutputValueClass(IntWritable.class);
+        
+        conf.setMapperClass(Map.class);
+        conf.setReducerClass(Reduce.class);
+        
+        conf.setInputFormat(InputMagic.class);
+        conf.setOutputFormat(TextOutputFormat.class);
+        
+        FileOutputFormat.setOutputPath(conf, new Path(output) );
+
+        //  actually run
+
+        JobClient.runJob( conf );
+    }
+}
diff --git a/buildscripts/emr/emr.py b/buildscripts/emr/emr.py
new file mode 100644
index 00000000000..e060779cef1
--- /dev/null
+++ b/buildscripts/emr/emr.py
@@ -0,0 +1,385 @@
+
+import os
+import sys
+import shutil
+import datetime
+import time
+import subprocess
+import urllib
+import urllib2
+import json
+import pprint
+
+import boto
+import simples3
+
+import pymongo
+
+def findSettingsSetup():
+    sys.path.append( "./" )
+    sys.path.append( "../" )
+    sys.path.append( "../../" )
+    sys.path.append( "../../../" )
+
+findSettingsSetup()
+import settings
+import buildscripts.utils as utils
+import buildscripts.smoke as smoke
+
+bucket = simples3.S3Bucket( settings.emr_bucket , settings.emr_id , settings.emr_key )
+
+def _get_status():
+    
+    def gh( cmds ):
+        txt = ""
+        for cmd in cmds:
+            res = utils.execsys( "git " + cmd )
+            txt = txt + res[0] + res[1]
+        return utils.md5string( txt )
+
+    return "%s-%s" % ( utils.execsys( "git describe" )[0].strip(),  gh( [ "diff" , "status" ] ) )
+
+def _get_most_recent_tgz( prefix ):
+    # this is icky, but works for now
+    all = []
+    for x in os.listdir( "." ):
+        if not x.startswith( prefix ) or not x.endswith( ".tgz" ):
+            continue
+        all.append( ( x , os.stat(x).st_mtime ) )
+
+    if len(all) == 0:
+        raise Exception( "can't find file with prefix: " + prefix )
+
+    all.sort( lambda x,y: int(y[1] - x[1]) )
+
+    return all[0][0]
+
+def get_build_info():
+    return ( os.environ.get('MONGO_BUILDER_NAME') , os.environ.get('MONGO_BUILD_NUMBER') )
+
+def make_tarball():
+
+    m = _get_most_recent_tgz( "mongodb-" )
+    
+    c = "test-code-emr.tgz"    
+    tar = "tar zcf %s src jstests buildscripts" % c
+
+    log_config = "log_config.py"
+    if os.path.exists( log_config ):
+        os.unlink( log_config )
+
+    credentials = do_credentials()
+    if credentials:
+
+        builder , buildnum = get_build_info()
+        
+        if builder and buildnum:
+
+            file = open( log_config , "wb" )
+            file.write( 'username="%s"\npassword="%s"\n' % credentials )
+            file.write( 'name="%s"\nnumber=%s\n'% ( builder , buildnum ) )
+
+            file.close()
+            
+            tar = tar + " " + log_config
+
+    utils.execsys( tar )
+    return ( m , c )
+
+def _put_ine( bucket , local , remote ):
+    print( "going to put\n\t%s\n\thttp://%s.s3.amazonaws.com/%s" % ( local , settings.emr_bucket , remote ) )
+
+    for x in bucket.listdir( prefix=remote ):
+        print( "\talready existed" )
+        return remote
+
+    bucket.put( remote , open( local , "rb" ).read() , acl="public-read" )
+    return remote
+
+def build_jar():
+    root = "build/emrjar"
+    src = "buildscripts/emr"
+
+    if os.path.exists( root ):
+        shutil.rmtree( root )
+    os.makedirs( root )
+
+    for x in os.listdir( src ):
+        if not x.endswith( ".java" ):
+            continue
+        shutil.copyfile( src + "/" + x , root + "/" + x )
+    shutil.copyfile( src + "/MANIFEST.MF" , root + "/MANIFEST.FM" )
+    
+    classpath = os.listdir( src + "/lib" )
+    for x in classpath:
+        shutil.copyfile( src + "/lib/" + x , root + "/" + x )
+    classpath.append( "." )
+    classpath = ":".join(classpath)
+
+    for x in os.listdir( root ):
+        if x.endswith( ".java" ):
+            if subprocess.call( [ "javac" , "-cp" , classpath , x ] , cwd=root) != 0:
+                raise Exception( "compiled failed" )
+
+    args = [ "jar" , "-cfm" , "emr.jar" , "MANIFEST.FM" ]
+    for x in os.listdir( root ):
+        if x.endswith( ".class" ):
+            args.append( x )
+    subprocess.call( args , cwd=root )
+    
+    shutil.copyfile( root + "/emr.jar" , "emr.jar" )
+
+    return "emr.jar"
+
+def push():
+    mongo , test_code = make_tarball()
+    print( mongo )
+    print( test_code )
+
+    root = "emr/%s/%s" % ( datetime.date.today().strftime("%Y-%m-%d") , os.uname()[0].lower() )
+    
+    def make_long_name(local,hash):
+        pcs = local.rpartition( "." )
+        h = _get_status()
+        if hash:
+            h = utils.md5sum( local )
+        return "%s/%s-%s.%s" % ( root , pcs[0] , h , pcs[2] )
+
+    mongo = _put_ine( bucket , mongo , make_long_name( mongo , False ) )
+    test_code = _put_ine( bucket , test_code , make_long_name( test_code , True ) )
+    
+    jar = build_jar()
+    jar = _put_ine( bucket , jar , make_long_name( jar , False ) )
+
+    setup = "buildscripts/emr/emrnodesetup.sh"
+    setup = _put_ine( bucket , setup , make_long_name( setup , True ) )
+
+    return mongo , test_code , jar , setup
+
+def run_tests( things , tests ):
+    if len(tests) == 0:
+        raise Exception( "no tests" )
+    oldNum = len(tests)
+    tests = fix_suites( tests )
+    print( "tests expanded from %d to %d" % ( oldNum , len(tests) ) )
+    
+    print( "things:%s\ntests:%s\n" % ( things , tests ) )
+
+    emr = boto.connect_emr( settings.emr_id , settings.emr_key )
+
+    def http(path):
+        return "http://%s.s3.amazonaws.com/%s" % ( settings.emr_bucket , path )
+    
+    run_s3_path = "emr/%s/%s/%s/" % ( os.getenv( "USER" ) , 
+                                      os.getenv( "HOST" ) , 
+                                      datetime.datetime.today().strftime( "%Y%m%d-%H%M" ) )
+
+    run_s3_root = "s3n://%s/%s/" % ( settings.emr_bucket , run_s3_path )
+
+    out = run_s3_root + "out"
+    logs = run_s3_root + "logs"
+
+    jar="s3n://%s/%s" % ( settings.emr_bucket , things[2] )
+    step_args=[ http(things[0]) , http(things[1]) , out , ",".join(tests) ]
+    
+    step = boto.emr.step.JarStep( "emr main" , jar=jar,step_args=step_args )
+    print( "jar:%s\nargs:%s" % ( jar , step_args ) )
+
+    setup = boto.emr.BootstrapAction( "setup" , "s3n://%s/%s" % ( settings.emr_bucket , things[3] ) , []  )
+
+    jobid = emr.run_jobflow( name = "Mongo EMR for %s from %s" % ( os.getenv( "USER" ) , os.getenv( "HOST" ) ) ,
+                             ec2_keyname = "emr1" , 
+                             slave_instance_type = "m1.large" ,
+                             ami_version = "latest" ,
+                             num_instances=5 ,
+                             log_uri = logs ,
+                             bootstrap_actions = [ setup ] , 
+                             steps = [ step ] )
+
+    
+    print( "%s jobid: %s" % ( datetime.datetime.today() , jobid ) )
+
+    while ( True ):
+        flow = emr.describe_jobflow( jobid )
+        print( "%s status: %s" % ( datetime.datetime.today() , flow.state ) )
+        if flow.state == "COMPLETED" or flow.state == "FAILED":
+            break
+        time.sleep(30)
+
+    syncdir = "build/emrout/" + jobid + "/"
+    sync_s3( run_s3_path , syncdir )
+    
+    final_out = "build/emrout/" + jobid + "/" 
+    
+    print("output in: " + final_out )
+    do_output( final_out )
+
+def sync_s3( remote_dir , local_dir ):
+    for x in bucket.listdir( remote_dir ):
+        out = local_dir + "/" + x[0]
+        
+        if os.path.exists( out ) and x[2].find( utils.md5sum( out ) ) >= 0:
+            continue
+
+        dir = out.rpartition( "/" )[0]
+        if not os.path.exists( dir ):
+            os.makedirs( dir )
+            
+        thing = bucket.get( x[0] )
+        open( out , "wb" ).write( thing.read() )
+
+def fix_suites( suites ):
+    fixed = []
+    for name,x in smoke.expand_suites( suites , False ):
+        idx = name.find( "/jstests" )
+        if idx >= 0:
+            name = name[idx+1:]
+        fixed.append( name )
+    return fixed
+
+def do_credentials():
+    root = "buildbot.tac"
+    
+    while len(root) < 40 :
+        if os.path.exists( root ):
+            break
+        root = "../" + root
+        
+    if not os.path.exists( root ):
+        return None
+    
+    credentials = {}
+    execfile(root, credentials, credentials)
+    
+    if "slavename" not in credentials:
+        return None
+
+    if "passwd" not in credentials:
+        return None
+    
+    return ( credentials["slavename"] , credentials["passwd"] )
+
+
+def do_output( dir ):
+
+    def go_down( start ):
+        lst = os.listdir(dir)
+        if len(lst) != 1:
+            raise Exception( "sad: " + start )
+        return start + "/" + lst[0]
+
+    while "out" not in os.listdir( dir ):
+        dir = go_down( dir )
+
+    dir = dir + "/out"
+    
+    pieces = os.listdir(dir)
+    pieces.sort()
+
+    passed = []
+    failed = []
+    times = {}
+
+    for x in pieces:
+        if not x.startswith( "part" ):
+            continue
+        full = dir + "/" + x
+        
+        for line in open( full , "rb" ):
+            if line.find( "-passed" ) >= 0:
+                passed.append( line.partition( "-passed" )[0] )
+                continue
+            
+            if line.find( "-failed" ) >= 0:
+                failed.append( line.partition( "-failed" )[0] )
+                continue
+                
+            if line.find( "-time-seconds" ) >= 0:
+                p = line.partition( "-time-seconds" )
+                times[p[0]] = p[2].strip()
+                continue
+            
+            print( "\t" + line.strip() )
+            
+    def print_list(name,lst):
+        print( name )
+        for x in lst:
+            print( "\t%s\t%s" % ( x , times[x] ) )
+
+    print_list( "passed" , passed )
+    print_list( "failed" , failed )
+
+    if do_credentials():
+        builder , buildnum = get_build_info()
+        if builder and buildnum:
+            conn = pymongo.Connection( "bbout1.10gen.cc" )
+            db = conn.buildlogs
+            q = { "builder" : builder , "buildnum" : int(buildnum) } 
+            doc = db.builds.find_one( q )
+            
+            if doc:
+                print( "\nhttp://buildlogs.mongodb.org/build/%s" % doc["_id"] )
+            
+
+if __name__ == "__main__":
+    if len(sys.argv) == 1:
+        print( "need an arg" )
+        
+    elif sys.argv[1] == "tarball":
+        make_tarball()
+    elif sys.argv[1] == "jar":
+        build_jar()
+    elif sys.argv[1] == "push":
+        print( push() )
+
+    elif sys.argv[1] == "sync":
+        sync_s3( sys.argv[2] , sys.argv[3] )
+
+    elif sys.argv[1] == "fix_suites":
+        for x in fix_suites( sys.argv[2:] ):
+            print(x)
+
+    elif sys.argv[1] == "credentials":
+        print( do_credentials() )
+
+    elif sys.argv[1] == "test":
+        m , c = make_tarball()
+        build_jar()
+        cmd = [ "java" , "-cp" , os.environ.get( "CLASSPATH" , "." ) + ":emr.jar" , "emr" ]
+
+        workingDir = "/data/emr/test"
+        cmd.append( "--workingDir" )
+        cmd.append( workingDir )
+        if os.path.exists( workingDir ):
+            shutil.rmtree( workingDir )
+        
+        cmd.append( "file://" + os.getcwd() + "/" + m )
+        cmd.append( "file://" + os.getcwd() + "/" + c )
+        
+        out = "/tmp/emrresults"
+        cmd.append( out )
+        if os.path.exists( out ):
+            shutil.rmtree( out )
+
+        cmd.append( "jstests/basic1.js" )
+
+        subprocess.call( cmd )
+        
+        for x in os.listdir( out ):
+            if x.startswith( "." ):
+                continue
+            print( x )
+            for z in open( out + "/" + x ):
+                print( "\t" + z.strip() )
+
+    elif sys.argv[1] == "output":
+        do_output( sys.argv[2] )
+
+    elif sys.argv[1] == "full":
+        things = push()
+        run_tests( things , sys.argv[2:] )
+
+    else:
+        things = push()
+        run_tests( things , sys.argv[1:] )
+
diff --git a/buildscripts/emr/emrnodesetup.sh b/buildscripts/emr/emrnodesetup.sh
new file mode 100644
index 00000000000..546becf3e27
--- /dev/null
+++ b/buildscripts/emr/emrnodesetup.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+sudo mkdir /mnt/data
+sudo ln -s /mnt/data /data
+sudo chown hadoop /mnt/data
+
+sudo easy_install pymongo
diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py
index dec1030ddad..ef5e1a88877 100755
--- a/buildscripts/errorcodes.py
+++ b/buildscripts/errorcodes.py
@@ -6,7 +6,7 @@ import re
 import utils
 
 
-assertNames = [ "uassert" , "massert" ]
+assertNames = [ "uassert" , "massert", "fassert", "fassertFailed" ]
 
 def assignErrorCodes():
     cur = 10000
@@ -32,10 +32,15 @@ def assignErrorCodes():
 codes = []
 
 def readErrorCodes( callback, replaceZero = False ):
-    ps = [ re.compile( "(([umsg]asser(t|ted))) *\(( *)(\d+)" ) ,
-           re.compile( "((User|Msg|MsgAssertion)Exceptio(n))\(( *)(\d+)" ) ,
-           re.compile( "(((verify))) *\(( *)(\d+)" )
+    
+    quick = [ "assert" , "Exception"]
+
+    ps = [ re.compile( "(([umsgf]asser(t|ted))) *\(( *)(\d+)" ) ,
+           re.compile( "((User|Msg|MsgAssertion)Exceptio(n))\(( *)(\d+)" ),
+           re.compile( "((fassertFailed)()) *\(( *)(\d+)" )
            ]
+
+    bad = [ re.compile( "\sassert *\(" ) ]
     
     for x in utils.getAllSourceFiles():
         
@@ -45,30 +50,46 @@ def readErrorCodes( callback, replaceZero = False ):
         lineNum = 1
         
         for line in open( x ):
-            
-            for p in ps:               
-                
-                def repl( m ):
-                    m = m.groups()
-                    
-                    start = m[0]
-                    spaces = m[3]
-                    code = m[4]
-                    if code == '0' and replaceZero :
-                        code = getNextCode( lastCodes )
-                        lastCodes.append( code )
-                        code = str( code )
-                        needReplace[0] = True
-                                            
-                        print( "Adding code " + code + " to line " + x + ":" + str( lineNum ) )
-                        
-                    else :
-                        codes.append( ( x , lineNum , line , code ) )
-                        callback( x , lineNum , line , code )
-                    
-                    return start + "(" + spaces + code
+
+            found = False
+            for zz in quick:
+                if line.find( zz ) >= 0:
+                    found = True
+                    break
+
+            if found:
                 
-                line = re.sub( p, repl, line )
+                if x.find( "src/mongo/" ) >= 0:
+                    for b in bad:
+                        if len(b.findall( line )) > 0:
+                            print( x )
+                            print( line )
+                            raise Exception( "you can't use a bare assert" )
+
+                for p in ps:               
+
+                    def repl( m ):
+                        m = m.groups()
+
+                        start = m[0]
+                        spaces = m[3]
+                        code = m[4]
+                        if code == '0' and replaceZero :
+                            code = getNextCode( lastCodes )
+                            lastCodes.append( code )
+                            code = str( code )
+                            needReplace[0] = True
+
+                            print( "Adding code " + code + " to line " + x + ":" + str( lineNum ) )
+
+                        else :
+                            codes.append( ( x , lineNum , line , code ) )
+                            callback( x , lineNum , line , code )
+
+                        return start + "(" + spaces + code
+
+                    line = re.sub( p, repl, line )
+                    # end if ps loop
             
             if replaceZero : lines.append( line )
             lineNum = lineNum + 1
@@ -78,6 +99,7 @@ def readErrorCodes( callback, replaceZero = False ):
             of = open( x + ".tmp", 'w' )
             of.write( "".join( lines ) )
             of.close()
+            os.remove(x)
             os.rename( x + ".tmp", x )
         
 
diff --git a/buildscripts/hacks_mandriva.py b/buildscripts/hacks_mandriva.py
deleted file mode 100644
index d46170960cc..00000000000
--- a/buildscripts/hacks_mandriva.py
+++ /dev/null
@@ -1,9 +0,0 @@
-
-import os
-import glob
-
-def insert( env , options ):
-    jslibPaths = glob.glob('/usr/include/js-*/')
-    if len(jslibPaths) >= 1:
-        jslibPath = jslibPaths.pop()
-        env.Append( CPPPATH=[ jslibPath ] )
\ No newline at end of file
diff --git a/buildscripts/hacks_ubuntu.py b/buildscripts/hacks_ubuntu.py
deleted file mode 100644
index 3de1a6f0b7e..00000000000
--- a/buildscripts/hacks_ubuntu.py
+++ /dev/null
@@ -1,54 +0,0 @@
-
-import os
-
-def insert( env , options ):
-
-    # now that sm is in the source tree, don't need this
-    # if not foundxulrunner( env , options ):
-    #     if os.path.exists( "usr/include/mozjs/" ):
-    #         env.Append( CPPDEFINES=[ "MOZJS" ] )
-
-    return
-
-def foundxulrunner( env , options ):
-    best = None
-
-    for x in os.listdir( "/usr/include" ):
-        if x.find( "xulrunner" ) != 0:
-            continue
-        if x == "xulrunner":
-            best = x
-            break
-        best = x
-        
-
-    if best is None:
-        print( "warning: using ubuntu without xulrunner-dev.  we recommend installing it" )
-        return False
-
-    incroot = "/usr/include/" + best + "/"
-    libroot = "/usr/lib"
-    if options["linux64"] and os.path.exists("/usr/lib64"):
-        libroot += "64";
-    libroot += "/" + best
-
-    
-    if not os.path.exists( libroot ):
-        print( "warning: found xulrunner include but not lib for: " + best )
-        return False
-    
-    env.Prepend( LIBPATH=[ libroot ] )
-    env.Prepend( RPATH=[ libroot ] )
-
-    env.Prepend( CPPPATH=[ incroot + "stable/" , 
-                           incroot + "unstable/" ,
-                           incroot ] )
-    env.Prepend( CPPPATH=[ "/usr/include/nspr/" ] )
-    
-    env.Append( CPPDEFINES=[ "XULRUNNER" , "OLDJS" ] )
-    if best.find( "1.9.0" ) >= 0 or best.endswith("1.9"):
-        if best.endswith( "1.9.1.9" ):
-            pass
-        else:
-            env.Append( CPPDEFINES=[ "XULRUNNER190" ] )
-    return True
diff --git a/buildscripts/make_archive.py b/buildscripts/make_archive.py
new file mode 100755
index 00000000000..4c12e901a64
--- /dev/null
+++ b/buildscripts/make_archive.py
@@ -0,0 +1,116 @@
+#!/usr/bin/python
+
+'''Helper script for constructing an archive (zip or tar) from a list of files.
+
+The output format (tar, tgz, zip) is determined from the file name, unless the user specifies
+--format on the command line.
+
+This script simplifies the specification of filename transformations, so that, e.g.,
+src/mongo/foo.cpp and build/linux2/normal/buildinfo.cpp can get put into the same
+directory in the archive, perhaps mongodb-2.0.2/src/mongo.
+
+Usage:
+
+make_archive.py -o <output-file> [--format (tar|tgz|zip)] \
+    [--transform match1=replacement1 [--transform match2=replacement2 [...]]] \
+    <input file 1> [...]
+
+If the input file names start with "@", the file is expected to contain a list of
+whitespace-separated file names to include in the archive.  This helps get around the Windows
+command line length limit.
+
+Transformations are processed in command-line order and are short-circuiting.  So, if a file matches
+match1, it is never compared against match2 or later.  Matches are just python startswith()
+comparisons.
+
+For a detailed usage example, see src/SConscript.client or src/mongo/SConscript.
+'''
+
+import optparse
+import os
+import sys
+
+def main(argv):
+    opts = parse_options(argv[1:])
+    archive = open_archive_for_write(opts.output_filename, opts.archive_format)
+    try:
+        for input_filename in opts.input_filenames:
+            archive.add(input_filename, arcname=get_preferred_filename(input_filename,
+                                                                       opts.transformations))
+    finally:
+        archive.close()
+
+def parse_options(args):
+    parser = optparse.OptionParser()
+    parser.add_option('-o', dest='output_filename', default=None,
+                      help='Name of the archive to output.', metavar='FILE')
+    parser.add_option('--format', dest='archive_format', default=None,
+                      choices=('zip', 'tar', 'tgz'),
+                      help='Format of archive to create.  '
+                      'If omitted, use the suffix of the output filename to decide.')
+    parser.add_option('--transform', action='append', dest='transformations', default=[])
+
+    (opts, input_filenames) = parser.parse_args(args)
+    opts.input_filenames = []
+
+    for input_filename in input_filenames:
+        if input_filename.startswith('@'):
+            opts.input_filenames.extend(open(input_filename[1:], 'r').read().split())
+        else:
+            opts.input_filenames.append(input_filename)
+
+    if opts.output_filename is None:
+        parser.error('-o switch is required')
+
+    if opts.archive_format is None:
+        if opts.output_filename.endswith('.zip'):
+            opts.archive_format = 'zip'
+        elif opts.output_filename.endswith('tar.gz') or opts.output_filename.endswith('.tgz'):
+            opts.archive_format = 'tgz'
+        elif opts.output_filename.endswith('.tar'):
+            opts.archive_format = 'tar'
+        else:
+            parser.error('Could not deduce archive format from output filename "%s"' %
+                         opts.output_filename)
+
+    try:
+        opts.transformations = [
+            xform.replace(os.path.altsep or os.path.sep, os.path.sep).split('=', 1)
+            for xform in opts.transformations]
+    except Exception, e:
+        parser.error(e)
+
+    return opts
+
+def open_archive_for_write(filename, archive_format):
+    '''Open a tar or zip archive for write, with the given format, and return it.
+
+    The type of archive is determined by the "archive_format" parameter, which should be
+    "tar", "tgz" (for gzipped tar) or "zip".
+    '''
+
+    if archive_format in ('tar', 'tgz'):
+        import tarfile
+        mode = 'w'
+        if archive_format is 'tgz':
+            mode += '|gz'
+        return tarfile.open(filename, mode)
+    if archive_format is 'zip':
+        import zipfile
+        # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical,
+        # for our purposes.  WrappedZipFile is a minimal adapter class.
+        class WrappedZipFile(zipfile.ZipFile):
+            def add(self, filename, arcname):
+                return self.write(filename, arcname)
+        return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
+    raise ValueError('Unsupported archive format "%s"' % archive_format)
+
+def get_preferred_filename(input_filename, transformations):
+    for match, replace in transformations:
+        if input_filename.startswith(match):
+            return replace + input_filename[len(match):]
+    return input_filename
+
+if __name__ == '__main__':
+    main(sys.argv)
+    sys.exit(0)
diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py
new file mode 100644
index 00000000000..42fc52094ca
--- /dev/null
+++ b/buildscripts/moduleconfig.py
@@ -0,0 +1,131 @@
+"""Utility functions for SCons to discover and configure
+MongoDB modules (sub-trees of db/modules/). This file exports
+two functions:
+
+    discover_modules, which returns a dictionary of module name
+        to the imported python module object for the module's
+        build.py file
+
+    configure_modules, which runs per-module configuration, and
+        is given the SCons environment, its own path, etc
+
+Each module must have a "build.py" script, which is expected to
+have a "configure" function, and optionally a "test" function
+if the module exposes per-module tests.
+"""
+
+__all__ = ('discover_modules', 'configure_modules', 'register_module_test')
+
+import imp
+from os import listdir, makedirs
+from os.path import abspath, dirname, join, isdir, isfile
+
+def discover_modules(mongo_root):
+    """Scan <mongo_root>/db/modules/ for directories that
+    look like MongoDB modules (i.e. they contain a "build.py"
+    file), and return a dictionary of module name (the directory
+    name) to build.py python modules.
+    """
+    found_modules = {}
+
+    module_root = abspath(join(mongo_root, 'db', 'modules'))
+    if not isdir(module_root):
+        return found_modules
+
+    for name in listdir(module_root):
+        root = join(module_root, name)
+        if '.' in name or not isdir(root):
+            continue
+
+        build_py = join(root, 'build.py')
+        module = None
+
+        if isfile(build_py):
+            print "adding module: %s" % name
+            fp = open(build_py, "r")
+            module = imp.load_module("module_" + name, fp, build_py, (".py", "r", imp.PY_SOURCE))
+            found_modules[name] = module
+            fp.close()
+
+    return found_modules
+
+def configure_modules(modules, conf, env):
+    """
+    Run the configure() function in the build.py python modules
+    for each module listed in the modules dictionary (as created
+    by discover_modules). The configure() function should use the
+    prepare the Mongo build system for building the module.
+
+    build.py files may specify a "customIncludes" flag, which, if
+    True, causes configure() to be called with three arguments:
+    the SCons Configure() object, the SCons environment, and an
+    empty list which should be modified in-place by the configure()
+    function; if false, configure() is called with only the first
+    two arguments, and the source files are discovered with a
+    glob against the <module_root>/src/*.cpp.
+
+    Returns a dictionary mapping module name to a list of source
+    files to be compiled for the module.
+    """
+    source_map = {}
+
+    for name, module in modules.items():
+        print "configuring module: %s" % name
+
+        root = dirname(module.__file__)
+        module_sources = []
+
+        if getattr(module, "customIncludes", False):
+            # then the module configures itself and its
+            # configure() takes 3 args
+            module.configure(conf, env, module_sources)
+        else:
+            # else we glob the files in the module's src/
+            # subdirectory, and its configure() takes 2 args
+            module.configure(conf, env)
+            module_sources.extend(env.Glob(join(root, "src/*.cpp")))
+
+        if not module_sources:
+            print "WARNING: no source files for module %s, module will not be built." % name
+        else:
+            source_map[name] = module_sources
+
+    _setup_module_tests_file(str(env.File(env['MODULETEST_LIST'])))
+
+    return source_map
+
+module_tests = []
+def register_module_test(*command):
+    """Modules can register tests as part of their configure(), which
+    are commands whose exit status indicates the success or failure of
+    the test.
+
+    Use this function from configure() like:
+
+        register_module_test('/usr/bin/python', '/path/to/module/tests/foo.py')
+        register_module_test('/bin/bash', '/path/to/module/tests/bar.sh')
+
+    The registered test commands can be run with "scons smokeModuleTests"
+    """
+    command = ' '.join(command)
+    module_tests.append(command)
+
+def _setup_module_tests_file(test_file):
+    """Modules' configure() functions may have called register_module_test,
+    in which case, we need to record the registered tests' commands into
+    a text file which smoke.py and SCons know how to work with.
+    """
+    if not module_tests:
+        return
+
+    folder = dirname(test_file)
+    if not isdir(folder):
+        makedirs(folder)
+
+    fp = file(test_file, 'w')
+    for test in module_tests:
+        fp.write(test)
+        fp.write('\n')
+    fp.close()
+    print "Generated %s" % test_file
+
diff --git a/buildscripts/packager.py b/buildscripts/packager.py
index 400239cfd54..d0dd492981a 100644
--- a/buildscripts/packager.py
+++ b/buildscripts/packager.py
@@ -376,7 +376,7 @@ def make_deb(distro, arch, spec, srcdir):
     oldcwd=os.getcwd()
     try:
         os.chdir(sdir)
-        sysassert(["dpkg-buildpackage", "-a"+distro_arch])
+        sysassert(["dpkg-buildpackage", "-a"+distro_arch, "-k Richard Kreuter <richard@10gen.com>"])
     finally:
         os.chdir(oldcwd)
     r=distro.repodir(arch)
@@ -576,7 +576,7 @@ Description: An object/document-oriented database
 """
     s=re.sub("@@PACKAGE_BASENAME@@", "mongodb%s" % spec.suffix(), s)
     conflict_suffixes=["", "-stable", "-unstable", "-nightly", "-10gen", "-10gen-unstable"]
-    conflict_suffixes.remove(spec.suffix())
+    conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
     s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongodb"+suffix for suffix in conflict_suffixes]), s)
     f=open(path, 'w')
     try:
@@ -686,7 +686,8 @@ binary-arch: build install
 #\tdh_installinfo
 \tdh_installman
 \tdh_link
-\tdh_strip
+# Appears to be broken on Ubuntu 11.10...?
+#\tdh_strip
 \tdh_compress
 \tdh_fixperms
 \tdh_installdeb
@@ -901,8 +902,10 @@ fi
 %{_bindir}/mongo
 %{_bindir}/mongodump
 %{_bindir}/mongoexport
-%{_bindir}/mongofiles
+#@@VERSION!=2.1.0@@%{_bindir}/mongofiles
 %{_bindir}/mongoimport
+#@@VERSION>=2.1.0@@%{_bindir}/mongooplog
+#@@VERSION>=2.1.0@@%{_bindir}/mongoperf
 %{_bindir}/mongorestore
 #@@VERSION>1.9@@%{_bindir}/mongotop
 %{_bindir}/mongostat
@@ -952,9 +955,9 @@ fi
     s=re.sub("@@PACKAGE_REVISION@@", str(int(spec.param("revision"))+1) if spec.param("revision") else "1", s)
     s=re.sub("@@BINARYDIR@@", BINARYDIR, s)
     conflict_suffixes=["", "-10gen", "-10gen-unstable"]
-    conflict_suffixes.remove(suffix)
+    conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
     s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongo"+_ for _ in conflict_suffixes]), s)
-    if suffix == "-10gen":
+    if suffix.endswith("-10gen"):
         s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-stable", s)
         s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-stable", s)
     elif suffix == "-10gen-unstable":
@@ -965,9 +968,25 @@ fi
 
     lines=[]
     for line in s.split("\n"):
-        m = re.search("@@VERSION>(.*)@@(.*)", line)
-        if m and spec.version_better_than(m.group(1)):
-            lines.append(m.group(2))
+        m = re.search("@@VERSION(>|>=|!=)(\d.*)@@(.*)", line)
+        if m:
+          op = m.group(1)
+          ver = m.group(2)
+          fn = m.group(3)
+          if op == '>':
+            if spec.version_better_than(ver):
+              lines.append(fn)
+          elif op == '>=':
+            if spec.version() == ver or spec.version_better_than(ver):
+              lines.append(fn)
+          elif op == '!=':
+            if spec.version() != ver:
+              lines.append(fn)
+          else:
+            # Since we're inventing our own template system for RPM
+            # specfiles here, we oughtn't use template syntax we don't
+            # support.
+            raise Exception("BUG: probable bug in packager script: %s, %s, %s" % (m.group(1), m.group(2), m.group(3)))
         else:
             lines.append(line)
     s="\n".join(lines)
diff --git a/buildscripts/s3md5.py b/buildscripts/s3md5.py
index 89800cd6898..3b0d0d82917 100644
--- a/buildscripts/s3md5.py
+++ b/buildscripts/s3md5.py
@@ -18,7 +18,7 @@ def check_dir( bucket , prefix ):
     zips = {}
     md5s = {}
     for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
-        if key.endswith( ".tgz" ) or key.endswith( ".zip" ):
+        if key.endswith( ".tgz" ) or key.endswith( ".zip" ) or key.endswith( ".tar.gz" ):
             zips[key] = etag.replace( '"' , '' )
         elif key.endswith( ".md5" ):
             md5s[key] = True
@@ -40,7 +40,7 @@ def run():
 
     bucket = simples3.S3Bucket( settings.bucket , settings.id , settings.key )
     
-    for x in [ "osx" , "linux" , "win32" , "sunos5" ]:
+    for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
         check_dir( bucket , x )
     
 
diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py
index c46b5d1879d..7740756b42f 100755
--- a/buildscripts/smoke.py
+++ b/buildscripts/smoke.py
@@ -33,14 +33,14 @@
 #   off all mongods on a box, which means you can't run two smoke.py
 #   jobs on the same host at once.  So something's gotta change.
 
-from __future__ import with_statement
-
+from datetime import datetime
 import glob
 from optparse import OptionParser
 import os
 import parser
 import re
 import shutil
+import shlex
 import socket
 from subprocess import (Popen,
                         PIPE,
@@ -49,20 +49,29 @@ import sys
 import time
 
 from pymongo import Connection
+from pymongo.errors import OperationFailure
 
 import utils
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 # TODO clean this up so we don't need globals...
 mongo_repo = os.getcwd() #'./'
+failfile = os.path.join(mongo_repo, 'failfile.smoke')
 test_path = None
 mongod_executable = None
 mongod_port = None
 shell_executable = None
 continue_on_failure = None
+file_of_commands_mode = False
 
 tests = []
 winners = []
 losers = {}
+fails = [] # like losers but in format of tests
 
 # For replication hash checking
 replicated_collections = []
@@ -72,6 +81,7 @@ screwy_in_slave = {}
 
 smoke_db_prefix = ''
 small_oplog = False
+small_oplog_rs = False
 
 # This class just implements the with statement API, for a sneaky
 # purpose below.
@@ -81,10 +91,24 @@ class Nothing(object):
     def __exit__(self, type, value, traceback):
         return not isinstance(value, Exception)
 
+def buildlogger(cmd, is_global=False):
+    # if the environment variable MONGO_USE_BUILDLOGGER
+    # is set to 'true', then wrap the command with a call
+    # to buildlogger.py, which sends output to the buidlogger
+    # machine; otherwise, return as usual.
+    if os.environ.get('MONGO_USE_BUILDLOGGER', '').lower().strip() == 'true':
+        if is_global:
+            return [utils.find_python(), 'buildscripts/buildlogger.py', '-g'] + cmd
+        else:
+            return [utils.find_python(), 'buildscripts/buildlogger.py'] + cmd
+    return cmd
+
+
 class mongod(object):
     def __init__(self, **kwargs):
         self.kwargs = kwargs
         self.proc = None
+        self.auth = False
 
     def __enter__(self):
         self.start()
@@ -122,6 +146,15 @@ class mongod(object):
         print >> sys.stderr, "timeout starting mongod"
         return False
 
+    def setup_admin_user(self, port=mongod_port):
+        try:
+            Connection( "localhost" , int(port) ).admin.add_user("admin","password")
+        except OperationFailure, e:
+            if e.message == 'need to login':
+                pass # SERVER-4225
+            else:
+                raise e
+
     def start(self):
         global mongod_port
         global mongod
@@ -139,25 +172,34 @@ class mongod(object):
             self.slave = True
         if os.path.exists(dir_name):
             if 'slave' in self.kwargs:
-                argv = ["python", "buildscripts/cleanbb.py", '--nokill', dir_name]
+                argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
             else:
-                argv = ["python", "buildscripts/cleanbb.py", dir_name]
+                argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
             call(argv)
         utils.ensureDir(dir_name)
         argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
         if self.kwargs.get('small_oplog'):
-            argv += ["--master", "--oplogSize", "256"]
+            argv += ["--master", "--oplogSize", "511"]
+        if self.kwargs.get('small_oplog_rs'):
+            argv += ["--replSet", "foo", "--oplogSize", "511"]
         if self.slave:
             argv += ['--slave', '--source', 'localhost:' + str(srcport)]
         if self.kwargs.get('no_journal'):
             argv += ['--nojournal']
         if self.kwargs.get('no_preallocj'):
             argv += ['--nopreallocj']
+        if self.kwargs.get('auth'):
+            argv += ['--auth']
+            self.auth = True
         print "running " + " ".join(argv)
-        self.proc = Popen(argv)
+        self.proc = self._start(buildlogger(argv, is_global=True))
+
         if not self.did_mongod_start(self.port):
             raise Exception("Failed to start mongod")
 
+        if self.auth:
+            self.setup_admin_user(self.port)
+
         if self.slave:
             local = Connection(port=self.port, slave_okay=True).local
             synced = False
@@ -166,20 +208,53 @@ class mongod(object):
                 for source in local.sources.find(fields=["syncedTo"]):
                     synced = synced and "syncedTo" in source and source["syncedTo"]
 
+    def _start(self, argv):
+        """In most cases, just call subprocess.Popen(). On windows,
+        add the started process to a new Job Object, so that any
+        child processes of this process can be killed with a single
+        call to TerminateJobObject (see self.stop()).
+        """
+        proc = Popen(argv)
+
+        if os.sys.platform == "win32":
+            # Create a job object with the "kill on job close"
+            # flag; this is inherited by child processes (ie
+            # the mongod started on our behalf by buildlogger)
+            # and lets us terminate the whole tree of processes
+            # rather than orphaning the mongod.
+            import win32job
+
+            self.job_object = win32job.CreateJobObject(None, '')
+
+            job_info = win32job.QueryInformationJobObject(
+                self.job_object, win32job.JobObjectExtendedLimitInformation)
+            job_info['BasicLimitInformation']['LimitFlags'] |= win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+            win32job.SetInformationJobObject(
+                self.job_object,
+                win32job.JobObjectExtendedLimitInformation,
+                job_info)
+
+            win32job.AssignProcessToJobObject(self.job_object, proc._handle)
+
+        return proc
+
     def stop(self):
         if not self.proc:
             print >> sys.stderr, "probable bug: self.proc unset in stop()"
             return
         try:
-            # This function not available in Python 2.5
-            self.proc.terminate()
-        except AttributeError:
             if os.sys.platform == "win32":
-                import win32process
-                win32process.TerminateProcess(self.proc._handle, -1)
+                import win32job
+                win32job.TerminateJobObject(self.job_object, -1)
+                import time
+                # Windows doesn't seem to kill the process immediately, so give it some time to die
+                time.sleep(5) 
             else:
-                from os import kill
-                kill(self.proc.pid, 15)
+                # This function not available in Python 2.5
+                self.proc.terminate()
+        except AttributeError:
+            from os import kill
+            kill(self.proc.pid, 15)
         self.proc.wait()
         sys.stderr.flush()
         sys.stdout.flush()
@@ -241,12 +316,31 @@ def check_db_hashes(master, slave):
             lost_in_master.append(db)
 
 
+def ternary( b , l="true", r="false" ):
+    if b:
+        return l
+    return r
+
 
 # Blech.
 def skipTest(path):
-    if small_oplog:
-        if os.path.basename(path) in ["cursor8.js", "indexh.js", "dropdb.js"]:
+    basename = os.path.basename(path)
+    parentDir = os.path.basename(os.path.dirname(path))
+    if small_oplog: # For tests running in parallel
+        if basename in ["cursor8.js", "indexh.js", "dropdb.js"]:
+            return True
+    if auth or keyFile: # For tests running with auth
+        # Skip any tests that run with auth explicitly
+        if parentDir == "auth" or "auth" in basename or parentDir == "tool": # SERVER-6368
+            return True
+        # These tests don't pass with authentication due to limitations of the test infrastructure,
+        # not due to actual bugs.
+        if os.path.join(parentDir,basename) in ["sharding/sync3.js", "sharding/sync6.js", "sharding/parallel.js", "jstests/bench_test1.js", "jstests/bench_test2.js", "jstests/bench_test3.js"]:
+            return True
+        # These tests fail due to bugs
+        if os.path.join(parentDir,basename) in ["sharding/sync_conn_cmd.js"]:
             return True
+
     return False
 
 def runTest(test):
@@ -259,11 +353,22 @@ def runTest(test):
     if skipTest(path):
         print "skipping " + path
         return
-    if ext == ".js":
+    if file_of_commands_mode:
+        # smoke.py was invoked like "--mode files --from-file foo",
+        # so don't try to interpret the test path too much
+        if os.sys.platform == "win32":
+            argv = [path]
+        else:
+            argv = shlex.split(path)
+        path = argv[0]
+        # if the command is a python script, use the script name
+        if os.path.basename(path) in ('python', 'python.exe'):
+            path = argv[1]
+    elif ext == ".js":
         argv = [shell_executable, "--port", mongod_port]
         if not usedb:
             argv += ["--nodb"]
-        if small_oplog:
+        if small_oplog or small_oplog_rs:
             argv += ["--eval", 'testingReplication = true;']
         argv += [path]
     elif ext in ["", ".exe"]:
@@ -271,36 +376,68 @@ def runTest(test):
         if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
             argv = [path]
         # more blech
-        elif os.path.basename(path) == 'mongos':
+        elif os.path.basename(path) in ['mongos', 'mongos.exe']:
             argv = [path, "--test"]
         else:
             argv = [test_path and os.path.abspath(os.path.join(test_path, path)) or path,
                     "--port", mongod_port]
     else:
         raise Bug("fell off in extenstion case: %s" % path)
-    sys.stderr.write( "starting test : %s \n" % os.path.basename(path) )
-    sys.stderr.flush()
-    print " *******************************************"
-    print "         Test : " + os.path.basename(path) + " ..."
-    t1 = time.time()
+
+    if keyFile:
+        f = open(keyFile, 'r')
+        keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace
+        f.close()
+    else:
+        keyFileData = None
+
+
+    # sys.stdout.write() is more atomic than print, so using it prevents
+    # lines being interrupted by, e.g., child processes
+    sys.stdout.write(" *******************************************\n")
+    sys.stdout.write("         Test : %s ...\n" % os.path.basename(path))
+    sys.stdout.flush()
+
     # FIXME: we don't handle the case where the subprocess
     # hangs... that's bad.
-    if argv[0].endswith( 'mongo' ) and not '--eval' in argv :
-        argv = argv + [ '--eval', 'TestData = new Object();' + 
-                                  'TestData.testPath = "' + path + '";' + 
-                                  'TestData.testFile = "' + os.path.basename( path ) + '";' +
-                                  'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + 
-                                  'TestData.noJournal = ' + ( 'true' if no_journal else 'false' )  + ";" +
-                                  'TestData.noJournalPrealloc = ' + ( 'true' if no_preallocj else 'false' )  + ";" ]
+    if ( argv[0].endswith( 'mongo' ) or argv[0].endswith( 'mongo.exe' ) ) and not '--eval' in argv :
+        evalString = 'TestData = new Object();' + \
+                     'TestData.testPath = "' + path + '";' + \
+                     'TestData.testFile = "' + os.path.basename( path ) + '";' + \
+                     'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \
+                     'TestData.noJournal = ' + ternary( no_journal )  + ";" + \
+                     'TestData.noJournalPrealloc = ' + ternary( no_preallocj )  + ";" + \
+                     'TestData.auth = ' + ternary( auth ) + ";" + \
+                     'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \
+                     'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";"
+        if os.sys.platform == "win32":
+            # double quotes in the evalString on windows; this
+            # prevents the backslashes from being removed when
+            # the shell (i.e. bash) evaluates this string. yuck.
+            evalString = evalString.replace('\\', '\\\\')
+
+        if auth and usedb:
+            evalString += 'jsTest.authenticate(db.getMongo());'
+
+        argv = argv + [ '--eval', evalString]
     
     if argv[0].endswith( 'test' ) and no_preallocj :
         argv = argv + [ '--nopreallocj' ]
     
     
-    print argv
-    r = call(argv, cwd=test_path)
+    sys.stdout.write("      Command : %s\n" % ' '.join(argv))
+    sys.stdout.write("         Date : %s\n" % datetime.now().ctime())
+    sys.stdout.flush()
+
+    os.environ['MONGO_TEST_FILENAME'] = os.path.basename(path)
+    t1 = time.time()
+    r = call(buildlogger(argv), cwd=test_path)
     t2 = time.time()
-    print "                " + str((t2 - t1) * 1000) + "ms"
+    del os.environ['MONGO_TEST_FILENAME']
+
+    sys.stdout.write("                %fms\n" % ((t2 - t1) * 1000))
+    sys.stdout.flush()
+
     if r != 0:
         raise TestExitFailure(path, r)
     
@@ -317,17 +454,49 @@ def run_tests(tests):
     # dbpath, etc., and so long as we shut ours down properly,
     # starting this mongod shouldn't break anything, though.)
     
-    # The reason we use with is so that we get __exit__ semantics
+    # The reason we want to use "with" is so that we get __exit__ semantics
+    # but "with" is only supported on Python 2.5+
 
-    with mongod(small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj) as master:
-        with mongod(slave=True) if small_oplog else Nothing() as slave:
-            if small_oplog:
+    master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+    try:
+        if small_oplog:
+            slave = mongod(slave=True).__enter__()
+        elif small_oplog_rs:
+            slave = mongod(slave=True,small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+            primary = Connection(port=master.port, slave_okay=True);
+
+            primary.admin.command({'replSetInitiate' : {'_id' : 'foo', 'members' : [
+                            {'_id': 0, 'host':'localhost:%s' % master.port},
+                            {'_id': 1, 'host':'localhost:%s' % slave.port,'priority':0}]}})
+
+            ismaster = False
+            while not ismaster:
+                result = primary.admin.command("ismaster");
+                ismaster = result["ismaster"]
+                time.sleep(1)
+        else:
+            slave = Nothing()
+
+        try:
+            if small_oplog or small_oplog_rs:
                 master.wait_for_repl()
 
-            for test in tests:
+            tests_run = 0
+            for tests_run, test in enumerate(tests):
                 try:
+                    fails.append(test)
                     runTest(test)
+                    fails.pop()
                     winners.append(test)
+
+                    if small_oplog or small_oplog_rs:
+                        master.wait_for_repl()
+                    elif test[1]: # reach inside test and see if startmongod is true
+                        if (tests_run+1) % 20 == 0:
+                            # restart mongo every 20 times, for our 32-bit machines
+                            master.__exit__(None, None, None)
+                            master = mongod(small_oplog_rs=small_oplog_rs,small_oplog=small_oplog,no_journal=no_journal,no_preallocj=no_preallocj,auth=auth).__enter__()
+
                 except TestFailure, f:
                     try:
                         print f
@@ -341,12 +510,15 @@ def run_tests(tests):
                             return 1
             if isinstance(slave, mongod):
                 check_db_hashes(master, slave)
-
+        finally:
+            slave.__exit__(None, None, None)
+    finally:
+        master.__exit__(None, None, None)
     return 0
 
 
 def report():
-    print "%d test%s succeeded" % (len(winners), '' if len(winners) == 1 else 's')
+    print "%d tests succeeded" % len(winners)
     num_missed = len(tests) - (len(winners) + len(losers.keys()))
     if num_missed:
         print "%d tests didn't get run" % num_missed
@@ -368,18 +540,35 @@ at the end of testing:""" % (src, dst)
 at the end of testing:"""
         for db in screwy_in_slave.keys():
             print "%s\t %s" % (db, screwy_in_slave[db])
-    if small_oplog and not (lost_in_master or lost_in_slave or screwy_in_slave):
+    if (small_oplog or small_oplog_rs) and not (lost_in_master or lost_in_slave or screwy_in_slave):
         print "replication ok for %d collections" % (len(replicated_collections))
     if losers or lost_in_slave or lost_in_master or screwy_in_slave:
         raise Exception("Test failures")
 
-
-def expand_suites(suites):
+suiteGlobalConfig = {"js": ("[!_]*.js", True),
+                     "quota": ("quota/*.js", True),
+                     "jsPerf": ("perf/*.js", True),
+                     "disk": ("disk/*.js", True),
+                     "jsSlowNightly": ("slowNightly/*.js", True),
+                     "jsSlowWeekly": ("slowWeekly/*.js", False),
+                     "parallel": ("parallel/*.js", True),
+                     "clone": ("clone/*.js", False),
+                     "repl": ("repl/*.js", False),
+                     "replSets": ("replsets/*.js", False),
+                     "dur": ("dur/*.js", False),
+                     "auth": ("auth/*.js", False),
+                     "sharding": ("sharding/*.js", False),
+                     "tool": ("tool/*.js", False),
+                     "aggregation": ("aggregation/*.js", True),
+                     "multiVersion": ("multiVersion/*.js", True )
+                     }
+
+def expand_suites(suites,expandUseDB=True):
     globstr = None
     tests = []
     for suite in suites:
         if suite == 'all':
-            return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'parallel', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'])
+            return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'],expandUseDB=expandUseDB)
         if suite == 'test':
             if os.sys.platform == "win32":
                 program = 'test.exe'
@@ -405,31 +594,31 @@ def expand_suites(suites):
                 program = 'mongos'
             tests += [(os.path.join(mongo_repo, program), False)]
         elif os.path.exists( suite ):
-            tests += [ ( os.path.join( mongo_repo , suite ) , True ) ]
+            usedb = True
+            for name in suiteGlobalConfig:
+                if suite in glob.glob( "jstests/" + suiteGlobalConfig[name][0] ):
+                    usedb = suiteGlobalConfig[name][1]
+                    break
+            tests += [ ( os.path.join( mongo_repo , suite ) , usedb ) ]
         else:
             try:
-                globstr, usedb = {"js": ("[!_]*.js", True),
-                                  "quota": ("quota/*.js", True),
-                                  "jsPerf": ("perf/*.js", True),
-                                  "disk": ("disk/*.js", True),
-                                  "jsSlowNightly": ("slowNightly/*.js", True),
-                                  "jsSlowWeekly": ("slowWeekly/*.js", True),
-                                  "parallel": ("parallel/*.js", True),
-                                  "clone": ("clone/*.js", False),
-                                  "repl": ("repl/*.js", False),
-                                  "replSets": ("replsets/*.js", False),
-                                  "dur": ("dur/*.js", False),
-                                  "auth": ("auth/*.js", False),
-                                  "sharding": ("sharding/*.js", False),
-                                  "tool": ("tool/*.js", False)}[suite]
+                globstr, usedb = suiteGlobalConfig[suite]
             except KeyError:
                 raise Exception('unknown test suite %s' % suite)
 
         if globstr:
-            globstr = os.path.join(mongo_repo, (os.path.join(('jstests/' if globstr.endswith('.js') else ''), globstr)))
-            paths = glob.glob(globstr)
-            paths.sort()
-            tests += [(path, usedb) for path in paths]
+            if usedb and not expandUseDB:
+                tests += [ (suite,False) ]
+            else:                
+                if globstr.endswith('.js'):
+                    loc = 'jstests/'
+                else:
+                    loc = ''
+                globstr = os.path.join(mongo_repo, (os.path.join(loc, globstr)))
+                globstr = os.path.normpath(globstr)
+                paths = glob.glob(globstr)
+                paths.sort()
+                tests += [(path, usedb) for path in paths]
 
     return tests
 
@@ -438,8 +627,118 @@ def add_exe(e):
         e += ".exe"
     return e
 
+def set_globals(options, tests):
+    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
+    global file_of_commands_mode
+    #Careful, this can be called multiple times
+    test_path = options.test_path
+
+    mongod_executable = add_exe(options.mongod_executable)
+    if not os.path.exists(mongod_executable):
+        raise Exception("no mongod found in this directory.")
+
+    mongod_port = options.mongod_port
+
+    shell_executable = add_exe( options.shell_executable )
+    if not os.path.exists(shell_executable):
+        raise Exception("no mongo shell found in this directory.")
+
+    continue_on_failure = options.continue_on_failure
+    smoke_db_prefix = options.smoke_db_prefix
+    small_oplog = options.small_oplog
+    if hasattr(options, "small_oplog_rs"):
+        small_oplog_rs = options.small_oplog_rs
+    no_journal = options.no_journal
+    no_preallocj = options.no_preallocj
+    if options.mode == 'suite' and tests == ['client']:
+        # The client suite doesn't work with authentication
+        if options.auth:
+            print "Not running client suite with auth even though --auth was provided"
+        auth = False;
+        keyFile = False;
+    else:
+        auth = options.auth
+        keyFile = options.keyFile
+
+    if auth and not keyFile:
+        # if only --auth was given to smoke.py, load the
+        # default keyFile from jstests/libs/authTestsKey
+        keyFile = os.path.join(mongo_repo, 'jstests', 'libs', 'authTestsKey')
+
+    # if smoke.py is running a list of commands read from a
+    # file (or stdin) rather than running a suite of js tests
+    file_of_commands_mode = options.File and options.mode == 'files'
+
+def clear_failfile():
+    if os.path.exists(failfile):
+        os.remove(failfile)
+
+def run_old_fails():
+    global tests
+
+    try:
+        f = open(failfile, 'r')
+        testsAndOptions = pickle.load(f)
+        f.close()
+    except Exception:
+        try:
+            f.close()
+        except:
+            pass
+        clear_failfile()
+        return # This counts as passing so we will run all tests
+
+    tests = [x[0] for x in testsAndOptions]
+    passed = []
+    try:
+        for (i, (test, options)) in enumerate(testsAndOptions):
+            # SERVER-5102: until we can figure out a better way to manage
+            # dependencies of the --only-old-fails build phase, just skip
+            # tests which we can't safely run at this point
+            path, usedb = test
+
+            if not os.path.exists(path):
+                passed.append(i)
+                winners.append(test)
+                continue
+
+            filename = os.path.basename(path)
+            if filename in ('test', 'test.exe') or filename.endswith('.js'):
+                set_globals(options, [filename])
+                oldWinners = len(winners)
+                run_tests([test])
+                if len(winners) != oldWinners: # can't use return value due to continue_on_failure
+                    passed.append(i)
+    finally:
+        for offset, i in enumerate(passed):
+            testsAndOptions.pop(i - offset)
+
+        if testsAndOptions:
+            f = open(failfile, 'w')
+            pickle.dump(testsAndOptions, f)
+        else:
+            clear_failfile()
+
+        report() # exits with failure code if there is an error
+
+def add_to_failfile(tests, options):
+    try:
+        f = open(failfile, 'r')
+        testsAndOptions = pickle.load(f)
+    except Exception:
+        testsAndOptions = []
+
+    for test in tests:
+        if (test, options) not in testsAndOptions:
+            testsAndOptions.append( (test, options) )
+
+    f = open(failfile, 'w')
+    pickle.dump(testsAndOptions, f)
+
+
+
 def main():
-    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, smoke_db_prefix, test_path
+    global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
     parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
     parser.add_option('--mode', dest='mode', default='suite',
                       help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
@@ -451,7 +750,7 @@ def main():
                       "currently only used for 'client' (%default)")
     parser.add_option('--mongod', dest='mongod_executable', default=os.path.join(mongo_repo, 'mongod'),
                       help='Path to mongod to run (%default)')
-    parser.add_option('--port', dest='mongod_port', default="32000",
+    parser.add_option('--port', dest='mongod_port', default="27999",
                       help='Port the mongod will bind to (%default)')
     parser.add_option('--mongo', dest='shell_executable', default=os.path.join(mongo_repo, 'mongo'),
                       help='Path to mongo, for .js test files (%default)')
@@ -465,55 +764,96 @@ def main():
     parser.add_option('--small-oplog', dest='small_oplog', default=False,
                       action="store_true",
                       help='Run tests with master/slave replication & use a small oplog')
+    parser.add_option('--small-oplog-rs', dest='small_oplog_rs', default=False,
+                      action="store_true",
+                      help='Run tests with replica set replication & use a small oplog')
     parser.add_option('--nojournal', dest='no_journal', default=False,
                       action="store_true",
                       help='Do not turn on journaling in tests')
     parser.add_option('--nopreallocj', dest='no_preallocj', default=False,
                       action="store_true",
                       help='Do not preallocate journal files in tests')
+    parser.add_option('--auth', dest='auth', default=False,
+                      action="store_true",
+                      help='Run standalone mongods in tests with authentication enabled')
+    parser.add_option('--keyFile', dest='keyFile', default=None,
+                      help='Path to keyFile to use to run replSet and sharding tests with authentication enabled')
+    parser.add_option('--ignore', dest='ignore_files', default=None,
+                      help='Pattern of files to ignore in tests')
+    parser.add_option('--only-old-fails', dest='only_old_fails', default=False,
+                      action="store_true",
+                      help='Check the failfile and only run all tests that failed last time')
+    parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
+                      action="store_true",
+                      help='Clear the failfile. Do this if all tests pass')
+    parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
+                      action="store_true",
+                      help='Clear database files from previous smoke.py runs')
+
+    # Buildlogger invocation from command line
+    parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
+                      action="store", help='Set the "builder name" for buildlogger')
+    parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
+                      action="store", help='Set the "build number" for buildlogger')
+    parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
+                      action="store", help='Path to Python file containing buildlogger credentials')
+    parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
+                      action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')
+
     global tests
     (options, tests) = parser.parse_args()
 
-    print tests
-
-    test_path = options.test_path
-
-    mongod_executable = add_exe(options.mongod_executable)
-    if not os.path.exists(mongod_executable):
-        raise Exception("no mongod found in this directory.")
-
-    mongod_port = options.mongod_port
-
-    shell_executable = add_exe( options.shell_executable )
-    if not os.path.exists(shell_executable):
-        raise Exception("no mongo shell found in this directory.")
+    set_globals(options, tests)
 
-    continue_on_failure = options.continue_on_failure
-    smoke_db_prefix = options.smoke_db_prefix
-    small_oplog = options.small_oplog
-    no_journal = options.no_journal
-    no_preallocj = options.no_preallocj
+    buildlogger_opts = (options.buildlogger_builder, options.buildlogger_buildnum, options.buildlogger_credentials)
+    if all(buildlogger_opts):
+        os.environ['MONGO_USE_BUILDLOGGER'] = 'true'
+        os.environ['MONGO_BUILDER_NAME'] = options.buildlogger_builder
+        os.environ['MONGO_BUILD_NUMBER'] = options.buildlogger_buildnum
+        os.environ['BUILDLOGGER_CREDENTIALS'] = options.buildlogger_credentials
+        if options.buildlogger_phase:
+            os.environ['MONGO_PHASE'] = options.buildlogger_phase
+    elif any(buildlogger_opts):
+        # some but not all of the required options were sete
+        raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")
 
     if options.File:
         if options.File == '-':
             tests = sys.stdin.readlines()
         else:
-            with open(options.File) as f:
-                tests = f.readlines()
+            f = open(options.File)
+            tests = f.readlines()
     tests = [t.rstrip('\n') for t in tests]
 
+    if options.only_old_fails:
+        run_old_fails()
+        return
+    elif options.reset_old_fails:
+        clear_failfile()
+        return
+
     # If we're in suite mode, tests is a list of names of sets of tests.
     if options.mode == 'suite':
         tests = expand_suites(tests)
     elif options.mode == 'files':
         tests = [(os.path.abspath(test), True) for test in tests]
 
+    if options.ignore_files != None :
+        ignore_patt = re.compile( options.ignore_files )
+        tests = filter( lambda x : ignore_patt.search( x[0] ) == None, tests )
+
     if not tests:
-        raise Exception( "no tests specified" )
+        print "warning: no tests specified"
+        return
+
+    if options.with_cleanbb:
+        dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
+        call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])
 
     try:
         run_tests(tests)
     finally:
+        add_to_failfile(fails, options)
         report()
 
 
diff --git a/buildscripts/utils.py b/buildscripts/utils.py
index bde2b083612..1f04d7b6af4 100644
--- a/buildscripts/utils.py
+++ b/buildscripts/utils.py
@@ -1,9 +1,14 @@
 
+import codecs
 import re
 import socket
 import time
 import os
+import os.path
+import itertools
+import subprocess
 import sys
+import hashlib
 
 # various utilities that are handy
 
@@ -136,34 +141,84 @@ def didMongodStart( port=27017 , timeout=20 ):
             timeout = timeout - 1
     return False
 
-def smoke_python_name():
-    # if this script is being run by py2.5 or greater,
-    # then we assume that "python" points to a 2.5 or
-    # greater python VM. otherwise, explicitly use 2.5
-    # which we assume to be installed.
-    min_version_tuple = (2, 5)
+def which(executable):
+    if sys.platform == 'win32':
+        paths = os.environ.get('Path', '').split(';')
+    else:
+        paths = os.environ.get('PATH', '').split(':')
+
+    for path in paths:
+        path = os.path.expandvars(path)
+        path = os.path.expanduser(path)
+        path = os.path.abspath(path)
+        executable_path = os.path.join(path, executable)
+        if os.path.exists(executable_path):
+            return executable_path
+
+    return executable
+
+def md5sum( file ):
+    #TODO error handling, etc..
+    return execsys( "md5sum " + file )[0].partition(" ")[0]
+
+def md5string( a_string ):
+    return hashlib.md5(a_string).hexdigest()
+
+def find_python(min_version=(2, 5)):
     try:
-        if sys.version_info >= min_version_tuple:
+        if sys.version_info >= min_version:
             return sys.executable
     except AttributeError:
         # In case the version of Python is somehow missing sys.version_info or sys.executable.
         pass
 
-    import subprocess
     version = re.compile(r'[Pp]ython ([\d\.]+)', re.MULTILINE)
-    binaries = ['python2.5', 'python2.6', 'python2.7', 'python25', 'python26', 'python27', 'python']
+    binaries = ('python27', 'python2.7', 'python26', 'python2.6', 'python25', 'python2.5', 'python')
     for binary in binaries:
         try:
-            # py-2.4 compatible replacement for shell backticks
             out, err = subprocess.Popen([binary, '-V'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
             for stream in (out, err):
                 match = version.search(stream)
                 if match:
                     versiontuple = tuple(map(int, match.group(1).split('.')))
-                    if versiontuple >= min_version_tuple:
-                        return binary
+                    if versiontuple >= min_version:
+                        return which(binary)
         except:
             pass
-    # if that all fails, fall back to "python"
-    return "python"
+
+    raise Exception('could not find suitable Python (version >= %s)' % '.'.join(str(v) for v in min_version))
+
+def smoke_command(*args):
+    # return a list of arguments that comprises a complete
+    # invocation of smoke.py
+    here = os.path.dirname(__file__)
+    smoke_py = os.path.abspath(os.path.join(here, 'smoke.py'))
+    # the --with-cleanbb argument causes smoke.py to run
+    # buildscripts/cleanbb.py before each test phase; this
+    # prevents us from running out of disk space on slaves
+    return [find_python(), smoke_py, '--with-cleanbb'] + list(args)
+
+def run_smoke_command(*args):
+    # to run a command line script from a scons Alias (or any
+    # Action), the command sequence must be enclosed in a list,
+    # otherwise SCons treats it as a list of dependencies.
+    return [smoke_command(*args)]
+
+# unicode is a pain. some strings cannot be unicode()'d
+# but we want to just preserve the bytes in a human-readable
+# fashion. this codec error handler will substitute the
+# repr() of the offending bytes into the decoded string
+# at the position they occurred
+def replace_with_repr(unicode_error):
+    offender = unicode_error.object[unicode_error.start:unicode_error.end]
+    return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end)
+
+codecs.register_error('repr', replace_with_repr)
+
+def unicode_dammit(string, encoding='utf8'):
+    # convert a string to a unicode, using the Python
+    # representation of non-ascii bytes when necessary
+    #
+    # name inpsired by BeautifulSoup's "UnicodeDammit"
+    return string.decode(encoding, 'repr')