summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorApollon Oikonomopoulos <apoikos@debian.org>2016-01-14 00:10:06 +0200
committerApollon Oikonomopoulos <apollon@skroutz.gr>2016-01-14 00:10:06 +0200
commit374e1947abcd3e127a2a613aff73ecffdb9199ea (patch)
treed83973c3c9802450acd5b5e86fe0d4e8e60a3a1b /buildscripts
parent65585c90b12d6523bea75a2aebaae2a2fdf9e641 (diff)
Imported Upstream version 2.6.11upstream/2.6.11
Diffstat (limited to 'buildscripts')
-rwxr-xr-xbuildscripts/build_and_test_client.py75
-rw-r--r--buildscripts/buildlogger.py4
-rw-r--r--buildscripts/cleanbb.py57
-rwxr-xr-xbuildscripts/consolidate-repos-enterprise.sh141
-rwxr-xr-xbuildscripts/consolidate-repos.sh78
-rwxr-xr-xbuildscripts/errorcodes.py398
-rwxr-xr-xbuildscripts/make_archive.py126
-rw-r--r--buildscripts/make_vcxproj.py145
-rw-r--r--buildscripts/moduleconfig.py4
-rwxr-xr-xbuildscripts/packager-enterprise.py701
-rw-r--r--buildscripts/packager.py467
-rwxr-xr-xbuildscripts/packaging/msi/.gitignore2
-rw-r--r--buildscripts/packaging/msi/Banner.bmpbin0 -> 114432 bytes
-rw-r--r--buildscripts/packaging/msi/Dialog.bmpbin0 -> 615320 bytes
-rwxr-xr-xbuildscripts/packaging/msi/GNU-AGPL-3.0.rtf241
-rw-r--r--buildscripts/packaging/msi/Installer_Icon_16x16.icobin0 -> 1150 bytes
-rw-r--r--buildscripts/packaging/msi/Installer_Icon_32x32.icobin0 -> 5430 bytes
-rwxr-xr-xbuildscripts/packaging/msi/MongoDB.wixproj60
-rwxr-xr-xbuildscripts/packaging/msi/MongoDBMsi.sln51
-rwxr-xr-xbuildscripts/packaging/msi/MongoDB_64.wixproj70
-rwxr-xr-xbuildscripts/packaging/msi/README.md36
-rwxr-xr-xbuildscripts/packaging/msi/build32bitmsi.bat100
-rw-r--r--buildscripts/packaging/msi/build64bit2008R2msi.bat37
-rwxr-xr-xbuildscripts/packaging/msi/build64bitmsi.bat37
-rw-r--r--buildscripts/packaging/msi/buildenterprisemsi.bat63
-rwxr-xr-xbuildscripts/s3sign.py105
-rw-r--r--buildscripts/setup_multiversion_mongodb.py134
-rwxr-xr-xbuildscripts/smoke.py646
-rw-r--r--buildscripts/utils.py6
-rw-r--r--buildscripts/vcxproj.header216
30 files changed, 3115 insertions, 885 deletions
diff --git a/buildscripts/build_and_test_client.py b/buildscripts/build_and_test_client.py
deleted file mode 100755
index 1b97f623cc5..00000000000
--- a/buildscripts/build_and_test_client.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/python
-
-'''Script to attempt an isolated build of the C++ driver and its examples.
-
-Working directory must be the repository root.
-
-Usage:
-
-./buildscripts/build_and_test_client.py <mongo client archive file> [optional scons arguments]
-
-The client is built in a temporary directory, and the sample programs are run against a mongod
-instance found in the current working directory. The temporary directory and its contents are
-destroyed at the end of execution.
-'''
-
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import tarfile
-import zipfile
-
-import utils
-
-def main(args):
- archive_file = args[1]
- scons_args = args[2:]
- build_and_test(archive_file, scons_args)
-
-def build_and_test(archive_name, scons_args):
- work_dir = tempfile.mkdtemp()
- try:
- archive = open_archive(archive_name)
- extracted_root = extract_archive(work_dir, archive)
- run_scons(extracted_root, scons_args)
- smoke_client(extracted_root)
- finally:
- shutil.rmtree(work_dir)
-
-def open_tar(archive_name):
- return tarfile.open(archive_name, 'r')
-
-def open_zip(archive_name):
- class ZipWrapper(zipfile.ZipFile):
- def getnames(self):
- return self.namelist()
- return ZipWrapper(archive_name, 'r')
-
-def open_archive(archive_name):
- try:
- return open_tar(archive_name)
- except:
- return open_zip(archive_name)
-
-def extract_archive(work_dir, archive_file):
- archive_file.extractall(path=work_dir)
- return os.path.join(
- work_dir,
- os.path.dirname([n for n in archive_file.getnames() if n.endswith('SConstruct')][0])
- )
-
-def run_scons(extracted_root, scons_args):
- rc = subprocess.call(['scons', '-C', extracted_root, ] + scons_args + ['clientTests'])
- if rc is not 0:
- sys.exit(rc)
-
-def smoke_client(extracted_root):
- rc = subprocess.call(utils.smoke_command("--test-path", extracted_root, "client"))
- if rc is not 0:
- sys.exit(rc)
-
-if __name__ == '__main__':
- main(sys.argv)
- sys.exit(0)
diff --git a/buildscripts/buildlogger.py b/buildscripts/buildlogger.py
index 9f3feacbe55..a31b3e2dfa1 100644
--- a/buildscripts/buildlogger.py
+++ b/buildscripts/buildlogger.py
@@ -90,7 +90,7 @@ for path in possible_paths:
pass
-URL_ROOT = 'http://buildlogs.mongodb.org/'
+URL_ROOT = os.environ.get('BUILDLOGGER_URL', 'http://buildlogs.mongodb.org/')
TIMEOUT_SECONDS = 10
socket.setdefaulttimeout(TIMEOUT_SECONDS)
@@ -251,7 +251,7 @@ def run_and_echo(command):
return proc.returncode
class LogAppender(object):
- def __init__(self, callback, args, send_after_lines=200, send_after_seconds=2):
+ def __init__(self, callback, args, send_after_lines=2000, send_after_seconds=10):
self.callback = callback
self.callback_args = args
diff --git a/buildscripts/cleanbb.py b/buildscripts/cleanbb.py
index dd52020d7e4..fee7efdc0c1 100644
--- a/buildscripts/cleanbb.py
+++ b/buildscripts/cleanbb.py
@@ -1,20 +1,14 @@
+#!/usr/bin/env python
+import re
import sys
import os, os.path
import utils
import time
from optparse import OptionParser
-# set cwd to the root mongo dir, one level up from this
-# file's location (if we're not already running from there)
-cwd = os.getcwd()
-if os.path.basename(cwd) == 'buildscripts':
- cwd = os.path.dirname(cwd)
+def shouldKill( c, root=None ):
-print( "cwd [" + cwd + "]" )
-
-def shouldKill( c ):
-
if "smoke.py" in c:
return False
@@ -24,10 +18,13 @@ def shouldKill( c ):
if "java" in c:
return False
- if c.find( cwd ) >= 0:
+ # if root directory is provided, see if command line matches mongod process running
+ # with the same data directory
+
+ if root and re.compile("(\W|^)mongod(.exe)?\s+.*--dbpath(\s+|=)%s(\s+|$)" % root).search( c ):
return True
- if ( c.find( "buildbot" ) >= 0 or c.find( "slave" ) ) and c.find( "/mongo/" ) >= 0:
+ if ( c.find( "buildbot" ) >= 0 or c.find( "slave" ) >= 0 ) and c.find( "/mongo/" ) >= 0:
return True
if c.find( "xml-data/build-dir" ) >= 0: # for bamboo
@@ -35,8 +32,7 @@ def shouldKill( c ):
return False
-def killprocs( signal="" ):
-
+def killprocs( signal="", root=None ):
killed = 0
if sys.platform == 'win32':
@@ -53,9 +49,9 @@ def killprocs( signal="" ):
for x in l:
x = x.lstrip()
- if not shouldKill( x ):
+ if not shouldKill( x, root=root ):
continue
-
+
pid = x.split( " " )[0]
print( "killing: " + x )
utils.execsys( "/bin/kill " + signal + " " + pid )
@@ -64,23 +60,38 @@ def killprocs( signal="" ):
return killed
+def tryToRemove(path):
+ for _ in range(60):
+ try:
+ os.remove(path)
+ return True
+ except OSError, e:
+ errno = getattr(e, 'winerror', None)
+ # check for the access denied and file in use WindowsErrors
+ if errno in (5, 32):
+ print("os.remove(%s) failed, retrying in one second." % path)
+ time.sleep(1)
+ else:
+ raise e
+ return False
+
+
def cleanup( root , nokill ):
-
if nokill:
print "nokill requested, not killing anybody"
else:
- if killprocs() > 0:
+ if killprocs( root=root ) > 0:
time.sleep(3)
- killprocs("-9")
+ killprocs( "-9", root=root )
# delete all regular files, directories can stay
# NOTE: if we delete directories later, we can't delete diskfulltest
for ( dirpath , dirnames , filenames ) in os.walk( root , topdown=False ):
- for x in filenames:
+ for x in filenames:
foo = dirpath + "/" + x
- print( "removing: " + foo )
- os.remove( foo )
-
+ if os.path.exists(foo):
+ if not tryToRemove(foo):
+ raise Exception("Couldn't remove file '%s' after 60 seconds" % foo)
if __name__ == "__main__":
parser = OptionParser(usage="read the script")
@@ -90,5 +101,5 @@ if __name__ == "__main__":
root = "/data/db/"
if len(args) > 0:
root = args[0]
-
+
cleanup( root , options.nokill )
diff --git a/buildscripts/consolidate-repos-enterprise.sh b/buildscripts/consolidate-repos-enterprise.sh
new file mode 100755
index 00000000000..33754005600
--- /dev/null
+++ b/buildscripts/consolidate-repos-enterprise.sh
@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# consolidate-repos-enterprise.sh
+#
+# Create new repo directory under /var/www-enterprise/repo.consolidated
+# containing every deb and every rpm under /var/www-enterprise/ with proper
+# repo metadata for apt and yum
+#
+
+source_dir=/var/www-enterprise/
+
+repodir=/var/www-enterprise/repo.consolidated
+
+gpg_recip='<richard@10gen.com>'
+
+stable_branch="2.6"
+unstable_branch="2.7"
+
+echo "Using directory: $repodir"
+
+# set up repo dirs if they don't exist
+#
+mkdir -p "$repodir/apt/ubuntu"
+mkdir -p "$repodir/apt/debian"
+mkdir -p "$repodir/yum/redhat"
+
+# to support different $releasever values in yum repo configurations
+#
+if [ ! -e "$repodir/yum/redhat/6Server" ]
+then
+ ln -s 6 "$repodir/yum/redhat/6Server"
+fi
+
+if [ ! -e "$repodir/yum/redhat/5Server" ]
+then
+ ln -s 5 "$repodir/yum/redhat/5Server"
+fi
+
+echo "Scanning and copying package files from $source_dir"
+echo ". = skipping existing file, @ = copying file"
+for package in $(find "$source_dir" -not \( -path "$repodir" -prune \) -and \( -name \*.rpm -o -name \*.deb -o -name Release \))
+do
+ new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www-enterprise\/[^\/]*//;')"
+ # skip if the directory structure looks weird
+ #
+ if echo "$new_package_location" | grep -q /repo/
+ then
+ continue
+ fi
+
+ # skip if not enterprise package
+ #
+ if ! echo "$new_package_location" | grep -q enterprise
+ then
+ continue
+ fi
+ # skip if it's already there
+ #
+ if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]
+ then
+ echo -n .
+ else
+ mkdir -p "$(dirname "$new_package_location")"
+ echo -n @
+ cp "$package" "$new_package_location"
+ fi
+done
+echo
+
+# packages are in place, now create metadata
+#
+for debian_dir in "$repodir"/apt/ubuntu "$repodir"/apt/debian
+do
+ cd "$debian_dir"
+ for section_dir in $(find dists -type d -name multiverse -o -name main)
+ do
+ for arch_dir in "$section_dir"/{binary-i386,binary-amd64}
+ do
+ echo "Generating Packages file under $debian_dir/$arch_dir"
+ if [ ! -d $arch_dir ]
+ then
+ mkdir $arch_dir
+ fi
+ dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages
+ gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz
+ done
+ done
+
+ for release_file in $(find "$debian_dir" -name Release)
+ do
+ release_dir=$(dirname "$release_file")
+ echo "Generating Release file under $release_dir"
+ cd $release_dir
+ tempfile=$(mktemp /tmp/ReleaseXXXXXX)
+ tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
+ mv Release $tempfile
+ head -7 $tempfile > $tempfile2
+ apt-ftparchive release . >> $tempfile2
+ cp $tempfile2 Release
+ chmod 644 Release
+ rm Release.gpg
+ echo "Signing Release file"
+ gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release
+ done
+done
+
+# Create symlinks for stable and unstable branches
+#
+# Examples:
+#
+# /var/www-enterprise/repo.consolidated/yum/redhat/5/mongodb-enterprise/unstable -> 2.5
+# /var/www-enterprise/repo.consolidated/yum/redhat/6/mongodb-enterprise/unstable -> 2.5
+# /var/www-enterprise/repo.consolidated/apt/ubuntu/dists/precise/mongodb-enterprise/unstable -> 2.5
+# /var/www-enterprise/repo.consolidated/apt/debian/dists/wheezy/mongodb-enterprise/unstable -> 2.5
+#
+for unstable_branch_dir in "$repodir"/yum/redhat/*/*/$unstable_branch "$repodir"/apt/debian/dists/*/*/$unstable_branch "$repodir"/apt/ubuntu/dists/*/*/$unstable_branch
+do
+ full_unstable_path=$(dirname "$unstable_branch_dir")/unstable
+ if [ -e "$unstable_branch_dir" -a ! -e "$full_unstable_path" ]
+ then
+ echo "Linking unstable branch directory $unstable_branch_dir to $full_unstable_path"
+ ln -s $unstable_branch $full_unstable_path
+ fi
+done
+
+for stable_branch_dir in "$repodir"/yum/redhat/*/*/$stable_branch "$repodir"/apt/debian/dists/*/*/$stable_branch "$repodir"/apt/ubuntu/dists/*/*/$stable_branch
+do
+ full_stable_path=$(dirname "$stable_branch_dir")/stable
+ if [ -e "$stable_branch_dir" -a ! -e "$full_stable_path" ]
+ then
+ echo "Linking stable branch directory $stable_branch_dir to $full_stable_path"
+ ln -s $stable_branch $full_stable_path
+ fi
+done
+
+for rpm_dir in $(find "$repodir"/yum/redhat "$repodir"/zypper/suse -type d -name x86_64 -o -name i386)
+do
+ echo "Generating redhat repo metadata under $rpm_dir"
+ cd "$rpm_dir"
+ createrepo .
+done
diff --git a/buildscripts/consolidate-repos.sh b/buildscripts/consolidate-repos.sh
new file mode 100755
index 00000000000..6c52508aa85
--- /dev/null
+++ b/buildscripts/consolidate-repos.sh
@@ -0,0 +1,78 @@
+#!/bin/sh
+#
+# consolidate-repos.sh
+#
+# Create new repo directory under /var/www/repo.consolidated
+# containing every deb and every rpm under /var/www/ with proper
+# repo metadata for Debian and Ubuntu
+#
+
+source_dir=/var/www
+
+repodir=/var/www/repo.consolidated
+
+gpg_recip='<richard@10gen.com>'
+
+echo "Using directory: $repodir"
+
+mkdir -p "$repodir"
+
+echo "Scanning and copying package files from $source_dir"
+echo ". = skipping existing file, @ = copying file"
+for package in $(find "$source_dir/" -not \( -path "$repodir" -prune \) -not -path \*enterprise\* -and \( -name \*.rpm -o -name \*.deb -o -name Release \))
+do
+ new_package_location="$repodir$(echo "$package" | sed 's/\/var\/www\/[^\/]*//;')"
+
+ # skip if the directory structure looks weird
+ #
+ if echo "$new_package_location" | grep -q /repo/
+ then
+ continue
+ fi
+
+ # skip if it's already there
+ #
+ if [ -e "$new_package_location" -a "$(basename "$package")" != "Release" ]
+ then
+ echo -n .
+ else
+ mkdir -p "$(dirname "$new_package_location")"
+ echo -n @
+ cp "$package" "$new_package_location"
+ fi
+done
+echo
+
+# packages are in place, now create metadata
+#
+for debian_dir in "$repodir"/ubuntu-* "$repodir"/debian-*
+do
+ cd "$debian_dir"
+ for arch_dir in dists/dist/10gen/*
+ do
+ echo "Generating Packages file under $debian_dir/$arch_dir"
+ dpkg-scanpackages --multiversion "$arch_dir" > "$arch_dir"/Packages
+ gzip -9c "$arch_dir"/Packages > "$arch_dir"/Packages.gz
+ done
+
+ release_dir="$debian_dir"/dists/dist
+ echo "Generating Release file under $release_dir"
+ cd $release_dir
+ tempfile=$(mktemp /tmp/ReleaseXXXXXX)
+ tempfile2=$(mktemp /tmp/ReleaseXXXXXX)
+ mv Release $tempfile
+ head -9 $tempfile > $tempfile2
+ apt-ftparchive release . >> $tempfile2
+ cp $tempfile2 Release
+ chmod 644 Release
+ rm Release.gpg
+ echo "Signing Release file"
+ gpg -r "$gpg_recip" --no-secmem-warning -abs --output Release.gpg Release
+done
+
+for rpm_dir in "$repodir"/redhat/os/* "$repodir"/suse/os/*
+do
+ echo "Generating rpm repo metadata under $redhat_dir"
+ cd "$rpm_dir"
+ createrepo .
+done
diff --git a/buildscripts/errorcodes.py b/buildscripts/errorcodes.py
index e49073cadce..826515e4478 100755
--- a/buildscripts/errorcodes.py
+++ b/buildscripts/errorcodes.py
@@ -1,16 +1,30 @@
#!/usr/bin/env python
+"""Produces a report of all assertions in the MongoDB server codebase.
+
+Parses .cpp files for assertions and verifies assertion codes are distinct.
+Optionally replaces zero codes in source code with new distinct values.
+"""
+
import os
-import sys
import re
import utils
+from collections import defaultdict, namedtuple
+from optparse import OptionParser
+
+ASSERT_NAMES = [ "uassert" , "massert", "fassert", "fassertFailed" ]
+MINIMUM_CODE = 10000
+
+codes = []
+# Each AssertLocation identifies the C++ source location of an assertion
+AssertLocation = namedtuple( "AssertLocation", ['sourceFile', 'lineNum', 'lines', 'code'] )
-assertNames = [ "uassert" , "massert", "fassert", "fassertFailed" ]
+# Of historical interest only
def assignErrorCodes():
- cur = 10000
- for root in assertNames:
+ cur = MINIMUM_CODE
+ for root in ASSERT_NAMES:
for x in utils.getAllSourceFiles():
print( x )
didAnything = False
@@ -29,151 +43,275 @@ def assignErrorCodes():
out.close()
-codes = []
+def parseSourceFiles( callback ):
+ """Walks MongoDB sourcefiles and invokes callback for each AssertLocation found."""
-def readErrorCodes( callback, replaceZero = False ):
-
quick = [ "assert" , "Exception"]
- ps = [ re.compile( "(([umsgf]asser(t|ted))) *\(( *)(\d+)" ) ,
- re.compile( "((User|Msg|MsgAssertion)Exceptio(n))\(( *)(\d+)" ),
- re.compile( "((fassertFailed)()) *\(( *)(\d+)" )
- ]
-
- bad = [ re.compile( "^\s*assert *\(" ) ]
-
- for x in utils.getAllSourceFiles():
-
- needReplace = [False]
- lines = []
- lastCodes = [0]
- lineNum = 1
-
- for line in open( x ):
-
- found = False
- for zz in quick:
- if line.find( zz ) >= 0:
- found = True
- break
-
- if found:
-
- if x.find( "src/mongo/" ) >= 0:
- for b in bad:
- if len(b.findall( line )) > 0:
- print( x )
- print( line )
- raise Exception( "you can't use a bare assert" )
-
- for p in ps:
-
- def repl( m ):
- m = m.groups()
-
- start = m[0]
- spaces = m[3]
- code = m[4]
- if code == '0' and replaceZero :
- code = getNextCode( lastCodes )
- lastCodes.append( code )
- code = str( code )
- needReplace[0] = True
-
- print( "Adding code " + code + " to line " + x + ":" + str( lineNum ) )
-
- else :
- codes.append( ( x , lineNum , line , code ) )
- callback( x , lineNum , line , code )
-
- return start + "(" + spaces + code
-
- line = re.sub( p, repl, line )
- # end if ps loop
-
- if replaceZero : lines.append( line )
- lineNum = lineNum + 1
-
- if replaceZero and needReplace[0] :
- print( "Replacing file " + x )
- of = open( x + ".tmp", 'w' )
- of.write( "".join( lines ) )
- of.close()
- os.remove(x)
- os.rename( x + ".tmp", x )
-
-
-def getNextCode( lastCodes = [0] ):
- highest = [max(lastCodes)]
- def check( fileName , lineNum , line , code ):
- code = int( code )
- if code > highest[0]:
- highest[0] = code
- readErrorCodes( check )
- return highest[0] + 1
+ patterns = [
+ re.compile( r"[umsgf]asser(?:t|ted) *\( *(\d+)" ) ,
+ re.compile( r"(?:User|Msg|MsgAssertion)Exception *\( *(\d+)" ),
+ re.compile( r"fassertFailed(?:NoTrace)? *\( *(\d+)" )
+ ]
+
+ bad = [ re.compile( r"^\s*assert *\(" ) ]
+
+ for sourceFile in utils.getAllSourceFiles():
+ if not sourceFile.find( "src/mongo/" ) >= 0:
+ # Skip generated sources
+ continue
+
+ with open(sourceFile) as f:
+ line_iterator = enumerate(f, 1)
+ for (lineNum, line) in line_iterator:
+
+ # See if we can skip regexes
+ if not any([zz in line for zz in quick]):
+ continue
+
+ for b in bad:
+ if b.search(line):
+ print( "%s\n%d" % (sourceFile, line) )
+ msg = "Bare assert prohibited. Replace with [umwdf]assert"
+ raise Exception(msg)
+
+ # no more than one pattern should ever match
+ matches = [x for x in [p.search(line) for p in patterns]
+ if x]
+ assert len(matches) <= 1, matches
+ if matches:
+ match = matches[0]
+ code = match.group(1)
+ span = match.span()
+
+ # Advance to statement terminator iff not on this line
+ lines = [line.strip()]
+
+ if not isTerminated(lines):
+ for (_lineNum, line) in line_iterator:
+ lines.append(line.strip())
+ if isTerminated(lines):
+ break
+
+ thisLoc = AssertLocation(sourceFile, lineNum, lines, code)
+ callback( thisLoc )
+
+ # end for sourceFile loop
+
+
+def isTerminated( lines ):
+ """Given .cpp/.h source lines as text, determine if assert is terminated."""
+ x = " ".join(lines)
+ return ';' in x \
+ or x.count('(') - x.count(')') <= 0
+
+
+def getNextCode():
+ """Finds next unused assertion code.
+
+ Called by: SConstruct and main()
+ Since SConstruct calls us, codes[] must be global OR WE REPARSE EVERYTHING
+ """
+ if not len(codes) > 0:
+ readErrorCodes()
+
+ highest = reduce( lambda x, y: max(int(x), int(y)),
+ (loc.code for loc in codes) )
+ return highest + 1
+
def checkErrorCodes():
+ """SConstruct expects a boolean response from this function.
+ """
+ (codes, errors) = readErrorCodes()
+ return len( errors ) == 0
+
+
+def readErrorCodes():
+ """Defines callback, calls parseSourceFiles() with callback,
+ and saves matches to global codes list.
+ """
seen = {}
errors = []
- def checkDups( fileName , lineNum , line , code ):
- if code in seen:
- print( "DUPLICATE IDS" )
- print( "%s:%d:%s %s" % ( fileName , lineNum , line.strip() , code ) )
- print( "%s:%d:%s %s" % seen[code] )
- errors.append( seen[code] )
- seen[code] = ( fileName , lineNum , line , code )
- readErrorCodes( checkDups, True )
- return len( errors ) == 0
-
-def getBestMessage( err , start ):
- err = err.partition( start )[2]
- if not err:
- return ""
- err = err.partition( "\"" )[2]
+ dups = defaultdict(list)
+
+ # define callback
+ def checkDups( assertLoc ):
+ codes.append( assertLoc )
+ code = assertLoc.code
+
+ if not code in seen:
+ seen[code] = assertLoc
+ else:
+ if not code in dups:
+ # on first duplicate, add original to dups, errors
+ dups[code].append( seen[code] )
+ errors.append( seen[code] )
+
+ dups[code].append( assertLoc )
+ errors.append( assertLoc )
+
+ parseSourceFiles( checkDups )
+
+ if seen.has_key("0"):
+ code = "0"
+ bad = seen[code]
+ errors.append( bad )
+ print( "ZERO_CODE:" )
+ print( " %s:%d:%s" % (bad.sourceFile, bad.lineNum, bad.lines[0]) )
+
+ for code, locations in dups.items():
+ print( "DUPLICATE IDS: %s" % code )
+ for loc in locations:
+ print( " %s:%d:%s" % (loc.sourceFile, loc.lineNum, loc.lines[0]) )
+
+ return (codes, errors)
+
+
+def replaceBadCodes( errors, nextCode ):
+ """Modifies C++ source files to replace invalid assertion codes.
+ For now, we only modify zero codes.
+
+ Args:
+ errors: list of AssertLocation
+ nextCode: int, next non-conflicting assertion code
+ """
+ zero_errors = [e for e in errors if int(e.code) == 0]
+ skip_errors = [e for e in errors if int(e.code) != 0]
+
+ for loc in skip_errors:
+ print "SKIPPING NONZERO code=%s: %s:%s" % (loc.code, loc.sourceFile, loc.lineNum)
+
+ for assertLoc in zero_errors:
+ (sourceFile, lineNum, lines, code) = assertLoc
+ print "UPDATING_FILE: %s:%s" % (sourceFile, lineNum)
+
+ ln = lineNum - 1
+
+ with open(sourceFile, 'r+') as f:
+ fileLines = f.readlines()
+ line = fileLines[ln]
+ print "LINE_%d_BEFORE:%s" % (lineNum, line.rstrip())
+ line = re.sub(r'(\( *)(\d+)',
+ r'\g<1>' + str(nextCode),
+ line)
+ print "LINE_%d_AFTER :%s" % (lineNum, line.rstrip())
+ fileLines[ln] = line
+ f.seek(0)
+ f.writelines(fileLines)
+ nextCode += 1
+
+
+def getBestMessage( lines , codeStr ):
+ """Extracts message from one AssertionLocation.lines entry
+
+ Args:
+ lines: list of contiguous C++ source lines
+ codeStr: assertion code found in first line
+ """
+ line = lines if isinstance(lines, str) else " ".join(lines)
+
+ err = line.partition( codeStr )[2]
if not err:
return ""
- err = err.rpartition( "\"" )[0]
- if not err:
+
+ # Trim to outer quotes
+ m = re.search(r'"(.*)"', err)
+ if not m:
return ""
- return err
-
-def genErrorOutput():
-
- if os.path.exists( "docs/errors.md" ):
- i = open( "docs/errors.md" , "r" )
-
-
- out = open( "docs/errors.md" , 'wb' )
- out.write( "MongoDB Error Codes\n==========\n\n\n" )
+ err = m.group(1)
+
+ # Trim inner quote pairs
+ err = re.sub(r'" +"', '', err)
+ err = re.sub(r'" *<< *"', '', err)
+ err = re.sub(r'" *<<[^<]+<< *"', '<X>', err)
+ err = re.sub(r'" *\+[^+]+\+ *"', '<X>', err)
+
+ # Trim escaped quotes
+ err = re.sub(r'\\"', '', err)
+
+ # Iff doublequote still present, trim that and any trailing text
+ err = re.sub(r'".*$', '', err)
+
+ return err.strip()
+
+
+def writeMarkdownReport( codes, outfile ):
+ """Write errors.md report to filesystem in Markdown format
+
+ Args:
+ codes: list of AssertLocation
+ outfile: string path to a file to overwrite
+ """
+
+ baseurl = "http://github.com/mongodb/mongo/blob/master"
+
+ if os.path.exists(outfile):
+ i = open(outfile, "r" )
+ i.close()
+
+ out = open(outfile, 'wb')
+ out.write("MongoDB Error Codes\n")
+ out.write("===================\n\n")
+ out.write("This file is generated by errorcodes.py. Do not edit.\n\n")
prev = ""
seen = {}
-
- codes.sort( key=lambda x: x[0]+"-"+x[3] )
- for f,l,line,num in codes:
- if num in seen:
+
+ # Sort by sourceFile, then code
+ codes.sort( key=lambda loc: loc.sourceFile+"-"+loc.code )
+
+ for assertLoc in codes:
+ if assertLoc.code in seen:
continue
- seen[num] = True
+ seen[assertLoc.code] = True
+
+ (sourceFile, lineNum, lines, code) = assertLoc
+
+ if sourceFile.startswith("./"):
+ sourceFile = sourceFile[2:]
- if f.startswith( "./" ):
- f = f[2:]
+ if sourceFile != prev:
+ out.write("\n\n%s\n----\n" % sourceFile)
+ prev = sourceFile
- if f != prev:
- out.write( "\n\n" )
- out.write( f + "\n----\n" )
- prev = f
+ url = "%s/%s#L%s" % (baseurl, sourceFile, lineNum)
+ message = getBestMessage(lines, str(code))
+ out.write("* %s [code](%s) %s\n" % (code, url, message))
- url = "http://github.com/mongodb/mongo/blob/master/" + f + "#L" + str(l)
-
- out.write( "* " + str(num) + " [code](" + url + ") " + getBestMessage( line , str(num) ) + "\n" )
-
out.write( "\n" )
out.close()
-if __name__ == "__main__":
- ok = checkErrorCodes()
- print( "ok:" + str( ok ) )
- print( "next: " + str( getNextCode() ) )
+
+def main():
+ parser = OptionParser(description=__doc__.strip())
+ parser.add_option("--fix", dest="replace",
+ action="store_true", default=False,
+ help="Fix zero codes in source files [default: %default]")
+ parser.add_option("-o", dest="outfile",
+ default="docs/errors.md",
+ help="Report file [default: %default]")
+ (options, args) = parser.parse_args()
+
+ (codes, errors) = readErrorCodes()
+ ok = len(errors) == 0
+ next = getNextCode()
+
+ print("ok: %s" % ok)
+ print("next: %s" % next)
+
if ok:
- genErrorOutput()
+ writeMarkdownReport(codes, options.outfile)
+ elif options.replace:
+ replaceBadCodes(errors, next)
+ else:
+ print ERROR_HELP
+
+ERROR_HELP = """
+ERRORS DETECTED. To correct, run "buildscripts/errorcodes.py --fix" to replace zero codes.
+Other errors require manual correction.
+"""
+
+if __name__ == "__main__":
+ main()
diff --git a/buildscripts/make_archive.py b/buildscripts/make_archive.py
index 4c12e901a64..5df995b41e9 100755
--- a/buildscripts/make_archive.py
+++ b/buildscripts/make_archive.py
@@ -29,17 +29,100 @@ For a detailed usage example, see src/SConscript.client or src/mongo/SConscript.
import optparse
import os
import sys
+import shlex
+import shutil
+import zipfile
+from subprocess import (Popen, PIPE, STDOUT)
def main(argv):
- opts = parse_options(argv[1:])
- archive = open_archive_for_write(opts.output_filename, opts.archive_format)
+ args = []
+ for arg in argv[1:]:
+ if arg.startswith("@"):
+ file_name = arg[1:]
+ f_handle = open(file_name, "r")
+ args.extend(s1.strip('"') for s1 in shlex.split(f_handle.readline(), posix=False))
+ f_handle.close()
+ else:
+ args.append(arg)
+
+ opts = parse_options(args)
+ if opts.archive_format in ('tar', 'tgz'):
+ make_tar_archive(opts)
+ elif opts.archive_format in ('zip'):
+ make_zip_archive(opts)
+ else:
+ raise ValueError('Unsupported archive format "%s"' % opts.archive_format)
+
+def delete_directory(dir):
+ '''Recursively deletes a directory and its contents.
+ '''
+ try:
+ shutil.rmtree(dir)
+ except Exception:
+ pass
+
+def make_tar_archive(opts):
+ '''Given the parsed options, generates the 'opt.output_filename'
+ tarball containing all the files in 'opt.input_filename' renamed
+ according to the mappings in 'opts.transformations'.
+
+ e.g. for an input file named "a/mongo/build/DISTSRC", and an
+ existing transformation {"a/mongo/build": "release"}, the input
+ file will be written to the tarball as "release/DISTSRC"
+
+ All files to be compressed are copied into new directories as
+ required by 'opts.transformations'. Once the tarball has been
+ created, all temporary directory structures created for the
+ purposes of compressing, are removed.
+ '''
+ tar_options = "cvf"
+ if opts.archive_format is 'tgz':
+ tar_options += "z"
+
+ # clean and create a temp directory to copy files to
+ enclosing_archive_directory = os.path.join("build", "archive")
+ delete_directory(enclosing_archive_directory)
+ os.makedirs(enclosing_archive_directory)
+ output_tarfile = os.path.join(os.getcwd(), opts.output_filename)
+
+ tar_command = ["tar", tar_options, output_tarfile]
+
+ for input_filename in opts.input_filenames:
+ preferred_filename = get_preferred_filename(input_filename, opts.transformations)
+ temp_file_location = os.path.join(enclosing_archive_directory, preferred_filename)
+ enclosing_file_directory = os.path.dirname(temp_file_location)
+ if not os.path.exists(enclosing_file_directory):
+ os.makedirs(enclosing_file_directory)
+ print "copying %s => %s" % (input_filename, temp_file_location)
+ shutil.copy2(input_filename, temp_file_location)
+ tar_command.append(preferred_filename)
+
+ print " ".join(tar_command)
+ # execute the full tar command
+ run_directory = os.path.join(os.getcwd(), enclosing_archive_directory)
+ proc = Popen(tar_command, stdout=PIPE, stderr=STDOUT, bufsize=0, cwd=run_directory)
+ proc.wait()
+
+ # delete temp directory
+ delete_directory(enclosing_archive_directory)
+
+def make_zip_archive(opts):
+ '''Given the parsed options, generates the 'opt.output_filename'
+ zipfile containing all the files in 'opt.input_filename' renamed
+ according to the mappings in 'opts.transformations'.
+
+ All files in 'opt.output_filename' are renamed before being
+ written into the zipfile.
+ '''
+ archive = open_zip_archive_for_write(opts.output_filename)
try:
for input_filename in opts.input_filenames:
archive.add(input_filename, arcname=get_preferred_filename(input_filename,
- opts.transformations))
+ opts.transformations))
finally:
archive.close()
+
def parse_options(args):
parser = optparse.OptionParser()
parser.add_option('-o', dest='output_filename', default=None,
@@ -82,32 +165,25 @@ def parse_options(args):
return opts
-def open_archive_for_write(filename, archive_format):
- '''Open a tar or zip archive for write, with the given format, and return it.
-
- The type of archive is determined by the "archive_format" parameter, which should be
- "tar", "tgz" (for gzipped tar) or "zip".
+def open_zip_archive_for_write(filename):
+ '''Open a zip archive for writing and return it.
'''
-
- if archive_format in ('tar', 'tgz'):
- import tarfile
- mode = 'w'
- if archive_format is 'tgz':
- mode += '|gz'
- return tarfile.open(filename, mode)
- if archive_format is 'zip':
- import zipfile
- # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical,
- # for our purposes. WrappedZipFile is a minimal adapter class.
- class WrappedZipFile(zipfile.ZipFile):
- def add(self, filename, arcname):
- return self.write(filename, arcname)
- return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
- raise ValueError('Unsupported archive format "%s"' % archive_format)
+ # Infuriatingly, Zipfile calls the "add" method "write", but they're otherwise identical,
+ # for our purposes. WrappedZipFile is a minimal adapter class.
+ class WrappedZipFile(zipfile.ZipFile):
+ def add(self, filename, arcname):
+ return self.write(filename, arcname)
+ return WrappedZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
def get_preferred_filename(input_filename, transformations):
+ '''Does a prefix subsitution on 'input_filename' for the
+ first matching transformation in 'transformations' and
+ returns the substituted string
+ '''
for match, replace in transformations:
- if input_filename.startswith(match):
+ match_lower = match.lower()
+ input_filename_lower = input_filename.lower()
+ if input_filename_lower.startswith(match_lower):
return replace + input_filename[len(match):]
return input_filename
diff --git a/buildscripts/make_vcxproj.py b/buildscripts/make_vcxproj.py
new file mode 100644
index 00000000000..c7aefcfc857
--- /dev/null
+++ b/buildscripts/make_vcxproj.py
@@ -0,0 +1,145 @@
+# generate vcxproj file(s)
+#
+# HOW TO USE
+#
+# scons --clean
+# # verify build/* is empty...
+# scons TARGET.exe > out
+# python buildscripts/make_vcxproj.py TARGET < out > my.vcxproj
+#
+# where TARGET is your target e.g., "mongod"
+#
+# NOTES
+#
+# (1)
+# directory paths are such that it is assumed the vcxproj is in the top level project directory.
+# this is easy and likely to change...
+#
+# (2)
+# machine generated files (error_codes, action_types, ...) are, for now, copied by this script
+# into the source tree -- see note below in function pyth() as to why.
+# if those files need refreshing, run scons to generate them, and then run make_vcxproj.py again
+# to copy them over. the rebuilding of the vcxproj file in that case should be moot, it is just
+# the copying over of the updated files we really want to happen.
+#
+# (3)
+# todo: i don't think the generated vcxproj perfectly handles switching from debug to release and
+# such yet. so for example:
+#
+# scons --clean all && scons --dd --win2008plus --64 mongod.exe && python ...
+#
+# should generate a file that will work for building mongod.exe, *if* you pick win2008plus and
+# Debug and 64 bit from the drop downs. The other variations so far, ymmv.
+#
+
+import sys
+import os
+
+target = sys.argv[1]
+
+footer= """
+ </ItemGroup>
+
+ <ItemGroup>
+ <None Include="src\\mongo\\db\\mongo.ico" />
+ </ItemGroup>
+
+ <ItemGroup>
+ <ResourceCompile Include="src\\mongo\\db\\db.rc" />
+ </ItemGroup>
+
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets"></ImportGroup>
+</Project>
+"""
+
+common_defines_str = "/DBOOST_ALL_NO_LIB /DMONGO_EXPOSE_MACROS /DSUPPORT_UTF8 /D_UNICODE /DUNICODE /D_CONSOLE /D_CRT_SECURE_NO_WARNINGS /D_WIN32_WINNT=0x0502 /DMONGO_HAVE___DECLSPEC_THREAD"
+
+def get_defines(x):
+ res = set()
+ for s in x:
+ if s.startswith('/D') or s.startswith('-D'):
+ d = s[2:]
+ res.add(d)
+ return res
+
+common_defines = get_defines(common_defines_str.split(' '))
+
+f = open('buildscripts/vcxproj.header', 'r')
+header = f.read().replace("_TARGET_", target)
+print header
+
+print "<!-- common_defines -->"
+print "<ItemDefinitionGroup><ClCompile><PreprocessorDefinitions>"
+print ';'.join(common_defines) + ";%(PreprocessorDefinitions)"
+print "</PreprocessorDefinitions></ClCompile></ItemDefinitionGroup>\n"
+print "<ItemGroup>\n"
+
+# we don't use _SCONS in vcxproj files, but it's in the input to this script, so add it to common_defines
+# so that it is ignored below and not declared:
+common_defines.add("_SCONS")
+# likewise we handle DEBUG and such in the vcxproj header:
+common_defines.add("DEBUG")
+common_defines.add("_DEBUG")
+common_defines.add("V8_TARGET_ARCH_X64")
+common_defines.add("V8_TARGET_ARCH_IA32")
+common_defines.add("NTDDI_VERSION")
+common_defines.add("_WIN32_WINNT")
+
+machine_path = ""
+
+def add_globally(path):
+ print "\n</ItemGroup>\n"
+ print "<ItemDefinitionGroup><ClCompile><AdditionalIncludeDirectories>" + machine_path + "</AdditionalIncludeDirectories></ClCompile></ItemDefinitionGroup>"
+ print "<ItemGroup>\n"
+
+def say(x,line):
+ # buildinfo.cpp is for scons only -- see version.cpp for more info
+ if not "buildinfo.cpp" in x:
+ if x.startswith('build\\'):
+ #global machine_path
+ #if not machine_path:
+ # machine_path = x.split("mongo")[0]
+ #sys.stderr.write("todo: adding machine gen'd include path " + machine_path + " to vcxproj\n")
+ sys.stderr.write("adding machine gen'd file " + x + " to vcxproj\n")
+ xtra = ""
+ if "v8\\src" in x: # or machine_path:
+ xtra = "<AdditionalIncludeDirectories>"
+ # it would be better to look at the command line inclusions comprehensively instead of hard code
+ # this way, but this will get us going...
+ if "v8\\src" in x:
+ xtra += "src\\third_party\\v8\\src;"
+ #if machine_path:
+ # xtra += machine_path
+ xtra += "</AdditionalIncludeDirectories>"
+ # add /D command line items that are uncommon
+ defines = ""
+ for s in get_defines(line):
+ if s.split('=')[0] not in common_defines:
+ defines += s
+ defines += ';'
+ if defines:
+ xtra += "<PreprocessorDefinitions>" + defines + "%(PreprocessorDefinitions)</PreprocessorDefinitions>"
+ print " <ClCompile Include=\"" + x + "\">" + xtra + "</ClCompile>"
+
+from shutil import copyfile
+
+# for the machine generated files we copy them into the src build tree locally.
+# this is annoying but vstudio doesn't seem to like having parallel sets of -I include
+# paths so had to do this to make it happy
+def pyth(x):
+ for s in x:
+ if s.startswith("build") and s.endswith(".h"):
+ sys.stderr.write("copying " + s + " to src/ tree\n")
+ copyfile(s, 'src\mongo' + s.split("mongo")[1])
+
+def main ():
+ for line in sys.stdin:
+ x = line.split(' ')
+ if x[0] == "cl":
+ say(x[3],x)
+ elif "python" in x[0]:
+ pyth(x)
+ print footer
+
+main()
diff --git a/buildscripts/moduleconfig.py b/buildscripts/moduleconfig.py
index 501ec182560..5cf6c63afd7 100644
--- a/buildscripts/moduleconfig.py
+++ b/buildscripts/moduleconfig.py
@@ -62,7 +62,7 @@ def discover_modules(module_root):
return found_modules
-def configure_modules(modules, conf, env):
+def configure_modules(modules, conf):
""" Run the configure() function in the build.py python modules for each module in "modules"
(as created by discover_modules).
@@ -73,7 +73,7 @@ def configure_modules(modules, conf, env):
print "configuring module: %s" % name
root = os.path.dirname(module.__file__)
- module.configure(conf, env)
+ module.configure(conf, conf.env)
def get_module_sconscripts(modules):
sconscripts = []
diff --git a/buildscripts/packager-enterprise.py b/buildscripts/packager-enterprise.py
new file mode 100755
index 00000000000..b24a65e9448
--- /dev/null
+++ b/buildscripts/packager-enterprise.py
@@ -0,0 +1,701 @@
+#!/usr/bin/python
+
+# This program makes Debian and RPM repositories for MongoDB, by
+# downloading our tarballs of statically linked executables and
+# insinuating them into Linux packages. It must be run on a
+# Debianoid, since Debian provides tools to make RPMs, but RPM-based
+# systems don't provide debian packaging crud.
+
+# Notes:
+#
+# * Almost anything that you want to be able to influence about how a
+# package construction must be embedded in some file that the
+# packaging tool uses for input (e.g., debian/rules, debian/control,
+# debian/changelog; or the RPM specfile), and the precise details are
+# arbitrary and silly. So this program generates all the relevant
+# inputs to the packaging tools.
+#
+# * Once a .deb or .rpm package is made, there's a separate layer of
+# tools that makes a "repository" for use by the apt/yum layers of
+# package tools. The layouts of these repositories are arbitrary and
+# silly, too.
+#
+# * Before you run the program on a new host, these are the
+# prerequisites:
+#
+# apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core libsnmp15
+# echo "Now put the dist gnupg signing keys in ~root/.gnupg"
+
+import errno
+import getopt
+from glob import glob
+from packager import httpget
+import os
+import re
+import stat
+import subprocess
+import sys
+import tempfile
+import time
+import urlparse
+
+# For the moment, this program runs on the host that also serves our
+# repositories to the world, so the last thing the program does is
+# move the repositories into place. Make this be the path where the
+# web server will look for repositories.
+REPOPATH="/var/www/repo"
+
+# The MongoDB names for the architectures we support.
+ARCHES=["x86_64"]
+
+# Made up names for the flavors of distribution we package for.
+DISTROS=["suse","debian","redhat","ubuntu"]
+
+
+class Spec(object):
+ def __init__(self, specstr):
+ tup = specstr.split(":")
+ self.ver = tup[0]
+ # Hack: the second item in the tuple is treated as a suffix if
+ # it lacks an equals sign; otherwise it's the start of named
+ # parameters.
+ self.suf = None
+ if len(tup) > 1 and tup[1].find("=") == -1:
+ self.suf = tup[1]
+ # Catch-all for any other parameters to the packaging.
+ i = 2 if self.suf else 1
+ self.params = dict([s.split("=") for s in tup[i:]])
+ for key in self.params.keys():
+ assert(key in ["suffix", "revision"])
+
+ def version(self):
+ return self.ver
+
+ def version_better_than(self, version_string):
+ # FIXME: this is wrong, but I'm in a hurry.
+ # e.g., "1.8.2" < "1.8.10", "1.8.2" < "1.8.2-rc1"
+ return self.ver > version_string
+
+ def suffix(self):
+ # suffix is what we tack on after pkgbase.
+ if self.suf:
+ return self.suf
+ elif "suffix" in self.params:
+ return self.params["suffix"]
+ else:
+ return "-enterprise" if int(self.ver.split(".")[1])%2==0 else "-enterprise-unstable"
+
+
+ def pversion(self, distro):
+ # Note: Debian packages have funny rules about dashes in
+ # version numbers, and RPM simply forbids dashes. pversion
+ # will be the package's version number (but we need to know
+ # our upstream version too).
+ if re.search("^(debian|ubuntu)", distro.name()):
+ return re.sub("-", "~", self.ver)
+ elif re.search("(suse|redhat|fedora|centos)", distro.name()):
+ return re.sub("\\d+-", "", self.ver)
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+ def param(self, param):
+ if param in self.params:
+ return self.params[param]
+ return None
+
+ def branch(self):
+ """Return the major and minor portions of the specificed version.
+ For example, if the version is "2.5.5" the branch would be "2.5"
+ """
+ return ".".join(self.ver.split(".")[0:2])
+
+class Distro(object):
+ def __init__(self, string):
+ self.n=string
+
+ def name(self):
+ return self.n
+
+ def pkgbase(self):
+ return "mongodb"
+
+ def archname(self, arch):
+ if re.search("^(debian|ubuntu)", self.n):
+ return "i386" if arch.endswith("86") else "amd64"
+ elif re.search("^(suse|centos|redhat|fedora)", self.n):
+ return "i686" if arch.endswith("86") else "x86_64"
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+ def repodir(self, arch, build_os, spec):
+ """Return the directory where we'll place the package files for
+ (distro, distro_version) in that distro's preferred repository
+ layout (as distinct from where that distro's packaging building
+ tools place the package files).
+
+ Examples:
+
+ repo/apt/ubuntu/dists/precise/mongodb-enterprise/2.5/multiverse/binary-amd64
+ repo/apt/ubuntu/dists/precise/mongodb-enterprise/2.5/multiverse/binary-i386
+
+ repo/apt/ubuntu/dists/trusty/mongodb-enterprise/2.5/multiverse/binary-amd64
+ repo/apt/ubuntu/dists/trusty/mongodb-enterprise/2.5/multiverse/binary-i386
+
+ repo/apt/debian/dists/wheezy/mongodb-enterprise/2.5/main/binary-amd64
+ repo/apt/debian/dists/wheezy/mongodb-enterprise/2.5/main/binary-i386
+
+ repo/yum/redhat/6/mongodb-enterprise/2.5/x86_64
+ yum/redhat/6/mongodb-enterprise/2.5/i386
+
+ repo/zypper/suse/11/mongodb-enterprise/2.5/x86_64
+ zypper/suse/11/mongodb-enterprise/2.5/i386
+
+ """
+
+ if re.search("^(debian|ubuntu)", self.n):
+ return "repo/apt/%s/dists/%s/mongodb-enterprise/%s/%s/binary-%s/" % (self.n, self.repo_os_version(build_os), spec.branch(), self.repo_component(), self.archname(arch))
+ elif re.search("(redhat|fedora|centos)", self.n):
+ return "repo/yum/%s/%s/mongodb-enterprise/%s/%s/RPMS/" % (self.n, self.repo_os_version(build_os), spec.branch(), self.archname(arch))
+ elif re.search("(suse)", self.n):
+ return "repo/zypper/%s/%s/mongodb-enterprise/%s/%s/RPMS/" % (self.n, self.repo_os_version(build_os), spec.branch(), self.archname(arch))
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+ def repo_component(self):
+ """Return the name of the section/component/pool we are publishing into -
+ e.g. "multiverse" for Ubuntu, "main" for debian."""
+ if self.n == 'ubuntu':
+ return "multiverse"
+ elif self.n == 'debian':
+ return "main"
+ else:
+ raise Exception("unsupported distro: %s" % self.n)
+
+ def repo_os_version(self, build_os):
+ """Return an OS version suitable for package repo directory
+ naming - e.g. 5, 6 or 7 for redhat/centos, "precise," "wheezy," etc.
+ for Ubuntu/Debian, 11 for suse"""
+ if self.n == 'suse':
+ return re.sub(r'^suse(\d+)$', r'\1', build_os)
+ if self.n == 'redhat':
+ return re.sub(r'^rhel(\d).*$', r'\1', build_os)
+ elif self.n == 'ubuntu':
+ if build_os == 'ubuntu1204':
+ return "precise"
+ elif build_os == 'ubuntu1404':
+ return "trusty"
+ else:
+ raise Exception("unsupported build_os: %s" % build_os)
+ elif self.n == 'debian':
+ if build_os == 'debian71':
+ return 'wheezy'
+ else:
+ raise Exception("unsupported build_os: %s" % build_os)
+ else:
+ raise Exception("unsupported distro: %s" % self.n)
+
+ def make_pkg(self, build_os, arch, spec, srcdir):
+ if re.search("^(debian|ubuntu)", self.n):
+ return make_deb(self, build_os, arch, spec, srcdir)
+ elif re.search("^(suse|centos|redhat|fedora)", self.n):
+ return make_rpm(self, build_os, arch, spec, srcdir)
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+ def build_os(self):
+ """Return the build os label in the binary package to download ("rhel57", "rhel62" and "rhel70"
+ for redhat, "ubuntu1204" and "ubuntu1404" for Ubuntu, "debian71" for Debian, and "suse11" for SUSE)"""
+
+ if re.search("(suse)", self.n):
+ return [ "suse11" ]
+ if re.search("(redhat|fedora|centos)", self.n):
+ return [ "rhel70", "rhel62", "rhel57" ]
+ elif self.n == 'ubuntu':
+ return [ "ubuntu1204", "ubuntu1404" ]
+ elif self.n == 'debian':
+ return [ "debian71" ]
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+ def release_dist(self, build_os):
+ """Return the release distribution to use in the rpm - "el5" for rhel 5.x,
+ "el6" for rhel 6.x, return anything else unchanged"""
+
+ return re.sub(r'^rh(el\d).*$', r'\1', build_os)
+def main(argv):
+ (flags, specs) = parse_args(argv[1:])
+ distros=[Distro(distro) for distro in DISTROS]
+
+ oldcwd=os.getcwd()
+ srcdir=oldcwd+"/../"
+
+ # We do all our work in a randomly-created directory. You can set
+ # TEMPDIR to influence where this program will do stuff.
+ prefix=tempfile.mkdtemp()
+ print "Working in directory %s" % prefix
+
+ # This will be a list of directories where we put packages in
+ # "repository layout".
+ repos=[]
+
+ os.chdir(prefix)
+ try:
+ # Download the binaries.
+ urlfmt="http://downloads.mongodb.com/linux/mongodb-linux-%s-enterprise-%s-%s.tgz"
+
+ # Build a pacakge for each distro/spec/arch tuple, and
+ # accumulate the repository-layout directories.
+ for (distro, spec, arch) in crossproduct(distros, specs, ARCHES):
+
+ for build_os in distro.build_os():
+
+ httpget(urlfmt % (arch, build_os, spec.version()), ensure_dir(tarfile(build_os, arch, spec)))
+
+ repo = make_package(distro, build_os, arch, spec, srcdir)
+ make_repo(repo, distro, build_os, spec)
+
+ finally:
+ os.chdir(oldcwd)
+ if "-n" not in flags:
+ move_repos_into_place(prefix+"/repo", REPOPATH)
+ # FIXME: try shutil.rmtree some day.
+ sysassert(["rm", "-rv", prefix])
+
+
+def parse_args(args):
+ if len(args) == 0:
+ print """Usage: packager.py [OPTS] SPEC1 SPEC2 ... SPECn
+
+Options:
+
+ -n: Just build the packages, don't publish them as a repo
+ or clean out the working directory
+
+Each SPEC is a mongodb version string optionally followed by a colon
+and some parameters, of the form <paramname>=<value>. Supported
+parameters:
+
+ suffix -- suffix to append to the package's base name. (If
+ unsupplied, suffixes default based on the parity of the
+ middle number in the version.)
+
+ revision -- least-order version number to packaging systems
+"""
+ sys.exit(0)
+
+ try:
+ (flags, args) = getopt.getopt(args, "n")
+ except getopt.GetoptError, err:
+ print str(err)
+ sys.exit(2)
+ flags=dict(flags)
+ specs=[Spec(arg) for arg in args]
+ return (flags, specs)
+
+def crossproduct(*seqs):
+ """A generator for iterating all the tuples consisting of elements
+ of seqs."""
+ l = len(seqs)
+ if l == 0:
+ pass
+ elif l == 1:
+ for i in seqs[0]:
+ yield [i]
+ else:
+ for lst in crossproduct(*seqs[:-1]):
+ for i in seqs[-1]:
+ lst2=list(lst)
+ lst2.append(i)
+ yield lst2
+
+def sysassert(argv):
+ """Run argv and assert that it exited with status 0."""
+ print "In %s, running %s" % (os.getcwd(), " ".join(argv))
+ sys.stdout.flush()
+ sys.stderr.flush()
+ assert(subprocess.Popen(argv).wait()==0)
+
+def backtick(argv):
+ """Run argv and return its output string."""
+ print "In %s, running %s" % (os.getcwd(), " ".join(argv))
+ sys.stdout.flush()
+ sys.stderr.flush()
+ return subprocess.Popen(argv, stdout=subprocess.PIPE).communicate()[0]
+
+def ensure_dir(filename):
+ """Make sure that the directory that's the dirname part of
+ filename exists, and return filename."""
+ dirpart = os.path.dirname(filename)
+ try:
+ os.makedirs(dirpart)
+ except OSError: # as exc: # Python >2.5
+ exc=sys.exc_value
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise exc
+ return filename
+
+
+def tarfile(build_os, arch, spec):
+ """Return the location where we store the downloaded tarball for
+ this package"""
+ return "dl/mongodb-linux-%s-enterprise-%s-%s.tar.gz" % (spec.version(), build_os, arch)
+
+def setupdir(distro, build_os, arch, spec):
+ # The setupdir will be a directory containing all inputs to the
+ # distro's packaging tools (e.g., package metadata files, init
+ # scripts, etc), along with the already-built binaries). In case
+ # the following format string is unclear, an example setupdir
+ # would be dst/x86_64/debian-sysvinit/wheezy/mongodb-org-unstable/
+ # or dst/x86_64/redhat/rhel57/mongodb-org-unstable/
+ return "dst/%s/%s/%s/%s%s-%s/" % (arch, distro.name(), build_os, distro.pkgbase(), spec.suffix(), spec.pversion(distro))
+
+def unpack_binaries_into(build_os, arch, spec, where):
+ """Unpack the tarfile for (build_os, arch, spec) into directory where."""
+ rootdir=os.getcwd()
+ ensure_dir(where)
+ # Note: POSIX tar doesn't require support for gtar's "-C" option,
+ # and Python's tarfile module prior to Python 2.7 doesn't have the
+ # features to make this detail easy. So we'll just do the dumb
+ # thing and chdir into where and run tar there.
+ os.chdir(where)
+ try:
+ sysassert(["tar", "xvzf", rootdir+"/"+tarfile(build_os, arch, spec)])
+ release_dir = glob('mongodb-linux-*')[0]
+ for releasefile in "bin", "snmp", "LICENSE.txt", "README", "THIRD-PARTY-NOTICES":
+ os.rename("%s/%s" % (release_dir, releasefile), releasefile)
+ os.rmdir(release_dir)
+ except Exception:
+ exc=sys.exc_value
+ os.chdir(rootdir)
+ raise exc
+ os.chdir(rootdir)
+
+def make_package(distro, build_os, arch, spec, srcdir):
+ """Construct the package for (arch, distro, spec), getting
+ packaging files from srcdir and any user-specified suffix from
+ suffixes"""
+
+ sdir=setupdir(distro, build_os, arch, spec)
+ ensure_dir(sdir)
+ # Note that the RPM packages get their man pages from the debian
+ # directory, so the debian directory is needed in all cases (and
+ # innocuous in the debianoids' sdirs).
+ for pkgdir in ["debian", "rpm"]:
+ print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir)
+ # FIXME: sh-dash-cee is bad. See if tarfile can do this.
+ sysassert(["sh", "-c", "(cd \"%s\" && git archive r%s %s/ ) | (cd \"%s\" && tar xvf -)" % (srcdir, spec.version(), pkgdir, sdir)])
+ # Splat the binaries and snmp files under sdir. The "build" stages of the
+ # packaging infrastructure will move the files to wherever they
+ # need to go.
+ unpack_binaries_into(build_os, arch, spec, sdir)
+ # Remove the mongosniff binary due to libpcap dynamic
+ # linkage. FIXME: this removal should go away
+ # eventually.
+ if os.path.exists(sdir + "bin/mongosniff"):
+ os.unlink(sdir + "bin/mongosniff")
+ return distro.make_pkg(build_os, arch, spec, srcdir)
+
+def make_repo(repodir, distro, build_os, spec):
+ if re.search("(debian|ubuntu)", repodir):
+ make_deb_repo(repodir, distro, build_os, spec)
+ elif re.search("(suse|centos|redhat|fedora)", repodir):
+ make_rpm_repo(repodir)
+ else:
+ raise Exception("BUG: unsupported platform?")
+
+def make_deb(distro, build_os, arch, spec, srcdir):
+ # I can't remember the details anymore, but the initscript/upstart
+ # job files' names must match the package name in some way; and
+ # see also the --name flag to dh_installinit in the generated
+ # debian/rules file.
+ suffix=spec.suffix()
+ sdir=setupdir(distro, build_os, arch, spec)
+ if re.search("debian", distro.name()):
+ os.link(sdir+"debian/init.d", sdir+"debian/%s%s-server.mongod.init" % (distro.pkgbase(), suffix))
+ os.unlink(sdir+"debian/mongod.upstart")
+ elif re.search("ubuntu", distro.name()):
+ os.link(sdir+"debian/mongod.upstart", sdir+"debian/%s%s-server.mongod.upstart" % (distro.pkgbase(), suffix))
+ os.unlink(sdir+"debian/init.d")
+ else:
+ raise Exception("unknown debianoid flavor: not debian or ubuntu?")
+ # Rewrite the control and rules files
+ write_debian_changelog(sdir+"debian/changelog", spec, srcdir)
+ distro_arch=distro.archname(arch)
+ sysassert(["cp", "-v", srcdir+"debian/%s%s.control" % (distro.pkgbase(), suffix), sdir+"debian/control"])
+ sysassert(["cp", "-v", srcdir+"debian/%s%s.rules" % (distro.pkgbase(), suffix), sdir+"debian/rules"])
+
+
+ # old non-server-package postinst will be hanging around for old versions
+ #
+ if os.path.exists(sdir+"debian/postinst"):
+ os.unlink(sdir+"debian/postinst")
+
+ # copy our postinst files
+ #
+ sysassert(["sh", "-c", "cp -v \"%sdebian/\"*.postinst \"%sdebian/\""%(srcdir, sdir)])
+
+ # Do the packaging.
+ oldcwd=os.getcwd()
+ try:
+ os.chdir(sdir)
+ sysassert(["dpkg-buildpackage", "-a"+distro_arch, "-k Richard Kreuter <richard@10gen.com>"])
+ finally:
+ os.chdir(oldcwd)
+ r=distro.repodir(arch, build_os, spec)
+ ensure_dir(r)
+ # FIXME: see if shutil.copyfile or something can do this without
+ # much pain.
+ #sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r])
+ sysassert(["sh", "-c", "cp -v \"%s/../\"*.deb \"%s\""%(sdir, r)])
+ return r
+
+def make_deb_repo(repo, distro, build_os, spec):
+ # Note: the Debian repository Packages files must be generated
+ # very carefully in order to be usable.
+ oldpwd=os.getcwd()
+ os.chdir(repo+"../../../../../../")
+ try:
+ dirs=set([os.path.dirname(deb)[2:] for deb in backtick(["find", ".", "-name", "*.deb"]).split()])
+ for d in dirs:
+ s=backtick(["dpkg-scanpackages", d, "/dev/null"])
+ f=open(d+"/Packages", "w")
+ try:
+ f.write(s)
+ finally:
+ f.close()
+ b=backtick(["gzip", "-9c", d+"/Packages"])
+ f=open(d+"/Packages.gz", "wb")
+ try:
+ f.write(b)
+ finally:
+ f.close()
+ finally:
+ os.chdir(oldpwd)
+ # Notes: the Release{,.gpg} files must live in a special place,
+ # and must be created after all the Packages.gz files have been
+ # done.
+ s="""Origin: mongodb
+Label: mongodb
+Suite: mongodb
+Codename: %s/mongodb-enterprise
+Architectures: amd64
+Components: %s
+Description: MongoDB packages
+""" % (distro.repo_os_version(build_os), distro.repo_component())
+ if os.path.exists(repo+"../../Release"):
+ os.unlink(repo+"../../Release")
+ if os.path.exists(repo+"../../Release.gpg"):
+ os.unlink(repo+"../../Release.gpg")
+ oldpwd=os.getcwd()
+ os.chdir(repo+"../../")
+ s2=backtick(["apt-ftparchive", "release", "."])
+ try:
+ f=open("Release", 'w')
+ try:
+ f.write(s)
+ f.write(s2)
+ finally:
+ f.close()
+
+ arg=None
+ for line in backtick(["gpg", "--list-keys"]).split("\n"):
+ tokens=line.split()
+ if len(tokens)>0 and tokens[0] == "uid":
+ arg=tokens[-1]
+ break
+ # Note: for some reason, I think --no-tty might be needed
+ # here, but maybe not.
+ sysassert(["gpg", "-r", arg, "--no-secmem-warning", "-abs", "--output", "Release.gpg", "Release"])
+ finally:
+ os.chdir(oldpwd)
+
+
+def move_repos_into_place(src, dst):
+ # Find all the stuff in src/*, move it to a freshly-created
+ # directory beside dst, then play some games with symlinks so that
+ # dst is a name the new stuff and dst+".old" names the previous
+ # one. This feels like a lot of hooey for something so trivial.
+
+ # First, make a crispy fresh new directory to put the stuff in.
+ i=0
+ while True:
+ date_suffix=time.strftime("%Y-%m-%d")
+ dname=dst+".%s.%d" % (date_suffix, i)
+ try:
+ os.mkdir(dname)
+ break
+ except OSError:
+ exc=sys.exc_value
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise exc
+ i=i+1
+
+ # Put the stuff in our new directory.
+ for r in os.listdir(src):
+ sysassert(["cp", "-rv", src + "/" + r, dname])
+
+ # Make a symlink to the new directory; the symlink will be renamed
+ # to dst shortly.
+ i=0
+ while True:
+ tmpnam=dst+".TMP.%d" % i
+ try:
+ os.symlink(dname, tmpnam)
+ break
+ except OSError: # as exc: # Python >2.5
+ exc=sys.exc_value
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise exc
+ i=i+1
+
+ # Make a symlink to the old directory; this symlink will be
+ # renamed shortly, too.
+ oldnam=None
+ if os.path.exists(dst):
+ i=0
+ while True:
+ oldnam=dst+".old.%d" % i
+ try:
+ os.symlink(os.readlink(dst), oldnam)
+ break
+ except OSError: # as exc: # Python >2.5
+ exc=sys.exc_value
+ if exc.errno == errno.EEXIST:
+ pass
+ else:
+ raise exc
+
+ os.rename(tmpnam, dst)
+ if oldnam:
+ os.rename(oldnam, dst+".old")
+
+
+def write_debian_changelog(path, spec, srcdir):
+ oldcwd=os.getcwd()
+ os.chdir(srcdir)
+ preamble=""
+ if spec.param("revision"):
+ preamble="""mongodb%s (%s-%s) unstable; urgency=low
+
+ * Bump revision number
+
+ -- Richard Kreuter <richard@10gen.com> %s
+
+""" % (spec.suffix(), spec.pversion(Distro("debian")), spec.param("revision"), time.strftime("%a, %d %b %Y %H:%m:%S %z"))
+ try:
+ s=preamble+backtick(["sh", "-c", "git archive r%s debian/changelog | tar xOf -" % spec.version()])
+ finally:
+ os.chdir(oldcwd)
+ f=open(path, 'w')
+ lines=s.split("\n")
+ # If the first line starts with "mongodb", it's not a revision
+ # preamble, and so frob the version number.
+ lines[0]=re.sub("^mongodb \\(.*\\)", "mongodb (%s)" % (spec.pversion(Distro("debian"))), lines[0])
+ # Rewrite every changelog entry starting in mongodb<space>
+ lines=[re.sub("^mongodb ", "mongodb%s " % (spec.suffix()), l) for l in lines]
+ lines=[re.sub("^ --", " --", l) for l in lines]
+ s="\n".join(lines)
+ try:
+ f.write(s)
+ finally:
+ f.close()
+
+def make_rpm(distro, build_os, arch, spec, srcdir):
+ # Create the specfile.
+ suffix=spec.suffix()
+ sdir=setupdir(distro, build_os, arch, spec)
+
+ # Use special suse init script if we're building for SUSE
+ #
+ if distro.name() == "suse":
+ os.unlink(sdir+"rpm/init.d-mongod")
+ os.link(sdir+"rpm/init.d-mongod.suse", sdir+"rpm/init.d-mongod")
+
+ specfile=srcdir+"rpm/mongodb%s.spec" % suffix
+ topdir=ensure_dir('%s/rpmbuild/%s/' % (os.getcwd(), build_os))
+ for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
+ ensure_dir("%s/%s/" % (topdir, subdir))
+ distro_arch=distro.archname(arch)
+ # RPM tools take these macro files that define variables in
+ # RPMland. Unfortunately, there's no way to tell RPM tools to use
+ # a given file *in addition* to the files that it would already
+ # load, so we have to figure out what it would normally load,
+ # augment that list, and tell RPM to use the augmented list. To
+ # figure out what macrofiles ordinarily get loaded, older RPM
+ # versions had a parameter called "macrofiles" that could be
+ # extracted from "rpm --showrc". But newer RPM versions don't
+ # have this. To tell RPM what macros to use, older versions of
+ # RPM have a --macros option that doesn't work; on these versions,
+ # you can put a "macrofiles" parameter into an rpmrc file. But
+ # that "macrofiles" setting doesn't do anything for newer RPM
+ # versions, where you have to use the --macros flag instead. And
+ # all of this is to let us do our work with some guarantee that
+ # we're not clobbering anything that doesn't belong to us. Why is
+ # RPM so braindamaged?
+ macrofiles=[l for l in backtick(["rpm", "--showrc"]).split("\n") if l.startswith("macrofiles")]
+ flags=[]
+ macropath=os.getcwd()+"/macros"
+
+ write_rpm_macros_file(macropath, topdir, distro.release_dist(build_os))
+ if len(macrofiles)>0:
+ macrofiles=macrofiles[0]+":"+macropath
+ rcfile=os.getcwd()+"/rpmrc"
+ write_rpmrc_file(rcfile, macrofiles)
+ flags=["--rpmrc", rcfile]
+ else:
+ # This hard-coded hooey came from some box running RPM
+ # 4.4.2.3. It may not work over time, but RPM isn't sanely
+ # configurable.
+ flags=["--macros", "/usr/lib/rpm/macros:/usr/lib/rpm/%s-linux/macros:/etc/rpm/macros.*:/etc/rpm/macros:/etc/rpm/%s-linux/macros:~/.rpmmacros:%s" % (distro_arch, distro_arch, macropath)]
+ # Put the specfile and the tar'd up binaries and stuff in
+ # place. FIXME: see if shutil.copyfile can do this without too
+ # much hassle.
+ sysassert(["cp", "-v", specfile, topdir+"SPECS/"])
+ oldcwd=os.getcwd()
+ os.chdir(sdir+"/../")
+ try:
+ sysassert(["tar", "-cpzf", topdir+"SOURCES/mongodb%s-%s.tar.gz" % (suffix, spec.pversion(distro)), os.path.basename(os.path.dirname(sdir))])
+ finally:
+ os.chdir(oldcwd)
+ # Do the build.
+ sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags + ["%s/SPECS/mongodb%s.spec" % (topdir, suffix)])
+ r=distro.repodir(arch, build_os, spec)
+ ensure_dir(r)
+ # FIXME: see if some combination of shutil.copy<hoohah> and glob
+ # can do this without shelling out.
+ sysassert(["sh", "-c", "cp -v \"%s/RPMS/%s/\"*.rpm \"%s\""%(topdir, distro_arch, r)])
+ return r
+
+def make_rpm_repo(repo):
+ oldpwd=os.getcwd()
+ os.chdir(repo+"../")
+ try:
+ sysassert(["createrepo", "."])
+ finally:
+ os.chdir(oldpwd)
+
+
+def write_rpmrc_file(path, string):
+ f=open(path, 'w')
+ try:
+ f.write(string)
+ finally:
+ f.close()
+
+def write_rpm_macros_file(path, topdir, release_dist):
+ f=open(path, 'w')
+ try:
+ f.write("%%_topdir %s\n" % topdir)
+ f.write("%%dist .%s\n" % release_dist)
+ f.write("%_use_internal_dependency_generator 0\n")
+ finally:
+ f.close()
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/buildscripts/packager.py b/buildscripts/packager.py
index dcddf0ab91b..2cf082ed334 100644
--- a/buildscripts/packager.py
+++ b/buildscripts/packager.py
@@ -28,7 +28,7 @@
import errno
import getopt
-import httplib
+import httplib2
import os
import re
import stat
@@ -44,17 +44,17 @@ import urlparse
# web server will look for repositories.
REPOPATH="/var/www/repo"
-# The 10gen names for the architectures we support.
+# The MongoDB names for the architectures we support.
ARCHES=["i686", "x86_64"]
# Made up names for the flavors of distribution we package for.
-DISTROS=["debian-sysvinit", "ubuntu-upstart", "redhat"]
+DISTROS=["suse", "debian-sysvinit", "ubuntu-upstart", "redhat"]
# When we're preparing a directory containing packaging tool inputs
# and our binaries, use this relative subdirectory for placing the
# binaries.
BINARYDIR="BINARIES"
-
+
class Spec(object):
def __init__(self, specstr):
tup = specstr.split(":")
@@ -86,7 +86,7 @@ class Spec(object):
elif "suffix" in self.params:
return self.params["suffix"]
else:
- return "-10gen" if int(self.ver.split(".")[1])%2==0 else "-10gen-unstable"
+ return "-org" if int(self.ver.split(".")[1])%2==0 else "-org-unstable"
def pversion(self, distro):
@@ -96,7 +96,7 @@ class Spec(object):
# our upstream version too).
if re.search("^(debian|ubuntu)", distro.name()):
return re.sub("-", "~", self.ver)
- elif re.search("(redhat|fedora|centos)", distro.name()):
+ elif re.search("(suse|redhat|fedora|centos)", distro.name()):
return re.sub("\\d+-", "", self.ver)
else:
raise Exception("BUG: unsupported platform?")
@@ -115,13 +115,14 @@ class Distro(object):
def pkgbase(self):
# pkgbase is the first part of the package's name on
- # this distro.
- return "mongo" if re.search("(redhat|fedora|centos)", self.n) else "mongodb"
+ # this distro (pre-2.5.3 was "mongo" for redhat and
+ # "mongodb" for debian")
+ return "mongodb"
def archname(self, arch):
if re.search("^(debian|ubuntu)", self.n):
return "i386" if arch.endswith("86") else "amd64"
- elif re.search("^(centos|redhat|fedora)", self.n):
+ elif re.search("^(suse|centos|redhat|fedora)", self.n):
return "i686" if arch.endswith("86") else "x86_64"
else:
raise Exception("BUG: unsupported platform?")
@@ -133,7 +134,7 @@ class Distro(object):
tools place the package files)."""
if re.search("^(debian|ubuntu)", self.n):
return "repo/%s/dists/dist/10gen/binary-%s/" % (self.n, self.archname(arch))
- elif re.search("(redhat|fedora|centos)", self.n):
+ elif re.search("(suse|redhat|fedora|centos)", self.n):
return "repo/%s/os/%s/RPMS/" % (self.n, self.archname(arch))
else:
raise Exception("BUG: unsupported platform?")
@@ -141,7 +142,7 @@ class Distro(object):
def make_pkg(self, arch, spec, srcdir):
if re.search("^(debian|ubuntu)", self.n):
return make_deb(self, arch, spec, srcdir)
- elif re.search("^(centos|redhat|fedora)", self.n):
+ elif re.search("^(suse|centos|redhat|fedora)", self.n):
return make_rpm(self, arch, spec, srcdir)
else:
raise Exception("BUG: unsupported platform?")
@@ -271,7 +272,7 @@ def setupdir(distro, arch, spec):
# distro's packaging tools (e.g., package metadata files, init
# scripts, etc), along with the already-built binaries). In case
# the following format string is unclear, an example setupdir
- # would be dst/x86_64/debian-sysvinit/mongodb-10gen-unstable/
+ # would be dst/x86_64/debian-sysvinit/mongodb-org-unstable/
return "dst/%s/%s/%s%s-%s/" % (arch, distro.name(), distro.pkgbase(), spec.suffix(), spec.pversion(distro))
def httpget(url, filename):
@@ -281,20 +282,17 @@ def httpget(url, filename):
u=urlparse.urlparse(url)
assert(u.scheme=='http')
try:
- conn = httplib.HTTPConnection(u.hostname)
- conn.request("GET", u.path)
+ h = httplib2.Http(cache = os.environ["HOME"] + "/.cache")
+ resp, content = h.request(url, "GET")
t=filename+'.TMP'
- res = conn.getresponse()
- # FIXME: follow redirects
- if res.status==200:
+ if resp.status==200:
f = open(t, 'w')
try:
- f.write(res.read())
+ f.write(content)
finally:
f.close()
-
else:
- raise Exception("HTTP error %d" % res.status)
+ raise Exception("HTTP error %d" % resp.status)
os.rename(t, filename)
finally:
if conn:
@@ -341,13 +339,14 @@ def make_package(distro, arch, spec, srcdir):
# Remove the mongosniff binary due to libpcap dynamic
# linkage. FIXME: this removal should go away
# eventually.
- os.unlink(sdir+("%s/usr/bin/mongosniff"%BINARYDIR))
+ if os.path.exists(sdir+("%s/usr/bin/mongosniff"%BINARYDIR)):
+ os.unlink(sdir+("%s/usr/bin/mongosniff"%BINARYDIR))
return distro.make_pkg(arch, spec, srcdir)
def make_repo(repodir):
if re.search("(debian|ubuntu)", repodir):
make_deb_repo(repodir)
- elif re.search("(centos|redhat|fedora)", repodir):
+ elif re.search("(suse|centos|redhat|fedora)", repodir):
make_rpm_repo(repodir)
else:
raise Exception("BUG: unsupported platform?")
@@ -360,18 +359,28 @@ def make_deb(distro, arch, spec, srcdir):
suffix=spec.suffix()
sdir=setupdir(distro, arch, spec)
if re.search("sysvinit", distro.name()):
- os.link(sdir+"debian/init.d", sdir+"debian/%s%s.mongodb.init" % (distro.pkgbase(), suffix))
- os.unlink(sdir+"debian/mongodb.upstart")
+ os.link(sdir+"debian/init.d", sdir+"debian/%s%s-server.mongod.init" % (distro.pkgbase(), suffix))
+ os.unlink(sdir+"debian/mongod.upstart")
elif re.search("upstart", distro.name()):
- os.link(sdir+"debian/mongodb.upstart", sdir+"debian/%s%s.upstart" % (distro.pkgbase(), suffix))
+ os.link(sdir+"debian/mongod.upstart", sdir+"debian/%s%s-server.mongod.upstart" % (distro.pkgbase(), suffix))
os.unlink(sdir+"debian/init.d")
else:
raise Exception("unknown debianoid flavor: not sysvinit or upstart?")
- # Rewrite the control and rules files
- write_debian_control_file(sdir+"debian/control", spec)
- write_debian_rules_file(sdir+"debian/rules", spec)
write_debian_changelog(sdir+"debian/changelog", spec, srcdir)
distro_arch=distro.archname(arch)
+ sysassert(["cp", "-v", srcdir+"debian/%s%s.control" % (distro.pkgbase(), suffix), sdir+"debian/control"])
+ sysassert(["cp", "-v", srcdir+"debian/%s%s.rules" % (distro.pkgbase(), suffix), sdir+"debian/rules"])
+
+
+ # old non-server-package postinst will be hanging around for old versions
+ #
+ if os.path.exists(sdir+"debian/postinst"):
+ os.unlink(sdir+"debian/postinst")
+
+ # copy our postinst files
+ #
+ sysassert(["sh", "-c", "cp -v \"%sdebian/\"*.postinst \"%sdebian/\""%(srcdir, sdir)])
+
# Do the packaging.
oldcwd=os.getcwd()
try:
@@ -383,7 +392,9 @@ def make_deb(distro, arch, spec, srcdir):
ensure_dir(r)
# FIXME: see if shutil.copyfile or something can do this without
# much pain.
- sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r])
+ # sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r])
+ # sysassert(["cp", "-v", sdir+"../*.deb", r])
+ sysassert(["sh", "-c", "cp -v \"%s/../\"*.deb \"%s\""%(sdir, r)])
return r
def make_deb_repo(repo):
@@ -412,14 +423,14 @@ def make_deb_repo(repo):
# and must be created after all the Packages.gz files have been
# done.
s="""
-Origin: 10gen
-Label: 10gen
-Suite: 10gen
+Origin: mongodb
+Label: mongodb
+Suite: mongodb
Codename: %s
Version: %s
Architectures: i386 amd64
Components: 10gen
-Description: 10gen packages
+Description: mongodb packages
""" % ("dist", "dist")
if os.path.exists(repo+"../../Release"):
os.unlink(repo+"../../Release")
@@ -542,181 +553,18 @@ def write_debian_changelog(path, spec, srcdir):
f.write(s)
finally:
f.close()
-
-def write_debian_control_file(path, spec):
- s="""Source: @@PACKAGE_BASENAME@@
-Section: devel
-Priority: optional
-Maintainer: Richard Kreuter <richard@10gen.com>
-Build-Depends:
-Standards-Version: 3.8.0
-Homepage: http://www.mongodb.org
-
-Package: @@PACKAGE_BASENAME@@
-Conflicts: @@PACKAGE_CONFLICTS@@
-Architecture: any
-Depends: libc6 (>= 2.3.2), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.1.1)
-Description: An object/document-oriented database
- MongoDB is a high-performance, open source, schema-free
- document-oriented data store that's easy to deploy, manage
- and use. It's network accessible, written in C++ and offers
- the following features :
- .
- * Collection oriented storage - easy storage of object-
- style data
- * Full index support, including on inner objects
- * Query profiling
- * Replication and fail-over support
- * Efficient storage of binary data including large
- objects (e.g. videos)
- * Auto-sharding for cloud-level scalability (Q209)
- .
- High performance, scalability, and reasonable depth of
- functionality are the goals for the project.
-"""
- s=re.sub("@@PACKAGE_BASENAME@@", "mongodb%s" % spec.suffix(), s)
- conflict_suffixes=["", "-stable", "-unstable", "-nightly", "-10gen", "-10gen-unstable"]
- conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
- s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongodb"+suffix for suffix in conflict_suffixes]), s)
- f=open(path, 'w')
- try:
- f.write(s)
- finally:
- f.close()
-
-def write_debian_rules_file(path, spec):
- # Note debian/rules is a makefile, so for visual disambiguation we
- # make all tabs here \t.
- s="""#!/usr/bin/make -f
-# -*- makefile -*-
-# Sample debian/rules that uses debhelper.
-# This file was originally written by Joey Hess and Craig Small.
-# As a special exception, when this file is copied by dh-make into a
-# dh-make output file, you may use that output file without restriction.
-# This special exception was added by Craig Small in version 0.37 of dh-make.
-
-# Uncomment this to turn on verbose mode.
-#export DH_VERBOSE=1
-
-
-configure: configure-stamp
-configure-stamp:
-\tdh_testdir
- # Add here commands to configure the package.
-
-\ttouch configure-stamp
-
-
-build: build-stamp
-
-build-stamp: configure-stamp
-\tdh_testdir
-
- # Add here commands to compile the package.
-# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED.
-\t# scons
- #docbook-to-man debian/mongodb.sgml > mongodb.1
-\tls debian/*.1 > debian/@@PACKAGE_NAME@@.manpages
-
-\ttouch $@
-
-clean:
-\tdh_testdir
-\tdh_testroot
-\trm -f build-stamp configure-stamp
-
-\t# FIXME: scons freaks out at the presence of target files
-\t# under debian/mongodb.
-\t#scons -c
-\trm -rf $(CURDIR)/debian/@@PACKAGE_NAME@@
-\trm -f config.log
-\trm -f mongo
-\trm -f mongod
-\trm -f mongoimportjson
-\trm -f mongoexport
-\trm -f mongorestore
-\trm -f mongodump
-\trm -f mongofiles
-\trm -f .sconsign.dblite
-\trm -f libmongoclient.a
-\trm -rf client/*.o
-\trm -rf tools/*.o
-\trm -rf shell/*.o
-\trm -rf .sconf_temp
-\trm -f buildscripts/*.pyc
-\trm -f *.pyc
-\trm -f buildinfo.cpp
-\tdh_clean debian/files
-
-install: build
-\tdh_testdir
-\tdh_testroot
-\tdh_prep
-\tdh_installdirs
-
-# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED.
-\t# scons --prefix=$(CURDIR)/debian/mongodb/usr install
-\tcp -v $(CURDIR)/@@BINARYDIR@@/usr/bin/* $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/bin
-\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/etc
-\tcp $(CURDIR)/debian/mongodb.conf $(CURDIR)/debian/@@PACKAGE_NAME@@/etc/mongodb.conf
-
-\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/
-\tinstall -m 644 $(CURDIR)/debian/lintian-overrides \
-\t\t$(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/@@PACKAGE_NAME@@
-
-# Build architecture-independent files here.
-binary-indep: build install
-# We have nothing to do by default.
-
-# Build architecture-dependent files here.
-binary-arch: build install
-\tdh_testdir
-\tdh_testroot
-\tdh_installchangelogs
-\tdh_installdocs
-\tdh_installexamples
-#\tdh_install
-#\tdh_installmenu
-#\tdh_installdebconf\t
-#\tdh_installlogrotate
-#\tdh_installemacsen
-#\tdh_installpam
-#\tdh_installmime
-\tdh_installinit --name=@@PACKAGE_BASENAME@@
-#\tdh_installinfo
-\tdh_installman
-\tdh_link
-# Appears to be broken on Ubuntu 11.10...?
-#\tdh_strip
-\tdh_compress
-\tdh_fixperms
-\tdh_installdeb
-\tdh_shlibdeps
-\tdh_gencontrol
-\tdh_md5sums
-\tdh_builddeb
-
-binary: binary-indep binary-arch
-.PHONY: build clean binary-indep binary-arch binary install configure
-"""
- s=re.sub("@@PACKAGE_NAME@@", "mongodb%s" % spec.suffix(), s)
- s=re.sub("@@PACKAGE_BASENAME@@", "mongodb", s)
- s=re.sub("@@BINARYDIR@@", BINARYDIR, s)
- f=open(path, 'w')
- try:
- f.write(s)
- finally:
- f.close()
- # FIXME: some versions of debianoids seem to
- # need the rules file to be 755?
- os.chmod(path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR|stat.S_IXGRP|stat.S_IRGRP|stat.S_IXOTH|stat.S_IWOTH)
-
def make_rpm(distro, arch, spec, srcdir):
# Create the specfile.
suffix=spec.suffix()
sdir=setupdir(distro, arch, spec)
- specfile=sdir+"rpm/mongo%s.spec" % suffix
- write_rpm_spec_file(specfile, spec)
+
+ # Use special suse init script if we're building for SUSE
+ #
+ if distro.name() == "suse":
+ os.unlink(sdir+"rpm/init.d-mongod")
+ os.link(sdir+"rpm/init.d-mongod.suse", sdir+"rpm/init.d-mongod")
+
+ specfile=srcdir+"rpm/mongodb%s.spec" % suffix
topdir=ensure_dir(os.getcwd()+'/rpmbuild/')
for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]:
ensure_dir("%s/%s/" % (topdir, subdir))
@@ -758,11 +606,11 @@ def make_rpm(distro, arch, spec, srcdir):
oldcwd=os.getcwd()
os.chdir(sdir+"/../")
try:
- sysassert(["tar", "-cpzf", topdir+"SOURCES/mongo%s-%s.tar.gz" % (suffix, spec.pversion(distro)), os.path.basename(os.path.dirname(sdir))])
+ sysassert(["tar", "-cpzf", topdir+"SOURCES/mongodb%s-%s.tar.gz" % (suffix, spec.pversion(distro)), os.path.basename(os.path.dirname(sdir))])
finally:
os.chdir(oldcwd)
# Do the build.
- sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags + ["%s/SPECS/mongo%s.spec" % (topdir, suffix)])
+ sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags + ["%s/SPECS/mongodb%s.spec" % (topdir, suffix)])
r=distro.repodir(arch)
ensure_dir(r)
# FIXME: see if some combination of shutil.copy<hoohah> and glob
@@ -793,212 +641,5 @@ def write_rpm_macros_file(path, topdir):
finally:
f.close()
-def write_rpm_spec_file(path, spec):
- s="""Name: @@PACKAGE_BASENAME@@
-Conflicts: @@PACKAGE_CONFLICTS@@
-Obsoletes: @@PACKAGE_OBSOLETES@@
-Version: @@PACKAGE_VERSION@@
-Release: mongodb_@@PACKAGE_REVISION@@%{?dist}
-Summary: mongo client shell and tools
-License: AGPL 3.0
-URL: http://www.mongodb.org
-Group: Applications/Databases
-
-Source0: %{name}-%{version}.tar.gz
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
-
-%description
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-It features dynamic profileable queries, full indexing, replication
-and fail-over support, efficient storage of large binary data objects,
-and auto-sharding.
-
-This package provides the mongo shell, import/export tools, and other
-client utilities.
-
-%package server
-Summary: mongo server, sharding server, and support scripts
-Group: Applications/Databases
-Requires: @@PACKAGE_BASENAME@@
-
-%description server
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-
-This package provides the mongo server software, mongo sharding server
-softwware, default configuration files, and init.d scripts.
-
-%package devel
-Summary: Headers and libraries for mongo development.
-Group: Applications/Databases
-
-%description devel
-Mongo (from "huMONGOus") is a schema-free document-oriented database.
-
-This package provides the mongo static library and header files needed
-to develop mongo client software.
-
-%prep
-%setup
-
-%build
-#scons --prefix=$RPM_BUILD_ROOT/usr all
-# XXX really should have shared library here
-
-%install
-#scons --prefix=$RPM_BUILD_ROOT/usr install
-mkdir -p $RPM_BUILD_ROOT/usr
-cp -rv @@BINARYDIR@@/usr/bin $RPM_BUILD_ROOT/usr
-mkdir -p $RPM_BUILD_ROOT/usr/share/man/man1
-cp debian/*.1 $RPM_BUILD_ROOT/usr/share/man/man1/
-# FIXME: remove this rm when mongosniff is back in the package
-rm -v $RPM_BUILD_ROOT/usr/share/man/man1/mongosniff.1*
-mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d
-cp -v rpm/init.d-mongod $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
-chmod a+x $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod
-mkdir -p $RPM_BUILD_ROOT/etc
-cp -v rpm/mongod.conf $RPM_BUILD_ROOT/etc/mongod.conf
-mkdir -p $RPM_BUILD_ROOT/etc/sysconfig
-cp -v rpm/mongod.sysconfig $RPM_BUILD_ROOT/etc/sysconfig/mongod
-mkdir -p $RPM_BUILD_ROOT/var/lib/mongo
-mkdir -p $RPM_BUILD_ROOT/var/log/mongo
-touch $RPM_BUILD_ROOT/var/log/mongo/mongod.log
-
-%clean
-#scons -c
-rm -rf $RPM_BUILD_ROOT
-
-%pre server
-if ! /usr/bin/id -g mongod &>/dev/null; then
- /usr/sbin/groupadd -r mongod
-fi
-if ! /usr/bin/id mongod &>/dev/null; then
- /usr/sbin/useradd -M -r -g mongod -d /var/lib/mongo -s /bin/false \
- -c mongod mongod > /dev/null 2>&1
-fi
-
-%post server
-if test $1 = 1
-then
- /sbin/chkconfig --add mongod
-fi
-
-%preun server
-if test $1 = 0
-then
- /sbin/chkconfig --del mongod
-fi
-
-%postun server
-if test $1 -ge 1
-then
- /sbin/service mongod condrestart >/dev/null 2>&1 || :
-fi
-
-%files
-%defattr(-,root,root,-)
-#%doc README GNU-AGPL-3.0.txt
-
-%{_bindir}/bsondump
-%{_bindir}/mongo
-%{_bindir}/mongodump
-%{_bindir}/mongoexport
-#@@VERSION!=2.1.0@@%{_bindir}/mongofiles
-%{_bindir}/mongoimport
-#@@VERSION>=2.1.0@@%{_bindir}/mongooplog
-#@@VERSION>=2.1.0@@%{_bindir}/mongoperf
-%{_bindir}/mongorestore
-#@@VERSION>1.9@@%{_bindir}/mongotop
-%{_bindir}/mongostat
-# FIXME: uncomment when mongosniff is back in the package
-#%{_bindir}/mongosniff
-
-# FIXME: uncomment this when there's a stable release whose source
-# tree contains a bsondump man page.
-#@@VERSION>1.9@@%{_mandir}/man1/bsondump.1*
-%{_mandir}/man1/mongo.1*
-%{_mandir}/man1/mongodump.1*
-%{_mandir}/man1/mongoexport.1*
-%{_mandir}/man1/mongofiles.1*
-%{_mandir}/man1/mongoimport.1*
-%{_mandir}/man1/mongorestore.1*
-%{_mandir}/man1/mongostat.1*
-# FIXME: uncomment when mongosniff is back in the package
-#%{_mandir}/man1/mongosniff.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongotop.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongoperf.1*
-#@@VERSION>=2.4.0@@%{_mandir}/man1/mongooplog.1*
-
-%files server
-%defattr(-,root,root,-)
-%config(noreplace) /etc/mongod.conf
-%{_bindir}/mongod
-%{_bindir}/mongos
-%{_mandir}/man1/mongod.1*
-%{_mandir}/man1/mongos.1*
-/etc/rc.d/init.d/mongod
-/etc/sysconfig/mongod
-#/etc/rc.d/init.d/mongos
-%attr(0755,mongod,mongod) %dir /var/lib/mongo
-%attr(0755,mongod,mongod) %dir /var/log/mongo
-%attr(0640,mongod,mongod) %config(noreplace) %verify(not md5 size mtime) /var/log/mongo/mongod.log
-
-%changelog
-* Thu Jan 28 2010 Richard M Kreuter <richard@10gen.com>
-- Minor fixes.
-
-* Sat Oct 24 2009 Joe Miklojcik <jmiklojcik@shopwiki.com> -
-- Wrote mongo.spec.
-"""
- suffix=spec.suffix()
- s=re.sub("@@PACKAGE_BASENAME@@", "mongo%s" % suffix, s)
- s=re.sub("@@PACKAGE_VERSION@@", spec.pversion(Distro("redhat")), s)
- # FIXME, maybe: the RPM guide says that Release numbers ought to
- # be integers starting at 1, but we use "mongodb_1{%dist}",
- # whatever the hell that means.
- s=re.sub("@@PACKAGE_REVISION@@", str(int(spec.param("revision"))+1) if spec.param("revision") else "1", s)
- s=re.sub("@@BINARYDIR@@", BINARYDIR, s)
- conflict_suffixes=["", "-10gen", "-10gen-unstable"]
- conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()]
- s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongo"+_ for _ in conflict_suffixes]), s)
- if suffix.endswith("-10gen"):
- s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-stable", s)
- s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-stable", s)
- elif suffix == "-10gen-unstable":
- s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-unstable", s)
- s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-unstable", s)
- else:
- raise Exception("BUG: unknown suffix %s" % suffix)
-
- lines=[]
- for line in s.split("\n"):
- m = re.search("@@VERSION(>|>=|!=)(\d.*)@@(.*)", line)
- if m:
- op = m.group(1)
- ver = m.group(2)
- fn = m.group(3)
- if op == '>':
- if spec.version_better_than(ver):
- lines.append(fn)
- elif op == '>=':
- if spec.version() == ver or spec.version_better_than(ver):
- lines.append(fn)
- elif op == '!=':
- if spec.version() != ver:
- lines.append(fn)
- else:
- # Since we're inventing our own template system for RPM
- # specfiles here, we oughtn't use template syntax we don't
- # support.
- raise Exception("BUG: probable bug in packager script: %s, %s, %s" % (m.group(1), m.group(2), m.group(3)))
- else:
- lines.append(line)
- s="\n".join(lines)
-
- f=open(path, 'w')
- try:
- f.write(s)
- finally:
- f.close()
-
if __name__ == "__main__":
main(sys.argv)
diff --git a/buildscripts/packaging/msi/.gitignore b/buildscripts/packaging/msi/.gitignore
new file mode 100755
index 00000000000..bd700f967b6
--- /dev/null
+++ b/buildscripts/packaging/msi/.gitignore
@@ -0,0 +1,2 @@
+bin
+obj \ No newline at end of file
diff --git a/buildscripts/packaging/msi/Banner.bmp b/buildscripts/packaging/msi/Banner.bmp
new file mode 100644
index 00000000000..a4111db8dde
--- /dev/null
+++ b/buildscripts/packaging/msi/Banner.bmp
Binary files differ
diff --git a/buildscripts/packaging/msi/Dialog.bmp b/buildscripts/packaging/msi/Dialog.bmp
new file mode 100644
index 00000000000..120b4d8c1ed
--- /dev/null
+++ b/buildscripts/packaging/msi/Dialog.bmp
Binary files differ
diff --git a/buildscripts/packaging/msi/GNU-AGPL-3.0.rtf b/buildscripts/packaging/msi/GNU-AGPL-3.0.rtf
new file mode 100755
index 00000000000..0b62f08acd5
--- /dev/null
+++ b/buildscripts/packaging/msi/GNU-AGPL-3.0.rtf
@@ -0,0 +1,241 @@
+{\rtf1\ansi\ansicpg1252\cocoartf1265
+{\fonttbl\f0\froman\fcharset0 Times-Roman;\f1\fmodern\fcharset0 Courier;}
+{\colortbl;\red255\green255\blue255;\red0\green0\blue233;}
+{\*\listtable{\list\listtemplateid1\listhybrid{\listlevel\levelnfc23\levelnfcn23\leveljc0\leveljcn0\levelfollow0\levelstartat1\levelspace360\levelindent0{\*\levelmarker \{disc\}}{\leveltext\leveltemplateid1\'01\uc0\u8226 ;}{\levelnumbers;}\fi-360\li720\lin720 }{\listname ;}\listid1}
+{\list\listtemplateid2\listhybrid{\listlevel\levelnfc23\levelnfcn23\leveljc0\leveljcn0\levelfollow0\levelstartat1\levelspace360\levelindent0{\*\levelmarker \{disc\}}{\leveltext\leveltemplateid101\'01\uc0\u8226 ;}{\levelnumbers;}\fi-360\li720\lin720 }{\listname ;}\listid2}
+{\list\listtemplateid3\listhybrid{\listlevel\levelnfc23\levelnfcn23\leveljc0\leveljcn0\levelfollow0\levelstartat1\levelspace360\levelindent0{\*\levelmarker \{disc\}}{\leveltext\leveltemplateid201\'01\uc0\u8226 ;}{\levelnumbers;}\fi-360\li720\lin720 }{\listname ;}\listid3}}
+{\*\listoverridetable{\listoverride\listid1\listoverridecount0\ls1}{\listoverride\listid2\listoverridecount0\ls2}{\listoverride\listid3\listoverridecount0\ls3}}
+{\info
+{\author Sridhar Nanjundeswaran}}\margl1501\margr1502\vieww6520\viewh14760\viewkind0
+\deftab720
+\pard\pardeftab720\sa280\qc
+
+\f0\b\fs28 \cf0 GNU AFFERO GENERAL PUBLIC LICENSE\
+\pard\pardeftab720\sa240\qc
+
+\b0\fs24 \cf0 Version 3, 19 November 2007\
+\pard\pardeftab720
+\cf0 Copyright \'a9 2007 Free Software Foundation, Inc. <{\field{\*\fldinst{HYPERLINK "http://fsf.org/"}}{\fldrslt \cf2 \ul \ulc2 http://fsf.org/}}>\
+\
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.\
+\
+\pard\pardeftab720\sa280
+
+\b\fs28 \cf0 Preamble\
+\pard\pardeftab720\sa240
+
+\b0\fs24 \cf0 The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.\
+The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.\
+When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.\
+Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.\
+A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.\
+The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.\
+An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.\
+The precise terms and conditions for copying, distribution and modification follow.\
+\pard\pardeftab720\sa280
+
+\b\fs28 \cf0 TERMS AND CONDITIONS\
+\pard\pardeftab720\sa319
+
+\fs24 \cf0 0. Definitions.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 "This License" refers to version 3 of the GNU Affero General Public License.\
+"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.\
+"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.\
+To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.\
+A "covered work" means either the unmodified Program or a work based on the Program.\
+To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.\
+To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.\
+An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.\
+\pard\pardeftab720\sa319
+
+\b \cf0 1. Source Code.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.\
+A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.\
+The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.\
+The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work.\
+The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.\
+The Corresponding Source for a work in source code form is that same work.\
+\pard\pardeftab720\sa319
+
+\b \cf0 2. Basic Permissions.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.\
+You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.\
+Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.\
+\pard\pardeftab720\sa319
+
+\b \cf0 3. Protecting Users' Legal Rights From Anti-Circumvention Law.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.\
+When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.\
+\pard\pardeftab720\sa319
+
+\b \cf0 4. Conveying Verbatim Copies.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.\
+You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.\
+\pard\pardeftab720\sa319
+
+\b \cf0 5. Conveying Modified Source Versions.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:\
+\pard\tx220\tx720\pardeftab720\li720\fi-720
+\ls1\ilvl0\cf0 {\listtext \'95 }a) The work must carry prominent notices stating that you modified it, and giving a relevant date.\
+{\listtext \'95 }b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".\
+{\listtext \'95 }c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.\
+{\listtext \'95 }d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.\
+\pard\pardeftab720\sa240
+\cf0 A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.\
+\pard\pardeftab720\sa319
+
+\b \cf0 6. Conveying Non-Source Forms.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:\
+\pard\tx220\tx720\pardeftab720\li720\fi-720
+\ls2\ilvl0\cf0 {\listtext \'95 }a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.\
+{\listtext \'95 }b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.\
+{\listtext \'95 }c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.\
+{\listtext \'95 }d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.\
+{\listtext \'95 }e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.\
+\pard\pardeftab720\sa240
+\cf0 A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.\
+A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.\
+"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.\
+If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).\
+The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.\
+Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.\
+\pard\pardeftab720\sa319
+
+\b \cf0 7. Additional Terms.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.\
+When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.\
+Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:\
+\pard\tx220\tx720\pardeftab720\li720\fi-720
+\ls3\ilvl0\cf0 {\listtext \'95 }a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or\
+{\listtext \'95 }b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or\
+{\listtext \'95 }c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or\
+{\listtext \'95 }d) Limiting the use for publicity purposes of names of licensors or authors of the material; or\
+{\listtext \'95 }e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or\
+{\listtext \'95 }f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.\
+\pard\pardeftab720\sa240
+\cf0 All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.\
+If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.\
+Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.\
+\pard\pardeftab720\sa319
+
+\b \cf0 8. Termination.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).\
+However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.\
+Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.\
+Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.\
+\pard\pardeftab720\sa319
+
+\b \cf0 9. Acceptance Not Required for Having Copies.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.\
+\pard\pardeftab720\sa319
+
+\b \cf0 10. Automatic Licensing of Downstream Recipients.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.\
+An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.\
+You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.\
+\pard\pardeftab720\sa319
+
+\b \cf0 11. Patents.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".\
+A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.\
+Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.\
+In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.\
+If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.\
+If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.\
+A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.\
+Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.\
+\pard\pardeftab720\sa319
+
+\b \cf0 12. No Surrender of Others' Freedom.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.\
+\pard\pardeftab720\sa319
+
+\b \cf0 13. Remote Network Interaction; Use with the GNU General Public License.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.\
+Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.\
+\pard\pardeftab720\sa319
+
+\b \cf0 14. Revised Versions of this License.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.\
+Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.\
+If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.\
+Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.\
+\pard\pardeftab720\sa319
+
+\b \cf0 15. Disclaimer of Warranty.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.\
+\pard\pardeftab720\sa319
+
+\b \cf0 16. Limitation of Liability.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\
+\pard\pardeftab720\sa319
+
+\b \cf0 17. Interpretation of Sections 15 and 16.\
+\pard\pardeftab720\sa240
+
+\b0 \cf0 If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.\
+END OF TERMS AND CONDITIONS\
+\pard\pardeftab720\sa280
+
+\b\fs28 \cf0 How to Apply These Terms to Your New Programs\
+\pard\pardeftab720\sa240
+
+\b0\fs24 \cf0 If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.\
+To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.\
+\pard\pardeftab720
+
+\f1 \cf0 <one line to give the program's name and a brief idea of what it does.>\
+ Copyright (C) <year> <name of author>\
+\
+ This program is free software: you can redistribute it and/or modify\
+ it under the terms of the GNU Affero General Public License as\
+ published by the Free Software Foundation, either version 3 of the\
+ License, or (at your option) any later version.\
+\
+ This program is distributed in the hope that it will be useful,\
+ but WITHOUT ANY WARRANTY; without even the implied warranty of\
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\
+ GNU Affero General Public License for more details.\
+\
+ You should have received a copy of the GNU Affero General Public License\
+ along with this program. If not, see <http://www.gnu.org/licenses/>.\
+\pard\pardeftab720\sa240
+
+\f0 \cf0 Also add information on how to contact you by electronic and paper mail.\
+If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.\
+You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see <{\field{\*\fldinst{HYPERLINK "http://www.gnu.org/licenses/"}}{\fldrslt \cf2 \ul \ulc2 http://www.gnu.org/licenses/}}>.\
+} \ No newline at end of file
diff --git a/buildscripts/packaging/msi/Installer_Icon_16x16.ico b/buildscripts/packaging/msi/Installer_Icon_16x16.ico
new file mode 100644
index 00000000000..59f7a38a22c
--- /dev/null
+++ b/buildscripts/packaging/msi/Installer_Icon_16x16.ico
Binary files differ
diff --git a/buildscripts/packaging/msi/Installer_Icon_32x32.ico b/buildscripts/packaging/msi/Installer_Icon_32x32.ico
new file mode 100644
index 00000000000..535ccb8d060
--- /dev/null
+++ b/buildscripts/packaging/msi/Installer_Icon_32x32.ico
Binary files differ
diff --git a/buildscripts/packaging/msi/MongoDB.wixproj b/buildscripts/packaging/msi/MongoDB.wixproj
new file mode 100755
index 00000000000..3149b72d1f8
--- /dev/null
+++ b/buildscripts/packaging/msi/MongoDB.wixproj
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">x86</Platform>
+ <ProductVersion>3.5</ProductVersion>
+ <ProjectGuid>{fc40ea06-5d8c-4edf-9e19-a0bdd9a3a7d5}</ProjectGuid>
+ <SchemaVersion>2.0</SchemaVersion>
+ <OutputName>MongoDB_$(Version)_x86_Standard</OutputName>
+ <OutputType>Package</OutputType>
+ <DefineSolutionProperties>false</DefineSolutionProperties>
+ <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
+ <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
+ <ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
+ <UpgradeCode Condition=" '$(UpgradeCode)' == '' ">867C1D1D-2040-4E90-B04E-1158F9CBDE96</UpgradeCode>
+ <Name>MongoDB</Name>
+ <OutputPath>bin\$(Configuration)\$(Platform)\</OutputPath>
+ <IntermediateOutputPath>obj\$(Configuration)\$(Platform)\</IntermediateOutputPath>
+ <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
+ <VerboseOutput>True</VerboseOutput>
+ </PropertyGroup>
+ <PropertyGroup>
+ <ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\normal\clientlib</ClientSource>
+ <License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
+ <Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\normal\mongo</Source>
+ <Version Condition=" '$(Version)' == '' ">2.4.0</Version>
+ <ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
+ <DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=Standard;ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);ClientSource=$(ClientSource);ClientHeaderSource=$(ClientHeaderSource)</DefineConstants>
+ </PropertyGroup>
+ <ItemGroup>
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer.wxs" />
+ <Compile Include="$(OutputPath)DriverInclude.wxs" />
+ </ItemGroup>
+ <ItemGroup>
+ <WixExtension Include="WixUIExtension">
+ <HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
+ <Name>WixUIExtension</Name>
+ </WixExtension>
+ </ItemGroup>
+ <ItemGroup>
+ <Folder Include="wxs\" />
+ </ItemGroup>
+ <Import Project="$(WixTargetsPath)" />
+ <PropertyGroup>
+ <PreBuildEvent>"%WIX%\bin\heat.exe" dir $(ClientHeaderSource) -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
+ </PropertyGroup>
+ <!--
+ To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Wix.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
diff --git a/buildscripts/packaging/msi/MongoDBMsi.sln b/buildscripts/packaging/msi/MongoDBMsi.sln
new file mode 100755
index 00000000000..a029e91ce5b
--- /dev/null
+++ b/buildscripts/packaging/msi/MongoDBMsi.sln
@@ -0,0 +1,51 @@
+
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB", "MongoDB.wixproj", "{FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}"
+EndProject
+Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "MongoDB_64", "MongoDB_64.wixproj", "{FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2E84BEF1-F2D9-4A6B-B102-EC941AF313AF}"
+ ProjectSection(SolutionItems) = preProject
+ build32bitmsi.bat = build32bitmsi.bat
+ build64bit2008R2msi.bat = build64bit2008R2msi.bat
+ build64bitmsi.bat = build64bitmsi.bat
+ buildenterprisemsi.bat = buildenterprisemsi.bat
+ README.md = README.md
+ EndProjectSection
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Mixed Platforms = Debug|Mixed Platforms
+ Debug|x64 = Debug|x64
+ Debug|x86 = Debug|x86
+ Release|Mixed Platforms = Release|Mixed Platforms
+ Release|x64 = Release|x64
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.ActiveCfg = Debug|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|Mixed Platforms.Build.0 = Debug|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x64.ActiveCfg = Debug|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.ActiveCfg = Debug|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Debug|x86.Build.0 = Debug|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.ActiveCfg = Release|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|Mixed Platforms.Build.0 = Release|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x64.ActiveCfg = Release|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.ActiveCfg = Release|x86
+ {FC40EA06-5D8C-4EDF-9E19-A0BDD9A3A7D5}.Release|x86.Build.0 = Release|x86
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.ActiveCfg = Debug|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|Mixed Platforms.Build.0 = Debug|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.ActiveCfg = Debug|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x64.Build.0 = Debug|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Debug|x86.ActiveCfg = Debug|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.ActiveCfg = Release|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|Mixed Platforms.Build.0 = Release|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.ActiveCfg = Release|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x64.Build.0 = Release|x64
+ {FA9DF7FC-A283-4EB8-B0C6-F9FA31E22CBC}.Release|x86.ActiveCfg = Release|x64
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/buildscripts/packaging/msi/MongoDB_64.wixproj b/buildscripts/packaging/msi/MongoDB_64.wixproj
new file mode 100755
index 00000000000..d3c3be6dcae
--- /dev/null
+++ b/buildscripts/packaging/msi/MongoDB_64.wixproj
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <PropertyGroup>
+ <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+ <Platform Condition=" '$(Platform)' == '' ">x64</Platform>
+ <ProductVersion>3.5</ProductVersion>
+ <ProjectGuid>{fa9df7fc-a283-4eb8-b0c6-f9fa31e22cbc}</ProjectGuid>
+ <SchemaVersion>2.0</SchemaVersion>
+ <OutputName>MongoDB_$(Version)_x64_$(Flavor)_$(Edition)</OutputName>
+ <OutputType>Package</OutputType>
+ <DefineSolutionProperties>false</DefineSolutionProperties>
+ <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
+ <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
+ <Name>MongoDB</Name>
+ <OutputPath>bin\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</OutputPath>
+ <IntermediateOutputPath>obj\$(Configuration)\$(Platform)\$(Flavor)\$(Edition)\</IntermediateOutputPath>
+ <TreatWarningsAsErrors>True</TreatWarningsAsErrors>
+ <ProductId Condition=" '$(ProductId)' == '' ">*</ProductId>
+ <UpgradeCode Condition=" '$(UpgradeCode)' == '' ">FCF901F6-E963-40B1-9A17-978242068587</UpgradeCode>
+ </PropertyGroup>
+ <PropertyGroup>
+ <ClientSource Condition=" '$(ClientSource)' == '' ">..\..\..\build\win32\64\client_build</ClientSource>
+ <Edition Condition=" '$(Edition)' == '' ">Standard</Edition>
+ <Flavor Condition=" '$(Flavor)' == '' ">2008R2Plus</Flavor>
+ <License Condition=" '$(License)' == '' ">..\..\..\distsrc</License>
+ <EnterpriseBase Condition=" '$(EnterpriseBase)' == '' ">..\..\..\src\mongo\db\modules\subscription</EnterpriseBase>
+ <SaslSource Condition=" '$(SaslSource)' == '' ">..\..\..\build\win32\64\mongo</SaslSource>
+ <SnmpSource Condition=" '$(SnmpSource)' == '' ">..\..\..\build\win32\64\mongo</SnmpSource>
+ <Source Condition=" '$(Source)' == '' ">..\..\..\build\win32\64\mongo</Source>
+ <SslSource Condition=" '$(SslSource)' == '' ">..\..\..\build\win32\64\mongo</SslSource>
+ <Version Condition=" '$(Version)' == '' ">2.4.0</Version>
+ <MergeModulesBasePath Condition=" '$(MergeModulesBasePath)' == '' ">C:\Program Files (x86)\Common Files\Merge Modules</MergeModulesBasePath>
+ <ClientHeaderSource Condition=" '$(ClientHeaderSource)' == '' ">..\..\..\build\win32\normal\clientlib\include</ClientHeaderSource>
+ <DefineConstants>MongoDBVersion=$(Version);LicenseSource=$(License);BinarySource=$(Source);Edition=$(Edition);SaslSource=$(SaslSource);SslSource=$(SslSource);SnmpSource=$(SnmpSource);ProductId=$(ProductId);UpgradeCode=$(UpgradeCode);Flavor=$(Flavor);ClientSource=$(ClientSource);EnterpriseBase=$(EnterpriseBase);ClientHeaderSource=$(ClientHeaderSource);MergeModulesBasePath=$(MergeModulesBasePath)</DefineConstants>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
+ <VerboseOutput>True</VerboseOutput>
+ </PropertyGroup>
+ <PropertyGroup Condition=" '$(Edition)' == 'Enterprise' ">
+ <TreatWarningsAsErrors>False</TreatWarningsAsErrors>
+ </PropertyGroup>
+ <ItemGroup>
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\BinaryFragment.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\FeatureFragment.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\Installer_64.wxs" />
+ <Compile Include="..\..\..\src\mongo\installer\msi\wxs\LicensingFragment.wxs" />
+ <Compile Include="$(OutputPath)DriverInclude.wxs" />
+ </ItemGroup>
+ <ItemGroup>
+ <WixExtension Include="WixUIExtension">
+ <HintPath>$(WixExtDir)\WixUIExtension.dll</HintPath>
+ <Name>WixUIExtension</Name>
+ </WixExtension>
+ </ItemGroup>
+ <ItemGroup>
+ <Folder Include="wxs\" />
+ </ItemGroup>
+ <Import Project="$(WixTargetsPath)" />
+ <PropertyGroup>
+ <PreBuildEvent>"%WIX%\bin\heat.exe" dir "$(ClientHeaderSource)" -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource</PreBuildEvent>
+ </PropertyGroup>
+ <!--
+ To modify your build process, add your task inside one of the targets below and uncomment it.
+ Other similar extension points exist, see Wix.targets.
+ <Target Name="BeforeBuild">
+ </Target>
+ <Target Name="AfterBuild">
+ </Target>
+ -->
+</Project>
diff --git a/buildscripts/packaging/msi/README.md b/buildscripts/packaging/msi/README.md
new file mode 100755
index 00000000000..93e6a593114
--- /dev/null
+++ b/buildscripts/packaging/msi/README.md
@@ -0,0 +1,36 @@
+## Prerequisites
+WiX Toolset v3.7.1224.0 from http://wixtoolset.org/
+
+## Features
+The following are the installer features with the executables they install.
+Each of these features can be installed independently using msiexec /ADDLOCAL
+or using the Installer GUI
+ * Server
+ * mongod.exe
+ * mongod.pdb
+ * Client
+ * mongo.exe
+ * MonitoringTools
+ * mongostat.exe
+ * mongotop.exe
+ * ImportExportTools
+ * mongodump.exe
+ * mongorestore.exe
+ * mongoexport.exe
+ * mongoimport.exe
+ * Router
+ * mongos.exe
+ * mongos.pdb
+ * MiscellaneousTools
+ * bsondump.exe
+ * mongofiles.exe
+ * mongooplog.exe
+ * mongoperf.exe
+
+## Typical install
+The typical (default) install, installs all except the Router and
+MiscellaneousTools features.
+
+## Configuring builds
+The version, location of binaries and license file can be configured when
+building. Refer to build32bitmsi.bat or build64bitmsi.bat for example
diff --git a/buildscripts/packaging/msi/build32bitmsi.bat b/buildscripts/packaging/msi/build32bitmsi.bat
new file mode 100755
index 00000000000..7fdf970eb7e
--- /dev/null
+++ b/buildscripts/packaging/msi/build32bitmsi.bat
@@ -0,0 +1,100 @@
+@ECHO OFF
+
+SET VERSION=2.4.0
+SET BINDIR=..\..\..\build\win32\normal\mongo
+SET CLIENTLIBDIR=..\..\..\build\win32\normal\client_build
+SET LICENSEDIR=..\..\..\distsrc
+SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
+SET WIXBINDIR=C:\Program Files (x86)\WiX Toolset v3.7\bin
+
+SET PLATFORM=x86
+SET GENERATEDWXSDIR=.\wxs
+SET EDITION=Standard
+SET CONFIGURATION=Release
+SET OUTPUTOBJDIR=obj\%CONFIGURATION%\%PLATFORM%\
+SET OUTPUTBINDIR=bin\%CONFIGURATION%\%PLATFORM%\
+SET PROJECTDIR=C:\git\sridharn\mongo\buildscripts\packaging\msi\
+SET TARGETNAME=MongoDB_%VERSION%_%PLATFORM%_%EDITION%
+
+:loop
+IF NOT "%1"=="" (
+ IF "%1"=="-version" (
+ SET VERSION=%2
+ SHIFT
+ )
+ IF "%1"=="-bindir" (
+ SET BINDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-licensedir" (
+ SET LICENSEDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientlibdir" (
+ SET CLIENTLIBDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientheaderdir" (
+ SET CLIENTHEADERDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-wixbindir" (
+ SET WIXBINDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-generatedwxsdir" (
+ SET GENERATEDWXSDIR=%2
+ SHIFT
+ )
+ SHIFT
+ GOTO :loop
+)
+
+REM ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
+REM %WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;
+REM Source=%BINDIR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB.wixproj
+
+ECHO Generating %GENERATEDWXSDIR%\DriverInclude.wxs from sources at %CLIENTHEADERDIR%
+"%WIXBINDIR%\heat.exe" dir %CLIENTHEADERDIR% -gg -g1 -frag -cg cg_DriverHeaders -nologo -directoryid -out %GENERATEDWXSDIR%\DriverInclude.wxs -dr Header -srd -var var.ClientHeaderSource
+
+ECHO Compiling wxs files to obj
+"%WIXBINDIR%\candle.exe" -wx^
+ -dMongoDBVersion=%VERSION%^
+ -dLicenseSource=%LICENSEDIR%^
+ -dBinarySource=%BINDIR%^
+ -dEdition=%EDITION%^
+ -d"ProductId=*"^
+ -dUpgradeCode=867C1D1D-2040-4E90-B04E-1158F9CBDE96^
+ -dClientSource=%CLIENTLIBDIR%^
+ -dClientHeaderSource=%CLIENTHEADERDIR%^
+ -dConfiguration=%CONFIGURATION%^
+ -dOutDir=%OUTPUTBINDIR%^
+ -dPlatform=%PLATFORM%^
+ -dProjectDir=%PROJECTDIR%^
+ -dProjectExt=.wixproj^
+ -dProjectFileName=MongoDB.wixproj^
+ -dProjectName=MongoDB^
+ -dProjectPath=%PROJECTDIR%\MongoDB.wixproj^
+ -dTargetDir=%OUTPUTBINDIR%^
+ -dTargetExt=.msi^
+ -dTargetFileName=%TARGETNAME%.msi^
+ -dTargetName=%TARGETNAME%^
+ -dTargetPath=%OUTPUTBINDIR%\%TARGETNAME%.msi^
+ -out %OUTPUTOBJDIR%^
+ -arch %PLATFORM%^
+ -ext "%WIXBINDIR%\WixUIExtension.dll"^
+ wxs\BinaryFragment.wxs wxs\FeatureFragment.wxs wxs\LicensingFragment.wxs wxs\Installer.wxs %GENERATEDWXSDIR%\DriverInclude.wxs
+
+ECHO Linking to msi
+"%WIXBINDIR%\Light.exe"^
+ -out %OUTPUTBINDIR%\%TARGETNAME%.msi^
+ -pdbout %OUTPUTBINDIR%\%TARGETNAME%.wixpdb^
+ -wx -cultures:null^
+ -ext "%WIXBINDIR%\WixUIExtension.dll"^
+ -contentsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindContentsFileListnull.txt^
+ -outputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindOutputsFileListnull.txt^
+ -builtoutputsfile %OUTPUTOBJDIR%\MongoDB.wixproj.BindBuiltOutputsFileListnull.txt^
+ -wixprojectfile %PROJECTDIR%\MongoDB.wixproj^
+ %OUTPUTOBJDIR%\BinaryFragment.wixobj %OUTPUTOBJDIR%\FeatureFragment.wixobj^
+ %OUTPUTOBJDIR%\\LicensingFragment.wixobj %OUTPUTOBJDIR%\Installer.wixobj^
+ %OUTPUTOBJDIR%\DriverInclude.wixobj
diff --git a/buildscripts/packaging/msi/build64bit2008R2msi.bat b/buildscripts/packaging/msi/build64bit2008R2msi.bat
new file mode 100644
index 00000000000..1dc0938ea94
--- /dev/null
+++ b/buildscripts/packaging/msi/build64bit2008R2msi.bat
@@ -0,0 +1,37 @@
+@ECHO OFF
+SET VERSION=2.4.0
+SET BINDIR=..\..\..\build\win32\64\mongo
+SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
+SET LICENSEDIR=..\..\..\distsrc
+SET EDITION=Standard
+SET FLAVOR=2008R2Plus
+SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
+
+:loop
+IF NOT "%1"=="" (
+ IF "%1"=="-version" (
+ SET VERSION=%2
+ SHIFT
+ )
+ IF "%1"=="-bindir" (
+ SET BINDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-licensedir" (
+ SET LICENSEDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientlibdir" (
+ SET CLIENTLIBDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientheaderdir" (
+ SET CLIENTHEADERDIR=%2
+ SHIFT
+ )
+ SHIFT
+ GOTO :loop
+)
+
+ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
+%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj \ No newline at end of file
diff --git a/buildscripts/packaging/msi/build64bitmsi.bat b/buildscripts/packaging/msi/build64bitmsi.bat
new file mode 100755
index 00000000000..ccc6f83f9fa
--- /dev/null
+++ b/buildscripts/packaging/msi/build64bitmsi.bat
@@ -0,0 +1,37 @@
+@ECHO OFF
+SET VERSION=2.4.0
+SET BINDIR=..\..\..\build\win32\64\mongo
+SET CLIENTLIBDIR=..\..\..\build\win32\64\client_build
+SET LICENSEDIR=..\..\..\distsrc
+SET EDITION=Standard
+SET FLAVOR=2008
+SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
+
+:loop
+IF NOT "%1"=="" (
+ IF "%1"=="-version" (
+ SET VERSION=%2
+ SHIFT
+ )
+ IF "%1"=="-bindir" (
+ SET BINDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-licensedir" (
+ SET LICENSEDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientlibdir" (
+ SET CLIENTLIBDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientheaderdir" (
+ SET CLIENTHEADERDIR=%2
+ SHIFT
+ )
+ SHIFT
+ GOTO :loop
+)
+
+ECHO Building msi for version %VERSION% with binaries from %BINDIR% and license files from %LICENSEDIR%
+%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;ClientHeaderSource=%CLIENTHEADERDIR% MongoDB_64.wixproj \ No newline at end of file
diff --git a/buildscripts/packaging/msi/buildenterprisemsi.bat b/buildscripts/packaging/msi/buildenterprisemsi.bat
new file mode 100644
index 00000000000..c9bba5bf73d
--- /dev/null
+++ b/buildscripts/packaging/msi/buildenterprisemsi.bat
@@ -0,0 +1,63 @@
+@ECHO OFF
+SET VERSION=2.6.0
+SET BINDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\mongo
+SET CLIENTLIBDIR=..\..\..\build\win32\64\dynamic-windows\extrapathdyn_c__Utils_sasl_c__Utils_snmp_c__Utils_ssl\release\ssl\client_build
+SET LICENSEDIR=..\..\..\distsrc
+SET ENTERPRISEBASEDIR=..\..\..\src\mongo\db\modules\subscription
+SET EDITION=Enterprise
+SET FLAVOR=2008R2Plus
+SET SASLDIR=..\..\..\..\..\..\Utils\sasl\bin
+SET OPENSSLDIR=..\..\..\..\..\..\Utils\openssl\bin
+SET SNMPDIR=..\..\..\..\..\..\Utils\snmp\bin
+SET CLIENTHEADERDIR=..\..\..\build\win32\normal\client_build\include
+SET MERGEMODULESBASEPATH="C:\Program Files (x86)\Common Files\Merge Modules"
+
+:loop
+IF NOT "%1"=="" (
+ IF "%1"=="-version" (
+ SET VERSION=%2
+ SHIFT
+ )
+ IF "%1"=="-bindir" (
+ SET BINDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-licensedir" (
+ SET LICENSEDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-sasldir" (
+ SET SASLDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-openssldir" (
+ SET OPENSSLDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-snmpdir" (
+ SET SNMPDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientlibdir" (
+ SET CLIENTLIBDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-enterprisebasedir" (
+ SET ENTERPRISEBASEDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-clientheaderdir" (
+ SET CLIENTHEADERDIR=%2
+ SHIFT
+ )
+ IF "%1"=="-mergemodulesbasepath" (
+ SET MERGEMODULESBASEPATH=%2
+ SHIFT
+ )
+ SHIFT
+ GOTO :loop
+)
+
+ECHO Building enterprise msi for version %VERSION% with binaries from %BINDIR%, sasl from %SASLDIR%, ssl from %OPENSSLDIR%, snmp from %SNMPDIR% and license files from %LICENSEDIR%
+
+%WINDIR%\Microsoft.NET\Framework64\v4.0.30319\msbuild /p:Configuration=Release;Version=%VERSION%;License=%LICENSEDIR%;Source=%BINDIR%;SaslSource=%SASLDIR%;SnmpSource=%SNMPDIR%;SslSource=%OPENSSLDIR%;Edition=%EDITION%;Flavor=%FLAVOR%;ClientSource=%CLIENTLIBDIR%;EnterpriseBase=%ENTERPRISEBASEDIR%;ClientHeaderSource=%CLIENTHEADERDIR%;MergeModulesBasePath=%MERGEMODULESBASEPATH% MongoDB_64.wixproj
diff --git a/buildscripts/s3sign.py b/buildscripts/s3sign.py
new file mode 100755
index 00000000000..356cc4b7c3a
--- /dev/null
+++ b/buildscripts/s3sign.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+#
+# Generate and upload detached gpg signatures for archive files in Amazon S3
+#
+# Requires standard MongoDB settings.py, like so:
+# bucket = "downloads.mongodb.org"
+# # downloads user
+# id = "xxxxx"
+# key = "xxxxx"
+#
+# Usage: s3sign.py [ --dry-run ] [ --bucket <overridden s3 bucket> ] [ --notary-url <notary url> ] [ --key-name <key name passed to notary service> ] [ --filter <filter> ]
+#
+
+
+import argparse
+import json
+import os
+import requests
+import sys
+
+sys.path.append("." )
+sys.path.append(".." )
+sys.path.append("../../" )
+sys.path.append("../../../" )
+
+import simples3
+import settings
+import subprocess
+
+# parse command line
+#
+parser = argparse.ArgumentParser(description='Sign MongoDB S3 Files')
+parser.add_argument('--dry-run', action='store_true', required=False, help='Do not write anything to S3', default = False);
+parser.add_argument('--bucket', required = False, help='Override bucket in settings.py', default = settings.bucket);
+parser.add_argument('--notary-url', required=False, help='URL base for notary service', default = 'http://localhost:5000');
+parser.add_argument('--key-name', required=False, help='Key parameter to notary service', default = 'test');
+parser.add_argument('--filter', required=False,
+ help='Only sign files matching case-insensitive substring filter', default = None);
+args = parser.parse_args()
+
+notary_urlbase = args.notary_url
+notary_url = notary_urlbase + '/api/sign'
+notary_payload = { 'key': args.key_name, 'comment': 'Automatic archive signing'}
+
+# check s3 for pgp signatures
+
+def check_dir( bucket , prefix ):
+
+ zips = {}
+ sigs = {}
+ for ( key , modify , etag , size ) in bucket.listdir( prefix=prefix ):
+ # filtered out
+ if args.filter and args.filter.lower() not in key.lower():
+ pass
+ # sign it
+ elif key.endswith(".tgz" ) or key.endswith(".zip" ) or key.endswith(".tar.gz" ) or key.endswith("md5"):
+ # generate signature
+ files = {'file': (key, bucket.get(key))}
+ response_json = {}
+ try:
+ r = requests.post(notary_url, files=files, data=notary_payload,
+ headers = { "Accept": "application/json" })
+ # get url for signature file
+ response_json = json.loads(r.text)
+ except Exception as e:
+ print('error contacting signing service for %s:\n%s' % (key, e.message))
+ continue
+ if 'permalink' in response_json:
+ signature_url = response_json['permalink']
+ try:
+ signature = requests.get(notary_urlbase + signature_url).text
+ zips[key] = signature
+ except Exception as e:
+ print('error downloading signature from signing service for %s:\n%s' % (key, e.message))
+ else:
+ print('error from signing service for %s:\n%s' % (key, response_json.get('message')))
+ # signatures
+ elif key.endswith(".sig" ) or key.endswith(".asc" ):
+ sigs[key] = True
+ # file types we don't need to sign
+ elif key.endswith(".msi" ) or key.endswith(".deb") or key.endswith(".rpm"):
+ pass
+ else:
+ print("unknown file type: %s" % key)
+
+ for x in zips:
+ m = x + ".sig"
+ if m in sigs:
+ continue
+
+ print("need to do: " + x + " to " + m )
+ if not args.dry_run:
+ bucket.put( m , zips[x] , acl="public-read" )
+
+
+def run():
+
+ bucket = simples3.S3Bucket( args.bucket , settings.id , settings.key )
+
+ for x in [ "osx" , "linux" , "win32" , "sunos5" , "src" ]:
+ check_dir( bucket , x )
+
+
+if __name__ == "__main__":
+ run()
diff --git a/buildscripts/setup_multiversion_mongodb.py b/buildscripts/setup_multiversion_mongodb.py
index a19f560bf20..740bed8b138 100644
--- a/buildscripts/setup_multiversion_mongodb.py
+++ b/buildscripts/setup_multiversion_mongodb.py
@@ -17,6 +17,28 @@ import gzip
# Only really tested/works on Linux.
#
+def version_tuple(version):
+ """Returns a version tuple that can be used for numeric sorting
+ of version strings such as '2.6.0-rc1' and '2.4.0'"""
+
+ RC_OFFSET = -100
+ version_parts = re.split(r'\.|-', version[0])
+
+ if version_parts[-1].startswith("rc"):
+ rc_part = version_parts.pop()
+ rc_part = rc_part.split('rc')[1]
+
+ # RC versions are weighted down to allow future RCs and general
+ # releases to be sorted in ascending order (e.g., 2.6.0-rc1,
+ # 2.6.0-rc2, 2.6.0).
+ version_parts.append(int(rc_part) + RC_OFFSET)
+ else:
+ # Non-RC releases have an extra 0 appended so version tuples like
+ # (2, 6, 0, -100) and (2, 6, 0, 0) sort in ascending order.
+ version_parts.append(0)
+
+ return tuple(map(int, version_parts))
+
class MultiVersionDownloader :
def __init__(self, install_dir, link_dir, platform):
@@ -25,13 +47,29 @@ class MultiVersionDownloader :
match = re.compile("(.*)\/(.*)").match(platform)
self.platform = match.group(1)
self.arch = match.group(2)
- self.links = self.download_links()
+ self._links = None
+
+ @property
+ def links(self):
+ if self._links is None:
+ self._links = self.download_links()
+ return self._links
def download_links(self):
href = "http://dl.mongodb.org/dl/%s/%s" \
% (self.platform.lower(), self.arch)
- html = urllib2.urlopen(href).read()
+ attempts_remaining = 5
+ timeout_seconds = 10
+ while True:
+ try:
+ html = urllib2.urlopen(href, timeout = timeout_seconds).read()
+ break
+ except Exception as e:
+ print "fetching links failed (%s), retrying..." % e
+ attempts_remaining -= 1
+ if attempts_remaining == 0 :
+ raise Exception("Failed to get links after multiple retries")
links = {}
for line in html.split():
@@ -70,39 +108,45 @@ class MultiVersionDownloader :
raise Exception("Cannot find a link for version %s, versions %s found." \
% (version, self.links))
- urls.sort()
+ urls.sort(key=version_tuple)
full_version = urls[-1][0]
url = urls[-1][1]
-
- temp_dir = tempfile.mkdtemp()
- temp_file = tempfile.mktemp(suffix=".tgz")
-
- data = urllib2.urlopen(url)
-
- print "Downloading data for version %s (%s)..." % (version, full_version)
-
- with open(temp_file, 'wb') as f:
- f.write(data.read())
- print "Uncompressing data for version %s (%s)..." % (version, full_version)
-
- # Can't use cool with syntax b/c of python 2.6
- tf = tarfile.open(temp_file, 'r:gz')
-
- try:
- tf.extractall(path=temp_dir)
- except:
+ extract_dir = url.split("/")[-1][:-4]
+
+ # only download if we don't already have the directory
+ already_downloaded = os.path.isdir(os.path.join( self.install_dir, extract_dir))
+ if already_downloaded:
+ print "Skipping download for version %s (%s) since the dest already exists '%s'" \
+ % (version, full_version, extract_dir)
+ else:
+ temp_dir = tempfile.mkdtemp()
+ temp_file = tempfile.mktemp(suffix=".tgz")
+
+ data = urllib2.urlopen(url)
+
+ print "Downloading data for version %s (%s)..." % (version, full_version)
+
+ with open(temp_file, 'wb') as f:
+ f.write(data.read())
+ print "Uncompressing data for version %s (%s)..." % (version, full_version)
+
+ # Can't use cool with syntax b/c of python 2.6
+ tf = tarfile.open(temp_file, 'r:gz')
+
+ try:
+ tf.extractall(path=temp_dir)
+ except:
+ tf.close()
+ raise
+
tf.close()
- raise
-
- tf.close()
-
- extract_dir = os.listdir(temp_dir)[0]
- temp_install_dir = os.path.join(temp_dir, extract_dir)
-
- shutil.move(temp_install_dir, self.install_dir)
-
- shutil.rmtree(temp_dir)
- os.remove(temp_file)
+
+ temp_install_dir = os.path.join(temp_dir, extract_dir)
+
+ shutil.move(temp_install_dir, self.install_dir)
+
+ shutil.rmtree(temp_dir)
+ os.remove(temp_file)
self.symlink_version(version, os.path.abspath(os.path.join(self.install_dir, extract_dir)))
@@ -120,20 +164,32 @@ class MultiVersionDownloader :
link_name = "%s-%s" % (executable, version)
- os.symlink(os.path.join(installed_dir, "bin", executable),\
- os.path.join(self.link_dir, link_name))
+ try:
+ os.symlink(os.path.join(installed_dir, "bin", executable),\
+ os.path.join(self.link_dir, link_name))
+ except OSError as exc:
+ if exc.errno == errno.EEXIST:
+ pass
+ else: raise
CL_HELP_MESSAGE = \
"""
-Downloads and installs particular mongodb versions into an install directory and symlinks the binaries with versions to
-another directory.
+Downloads and installs particular mongodb versions (each binary is renamed to include its version)
+into an install directory and symlinks the binaries with versions to another directory.
+
+Usage: setup_multiversion_mongodb.py INSTALL_DIR LINK_DIR PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...]
+
+Ex: setup_multiversion_mongodb.py ./install ./link "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3"
+Ex: setup_multiversion_mongodb.py ./install ./link "OSX/x86_64" "2.4" "2.2"
-Usage: install_multiversion_mongodb.sh INSTALL_DIR LINK_DIR PLATFORM_AND_ARCH VERSION1 [VERSION2 VERSION3 ...]
+After running the script you will have a directory structure like this:
+./install/[mongodb-osx-x86_64-2.4.9, mongodb-osx-x86_64-2.2.7]
+./link/[mongod-2.4.9, mongod-2.2.7, mongo-2.4.9...]
-Ex: install_multiversion_mongodb.sh ./install ./link "Linux/x86_64" "2.0.6" "2.0.3-rc0" "2.0" "2.2" "2.3"
+You should then add ./link/ to your path so multi-version tests will work.
-If "rc" is included in the version name, we'll use the exact rc, otherwise we'll pull the highest non-rc
+Note: If "rc" is included in the version name, we'll use the exact rc, otherwise we'll pull the highest non-rc
version compatible with the version specified.
"""
diff --git a/buildscripts/smoke.py b/buildscripts/smoke.py
index ebc0641e538..85a62c76f60 100755
--- a/buildscripts/smoke.py
+++ b/buildscripts/smoke.py
@@ -34,24 +34,24 @@
# jobs on the same host at once. So something's gotta change.
from datetime import datetime
+from itertools import izip
import glob
from optparse import OptionParser
import os
-import parser
+import pprint
import re
-import shutil
import shlex
import socket
import stat
-from subprocess import (Popen,
- PIPE,
- call)
+from subprocess import (PIPE, Popen, STDOUT)
import sys
import time
-from pymongo import Connection
+from pymongo import MongoClient
from pymongo.errors import OperationFailure
+from pymongo import ReadPreference
+import cleanbb
import utils
try:
@@ -83,6 +83,9 @@ shell_executable = None
continue_on_failure = None
file_of_commands_mode = False
start_mongod = True
+temp_path = None
+clean_every_n_tests = 1
+clean_whole_dbroot = False
tests = []
winners = []
@@ -99,7 +102,8 @@ smoke_db_prefix = ''
small_oplog = False
small_oplog_rs = False
-all_test_results = []
+test_report = { "results": [] }
+report_file = None
# This class just implements the with statement API, for a sneaky
# purpose below.
@@ -122,6 +126,15 @@ def buildlogger(cmd, is_global=False):
return cmd
+def clean_dbroot(dbroot="", nokill=False):
+ # Clean entire /data/db dir if --with-cleanbb, else clean specific database path.
+ if clean_whole_dbroot and not small_oplog:
+ dbroot = os.path.normpath(smoke_db_prefix + "/data/db")
+ if os.path.exists(dbroot):
+ print("clean_dbroot: %s" % dbroot)
+ cleanbb.cleanup(dbroot, nokill)
+
+
class mongod(object):
def __init__(self, **kwargs):
self.kwargs = kwargs
@@ -151,28 +164,25 @@ class mongod(object):
sock.settimeout(1)
sock.connect(("localhost", int(port)))
sock.close()
-
+
+ def is_mongod_up(self, port=mongod_port):
+ try:
+ self.check_mongo_port(int(port))
+ return True
+ except Exception,e:
+ print >> sys.stderr, e
+ return False
+
def did_mongod_start(self, port=mongod_port, timeout=300):
while timeout > 0:
time.sleep(1)
- try:
- self.check_mongo_port(int(port))
+ is_up = self.is_mongod_up(port)
+ if is_up:
return True
- except Exception,e:
- print >> sys.stderr, e
- timeout = timeout - 1
+ timeout = timeout - 1
print >> sys.stderr, "timeout starting mongod"
return False
- def setup_admin_user(self, port=mongod_port):
- try:
- Connection( "localhost" , int(port) ).admin.add_user("admin","password")
- except OperationFailure, e:
- if e.message == 'need to login':
- pass # SERVER-4225
- else:
- raise e
-
def start(self):
global mongod_port
global mongod
@@ -188,18 +198,19 @@ class mongod(object):
srcport = mongod_port
self.port += 1
self.slave = True
- if os.path.exists(dir_name):
- if 'slave' in self.kwargs:
- argv = [utils.find_python(), "buildscripts/cleanbb.py", '--nokill', dir_name]
- else:
- argv = [utils.find_python(), "buildscripts/cleanbb.py", dir_name]
- call(argv)
+
+ clean_dbroot(dbroot=dir_name, nokill=self.slave)
utils.ensureDir(dir_name)
+
argv = [mongod_executable, "--port", str(self.port), "--dbpath", dir_name]
- # This should always be set for tests
- argv += ['--setParameter', 'enableTestCommands=1']
+ # These parameters are alwas set for tests
+ # SERVER-9137 Added httpinterface parameter to keep previous behavior
+ argv += ['--setParameter', 'enableTestCommands=1', '--httpinterface']
if self.kwargs.get('small_oplog'):
argv += ["--master", "--oplogSize", "511"]
+ params = self.kwargs.get('set_parameters', None)
+ if params:
+ for p in params.split(','): argv += ['--setParameter', p]
if self.kwargs.get('small_oplog_rs'):
argv += ["--replSet", "foo", "--oplogSize", "511"]
if self.slave:
@@ -209,32 +220,34 @@ class mongod(object):
if self.kwargs.get('no_preallocj'):
argv += ['--nopreallocj']
if self.kwargs.get('auth'):
- argv += ['--auth']
+ argv += ['--auth', '--setParameter', 'enableLocalhostAuthBypass=false']
authMechanism = self.kwargs.get('authMechanism', 'MONGODB-CR')
if authMechanism != 'MONGODB-CR':
- argv.append('--setParameter=authenticationMechanisms=' + authMechanism)
+ argv += ['--setParameter', 'authenticationMechanisms=' + authMechanism]
self.auth = True
- if self.kwargs.get('use_ssl'):
- argv += ['--sslOnNormalPorts',
+ if self.kwargs.get('keyFile'):
+ argv += ['--keyFile', self.kwargs.get('keyFile')]
+ if self.kwargs.get('use_ssl') or self.kwargs.get('use_x509'):
+ argv += ['--sslMode', "requireSSL",
'--sslPEMKeyFile', 'jstests/libs/server.pem',
'--sslCAFile', 'jstests/libs/ca.pem',
'--sslWeakCertificateValidation']
-
+ if self.kwargs.get('use_x509'):
+ argv += ['--clusterAuthMode','x509'];
+ self.auth = True
print "running " + " ".join(argv)
self.proc = self._start(buildlogger(argv, is_global=True))
if not self.did_mongod_start(self.port):
raise Exception("Failed to start mongod")
- if self.auth:
- self.setup_admin_user(self.port)
-
if self.slave:
- local = Connection(port=self.port, slave_okay=True).local
+ local = MongoClient(port=self.port,
+ read_preference=ReadPreference.SECONDARY_PREFERRED).local
synced = False
while not synced:
synced = True
- for source in local.sources.find(fields=["syncedTo"]):
+ for source in local.sources.find({}, ["syncedTo"]):
synced = synced and "syncedTo" in source and source["syncedTo"]
def _start(self, argv):
@@ -243,7 +256,6 @@ class mongod(object):
child processes of this process can be killed with a single
call to TerminateJobObject (see self.stop()).
"""
- proc = Popen(argv)
if os.sys.platform == "win32":
# Create a job object with the "kill on job close"
@@ -253,6 +265,12 @@ class mongod(object):
# rather than orphaning the mongod.
import win32job
+ # Magic number needed to allow job reassignment in Windows 7
+ # see: MSDN - Process Creation Flags - ms684863
+ CREATE_BREAKAWAY_FROM_JOB = 0x01000000
+
+ proc = Popen(argv, creationflags=CREATE_BREAKAWAY_FROM_JOB)
+
self.job_object = win32job.CreateJobObject(None, '')
job_info = win32job.QueryInformationJobObject(
@@ -265,6 +283,9 @@ class mongod(object):
win32job.AssignProcessToJobObject(self.job_object, proc._handle)
+ else:
+ proc = Popen(argv)
+
return proc
def stop(self):
@@ -277,7 +298,7 @@ class mongod(object):
win32job.TerminateJobObject(self.job_object, -1)
import time
# Windows doesn't seem to kill the process immediately, so give it some time to die
- time.sleep(5)
+ time.sleep(5)
else:
# This function not available in Python 2.5
self.proc.terminate()
@@ -289,7 +310,9 @@ class mongod(object):
sys.stdout.flush()
def wait_for_repl(self):
- Connection(port=self.port).test.smokeWait.insert({}, w=2, wtimeout=5*60*1000)
+ print "Awaiting replicated (w:2, wtimeout:5min) insert (port:" + str(self.port) + ")"
+ MongoClient(port=self.port).testing.smokeWait.insert({}, w=2, wtimeout=5*60*1000)
+ print "Replicated write completed -- done wait_for_repl"
class Bug(Exception):
def __str__(self):
@@ -301,8 +324,8 @@ class TestFailure(Exception):
class TestExitFailure(TestFailure):
def __init__(self, *args):
self.path = args[0]
-
self.status=args[1]
+
def __str__(self):
return "test %s exited with status %d" % (self.path, self.status)
@@ -319,29 +342,59 @@ def check_db_hashes(master, slave):
if not slave.slave:
raise(Bug("slave instance doesn't have slave attribute set"))
- print "waiting for slave to catch up"
+ print "waiting for slave (%s) to catch up to master (%s)" % (slave.port, master.port)
master.wait_for_repl()
print "caught up!"
# FIXME: maybe make this run dbhash on all databases?
for mongod in [master, slave]:
- mongod.dbhash = Connection(port=mongod.port, slave_okay=True).test.command("dbhash")
+ client = MongoClient(port=mongod.port, read_preference=ReadPreference.SECONDARY_PREFERRED)
+ mongod.dbhash = client.test.command("dbhash")
mongod.dict = mongod.dbhash["collections"]
global lost_in_slave, lost_in_master, screwy_in_slave, replicated_collections
replicated_collections += master.dict.keys()
-
- for db in replicated_collections:
- if db not in slave.dict:
- lost_in_slave.append(db)
- mhash = master.dict[db]
- shash = slave.dict[db]
+
+ for coll in replicated_collections:
+ if coll not in slave.dict and coll not in lost_in_slave:
+ lost_in_slave.append(coll)
+ mhash = master.dict[coll]
+ shash = slave.dict[coll]
if mhash != shash:
- screwy_in_slave[db] = mhash + "/" + shash
+ mTestDB = MongoClient(port=master.port).test
+ sTestDB = MongoClient(port=slave.port,
+ read_preference=ReadPreference.SECONDARY_PREFERRED).test
+ mCount = mTestDB[coll].count()
+ sCount = sTestDB[coll].count()
+ stats = {'hashes': {'master': mhash, 'slave': shash},
+ 'counts':{'master': mCount, 'slave': sCount}}
+ try:
+ mDocs = list(mTestDB[coll].find().sort("_id", 1))
+ sDocs = list(sTestDB[coll].find().sort("_id", 1))
+ mDiffDocs = list()
+ sDiffDocs = list()
+ for left, right in izip(mDocs, sDocs):
+ if left != right:
+ mDiffDocs.append(left)
+ sDiffDocs.append(right)
+
+ stats["docs"] = {'master': mDiffDocs, 'slave': sDiffDocs }
+ except Exception, e:
+ stats["error-docs"] = e;
+
+ screwy_in_slave[coll] = stats
+ if mhash == "no _id _index":
+ mOplog = mTestDB.connection.local["oplog.$main"];
+ oplog_entries = list(mOplog.find({"$or": [{"ns":mTestDB[coll].full_name}, \
+ {"op":"c"}]}).sort("$natural", 1))
+ print "oplog for %s" % mTestDB[coll].full_name
+ for doc in oplog_entries:
+ pprint.pprint(doc, width=200)
+
for db in slave.dict.keys():
- if db not in master.dict:
+ if db not in master.dict and db not in lost_in_master:
lost_in_master.append(db)
@@ -350,18 +403,28 @@ def ternary( b , l="true", r="false" ):
return l
return r
-
# Blech.
def skipTest(path):
basename = os.path.basename(path)
parentPath = os.path.dirname(path)
parentDir = os.path.basename(parentPath)
if small_oplog: # For tests running in parallel
- if basename in ["cursor8.js", "indexh.js", "dropdb.js", "connections_opened.js", "opcounters.js"]:
+ if basename in ["cursor8.js", "indexh.js", "dropdb.js", "dropdb_race.js",
+ "connections_opened.js", "opcounters.js", "dbadmin.js"]:
return True
- if auth or keyFile: # For tests running with auth
+ if use_ssl:
+ # Skip tests using mongobridge since it does not support SSL
+ # TODO: Remove when SERVER-10910 has been resolved.
+ if basename in ["gridfs.js", "initial_sync3.js", "majority.js", "no_chaining.js",
+ "rollback4.js", "slavedelay3.js", "sync2.js", "tags.js"]:
+ return True
+ # TODO: For now skip tests using MongodRunner, remove when SERVER-10909 has been resolved
+ if basename in ["fastsync.js", "index_retry.js", "ttl_repl_maintenance.js",
+ "unix_socket1.js"]:
+ return True;
+ if auth or keyFile or use_x509: # For tests running with auth
# Skip any tests that run with auth explicitly
- if parentDir == "auth" or "auth" in basename:
+ if parentDir.lower() == "auth" or "auth" in basename.lower():
return True
if parentPath == mongo_repo: # Skip client tests
return True
@@ -372,19 +435,18 @@ def skipTest(path):
if parentDir == "disk": # SERVER-7356
return True
- authTestsToSkip = [("sharding", "gle_with_conf_servers.js"), # SERVER-6972
- ("sharding", "read_pref.js"), # SERVER-6972
- ("sharding", "read_pref_cmd.js"), # SERVER-6972
- ("sharding", "read_pref_rs_client.js"), # SERVER-6972
- ("sharding", "sync_conn_cmd.js"), #SERVER-6327
+ authTestsToSkip = [("jstests", "drop2.js"), # SERVER-8589,
+ ("jstests", "killop.js"), # SERVER-10128
("sharding", "sync3.js"), # SERVER-6388 for this and those below
("sharding", "sync6.js"),
("sharding", "parallel.js"),
+ ("sharding", "copydb_from_mongos.js"), # SERVER-13080
("jstests", "bench_test1.js"),
("jstests", "bench_test2.js"),
("jstests", "bench_test3.js"),
- ("jstests", "drop2.js"), # SERVER-8589
- ("jstests", "killop.js") # SERVER-10128
+ ("core", "bench_test1.js"),
+ ("core", "bench_test2.js"),
+ ("core", "bench_test3.js"),
]
if os.path.join(parentDir,basename) in [ os.path.join(*test) for test in authTestsToSkip ]:
@@ -392,16 +454,29 @@ def skipTest(path):
return False
-def runTest(test):
+forceCommandsForDirs = ["aggregation", "auth", "core", "parallel", "replsets"]
+# look for jstests and one of the above suites separated by either posix or windows slashes
+forceCommandsRE = re.compile(r"jstests[/\\](%s)" % ('|'.join(forceCommandsForDirs)))
+def setShellWriteModeForTest(path, argv):
+ swm = shell_write_mode
+ if swm == "legacy": # change when the default changes to "commands"
+ if use_write_commands or forceCommandsRE.search(path):
+ swm = "commands"
+ argv += ["--writeMode", swm]
+
+def runTest(test, result):
+ # result is a map containing test result details, like result["url"]
+
# test is a tuple of ( filename , usedb<bool> )
# filename should be a js file to run
# usedb is true if the test expects a mongod to be running
(path, usedb) = test
(ignore, ext) = os.path.splitext(path)
- if skipTest(path):
- print "skipping " + path
- return
+ test_mongod = mongod()
+ mongod_is_up = test_mongod.is_mongod_up(mongod_port)
+ result["mongod_running_at_start"] = mongod_is_up;
+
if file_of_commands_mode:
# smoke.py was invoked like "--mode files --from-file foo",
# so don't try to interpret the test path too much
@@ -415,19 +490,27 @@ def runTest(test):
path = argv[1]
elif ext == ".js":
argv = [shell_executable, "--port", mongod_port, '--authenticationMechanism', authMechanism]
+
+ setShellWriteModeForTest(path, argv)
+
if not usedb:
argv += ["--nodb"]
if small_oplog or small_oplog_rs:
argv += ["--eval", 'testingReplication = true;']
- if use_ssl:
+ if use_ssl:
argv += ["--ssl",
"--sslPEMKeyFile", "jstests/libs/client.pem",
- "--sslCAFile", "jstests/libs/ca.pem"]
+ "--sslCAFile", "jstests/libs/ca.pem",
+ "--sslAllowInvalidCertificates"]
argv += [path]
elif ext in ["", ".exe"]:
# Blech.
if os.path.basename(path) in ["test", "test.exe", "perftest", "perftest.exe"]:
argv = [path]
+ # default data directory for test and perftest is /tmp/unittest
+ if smoke_db_prefix:
+ dir_name = smoke_db_prefix + '/unittests'
+ argv.extend(["--dbpath", dir_name] )
# more blech
elif os.path.basename(path) in ['mongos', 'mongos.exe']:
argv = [path, "--test"]
@@ -437,17 +520,7 @@ def runTest(test):
else:
raise Bug("fell off in extension case: %s" % path)
- if keyFile:
- f = open(keyFile, 'r')
- keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace
- f.close()
- os.chmod(keyFile, stat.S_IRUSR | stat.S_IWUSR)
- else:
- keyFileData = None
-
mongo_test_filename = os.path.basename(path)
- if 'sharedclient' in path:
- mongo_test_filename += "-sharedclient"
# sys.stdout.write() is more atomic than print, so using it prevents
# lines being interrupted by, e.g., child processes
@@ -462,11 +535,20 @@ def runTest(test):
'TestData.testPath = "' + path + '";' + \
'TestData.testFile = "' + os.path.basename( path ) + '";' + \
'TestData.testName = "' + re.sub( ".js$", "", os.path.basename( path ) ) + '";' + \
+ 'TestData.setParameters = "' + ternary( set_parameters, set_parameters, "" ) + '";' + \
+ 'TestData.setParametersMongos = "' + ternary( set_parameters_mongos, set_parameters_mongos, "" ) + '";' + \
'TestData.noJournal = ' + ternary( no_journal ) + ";" + \
'TestData.noJournalPrealloc = ' + ternary( no_preallocj ) + ";" + \
'TestData.auth = ' + ternary( auth ) + ";" + \
'TestData.keyFile = ' + ternary( keyFile , '"' + str(keyFile) + '"' , 'null' ) + ";" + \
- 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";"
+ 'TestData.keyFileData = ' + ternary( keyFile , '"' + str(keyFileData) + '"' , 'null' ) + ";" + \
+ 'TestData.authMechanism = ' + ternary( authMechanism,
+ '"' + str(authMechanism) + '"', 'null') + ";" + \
+ 'TestData.useSSL = ' + ternary( use_ssl ) + ";" + \
+ 'TestData.useX509 = ' + ternary( use_x509 ) + ";"
+ # this updates the default data directory for mongod processes started through shell (src/mongo/shell/servers.js)
+ evalString += 'MongoRunner.dataDir = "' + os.path.abspath(smoke_db_prefix + '/data/db') + '";'
+ evalString += 'MongoRunner.dataPath = MongoRunner.dataDir + "/";'
if os.sys.platform == "win32":
# double quotes in the evalString on windows; this
# prevents the backslashes from being removed when
@@ -477,18 +559,41 @@ def runTest(test):
evalString += 'jsTest.authenticate(db.getMongo());'
argv = argv + [ '--eval', evalString]
-
- if argv[0].endswith( 'test' ) and no_preallocj :
- argv = argv + [ '--nopreallocj' ]
-
-
+
+ if argv[0].endswith( 'test' ) or argv[0].endswith( 'test.exe' ):
+ if no_preallocj :
+ argv = argv + [ '--nopreallocj' ]
+ if temp_path:
+ argv = argv + [ '--tempPath', temp_path ]
+
+
sys.stdout.write(" Command : %s\n" % ' '.join(argv))
sys.stdout.write(" Date : %s\n" % datetime.now().ctime())
sys.stdout.flush()
os.environ['MONGO_TEST_FILENAME'] = mongo_test_filename
t1 = time.time()
- r = call(buildlogger(argv), cwd=test_path)
+
+ proc = Popen(buildlogger(argv), cwd=test_path, stdout=PIPE, stderr=STDOUT, bufsize=0)
+ first_line = proc.stdout.readline() # Get suppressed output URL
+ m = re.search(r"\s*\(output suppressed; see (?P<url>.*)\)" + os.linesep, first_line)
+ if m:
+ result["url"] = m.group("url")
+ sys.stdout.write(first_line)
+ sys.stdout.flush()
+ while True:
+ # print until subprocess's stdout closed.
+ # Not using "for line in file" since that has unwanted buffering.
+ line = proc.stdout.readline()
+ if not line:
+ break;
+
+ sys.stdout.write(line)
+ sys.stdout.flush()
+
+ proc.wait() # wait if stdout is closed before subprocess exits.
+ r = proc.returncode
+
t2 = time.time()
del os.environ['MONGO_TEST_FILENAME']
@@ -507,15 +612,19 @@ def runTest(test):
sys.stdout.write(" %10.4f %s\n" % ((timediff) * scale, suffix))
sys.stdout.flush()
+ result["exit_code"] = r
+
+ is_mongod_still_up = test_mongod.is_mongod_up(mongod_port)
+ if not is_mongod_still_up:
+ print "mongod is not running after test"
+ result["mongod_running_at_end"] = is_mongod_still_up;
+ if start_mongod:
+ raise TestServerFailure(path)
+
+ result["mongod_running_at_end"] = is_mongod_still_up;
+
if r != 0:
raise TestExitFailure(path, r)
-
- if start_mongod:
- try:
- c = Connection(host="127.0.0.1", port=int(mongod_port), ssl=use_ssl)
- except Exception,e:
- print "Exception from pymongo: ", e
- raise TestServerFailure(path)
print ""
@@ -524,7 +633,7 @@ def run_tests(tests):
# need this. (So long as there are no conflicts with port,
# dbpath, etc., and so long as we shut ours down properly,
# starting this mongod shouldn't break anything, though.)
-
+
# The reason we want to use "with" is so that we get __exit__ semantics
# but "with" is only supported on Python 2.5+
@@ -532,25 +641,32 @@ def run_tests(tests):
master = mongod(small_oplog_rs=small_oplog_rs,
small_oplog=small_oplog,
no_journal=no_journal,
+ set_parameters=set_parameters,
no_preallocj=no_preallocj,
auth=auth,
authMechanism=authMechanism,
- use_ssl=use_ssl).__enter__()
+ keyFile=keyFile,
+ use_ssl=use_ssl,
+ use_x509=use_x509).__enter__()
else:
master = Nothing()
try:
if small_oplog:
- slave = mongod(slave=True).__enter__()
+ slave = mongod(slave=True,
+ set_parameters=set_parameters).__enter__()
elif small_oplog_rs:
slave = mongod(slave=True,
small_oplog_rs=small_oplog_rs,
small_oplog=small_oplog,
no_journal=no_journal,
+ set_parameters=set_parameters,
no_preallocj=no_preallocj,
auth=auth,
authMechanism=authMechanism,
- use_ssl=use_ssl).__enter__()
- primary = Connection(port=master.port, slave_okay=True);
+ keyFile=keyFile,
+ use_ssl=use_ssl,
+ use_x509=use_x509).__enter__()
+ primary = MongoClient(port=master.port);
primary.admin.command({'replSetInitiate' : {'_id' : 'foo', 'members' : [
{'_id': 0, 'host':'localhost:%s' % master.port},
@@ -568,38 +684,63 @@ def run_tests(tests):
if small_oplog or small_oplog_rs:
master.wait_for_repl()
- tests_run = 0
for tests_run, test in enumerate(tests):
- test_result = { "test": test[0], "start": time.time() }
+ tests_run += 1 # enumerate from 1, python 2.5 compatible
+ test_result = { "start": time.time() }
+
+ (test_path, use_db) = test
+
+ if test_path.startswith(mongo_repo + os.path.sep):
+ test_result["test_file"] = test_path[len(mongo_repo)+1:]
+ else:
+ # user could specify a file not in repo. leave it alone.
+ test_result["test_file"] = test_path
+
try:
- fails.append(test)
- runTest(test)
- fails.pop()
- winners.append(test)
+ if skipTest(test_path):
+ test_result["status"] = "skip"
- test_result["passed"] = True
- test_result["end"] = time.time()
- all_test_results.append( test_result )
+ print "skipping " + test_path
+ else:
+ fails.append(test)
+ runTest(test, test_result)
+ fails.pop()
+ winners.append(test)
+
+ test_result["status"] = "pass"
+ test_result["end"] = time.time()
+ test_result["elapsed"] = test_result["end"] - test_result["start"]
+ test_report["results"].append( test_result )
if small_oplog or small_oplog_rs:
master.wait_for_repl()
- elif test[1]: # reach inside test and see if "usedb" is true
- if (tests_run+1) % 20 == 0:
- # restart mongo every 20 times, for our 32-bit machines
+ # check the db_hashes
+ if isinstance(slave, mongod):
+ check_db_hashes(master, slave)
+ check_and_report_replication_dbhashes()
+
+ elif use_db: # reach inside test and see if "usedb" is true
+ if clean_every_n_tests and (tests_run % clean_every_n_tests) == 0:
+ # Restart mongod periodically to clean accumulated test data
+ # clean_dbroot() is invoked by mongod.start()
master.__exit__(None, None, None)
master = mongod(small_oplog_rs=small_oplog_rs,
small_oplog=small_oplog,
no_journal=no_journal,
+ set_parameters=set_parameters,
no_preallocj=no_preallocj,
auth=auth,
authMechanism=authMechanism,
- use_ssl=use_ssl).__enter__()
+ keyFile=keyFile,
+ use_ssl=use_ssl,
+ use_x509=use_x509).__enter__()
except TestFailure, f:
- test_result["passed"] = False
test_result["end"] = time.time()
+ test_result["elapsed"] = test_result["end"] - test_result["start"]
test_result["error"] = str(f)
- all_test_results.append( test_result )
+ test_result["status"] = "fail"
+ test_report["results"].append( test_result )
try:
print f
# Record the failing test and re-raise.
@@ -619,42 +760,81 @@ def run_tests(tests):
return 0
-def report():
- print "%d tests succeeded" % len(winners)
- num_missed = len(tests) - (len(winners) + len(losers.keys()))
- if num_missed:
- print "%d tests didn't get run" % num_missed
- if losers:
- print "The following tests failed (with exit code):"
- for loser in losers:
- print "%s\t%d" % (loser, losers[loser])
-
+def check_and_report_replication_dbhashes():
def missing(lst, src, dst):
if lst:
print """The following collections were present in the %s but not the %s
at the end of testing:""" % (src, dst)
for db in lst:
print db
+
missing(lost_in_slave, "master", "slave")
missing(lost_in_master, "slave", "master")
if screwy_in_slave:
print """The following collections has different hashes in master and slave
at the end of testing:"""
- for db in screwy_in_slave.keys():
- print "%s\t %s" % (db, screwy_in_slave[db])
+ for coll in screwy_in_slave.keys():
+ stats = screwy_in_slave[coll]
+ # Counts are "approx" because they are collected after the dbhash runs and may not
+ # reflect the states of the collections that were hashed. If the hashes differ, one
+ # possibility is that a test exited with writes still in-flight.
+ print "collection: %s\t (master/slave) hashes: %s/%s counts (approx): %i/%i" % (coll, stats['hashes']['master'], stats['hashes']['slave'], stats['counts']['master'], stats['counts']['slave'])
+ if "docs" in stats:
+ if (("master" in stats["docs"] and len(stats["docs"]["master"]) != 0) or
+ ("slave" in stats["docs"] and len(stats["docs"]["slave"]) != 0)):
+ print "All docs matched!"
+ else:
+ print "Different Docs"
+ print "Master docs:"
+ pprint.pprint(stats["docs"]["master"], indent=2)
+ print "Slave docs:"
+ pprint.pprint(stats["docs"]["slave"], indent=2)
+ if "error-docs" in stats:
+ print "Error getting docs to diff:"
+ pprint.pprint(stats["error-docs"])
+ return True
+
if (small_oplog or small_oplog_rs) and not (lost_in_master or lost_in_slave or screwy_in_slave):
print "replication ok for %d collections" % (len(replicated_collections))
+
+ return False
+
+
+def report():
+ print "%d tests succeeded" % len(winners)
+ num_missed = len(tests) - (len(winners) + len(losers.keys()))
+ if num_missed:
+ print "%d tests didn't get run" % num_missed
+ if losers:
+ print "The following tests failed (with exit code):"
+ for loser in losers:
+ print "%s\t%d" % (loser, losers[loser])
+
+ test_result = { "start": time.time() }
+ if check_and_report_replication_dbhashes():
+ test_result["end"] = time.time()
+ test_result["elapsed"] = test_result["end"] - test_result["start"]
+ test_result["test_file"] = "/#dbhash#"
+ test_result["error"] = "dbhash mismatch"
+ test_result["status"] = "fail"
+ test_report["results"].append( test_result )
+
+ if report_file:
+ f = open( report_file, "wb" )
+ f.write( json.dumps( test_report ) )
+ f.close()
+
if losers or lost_in_slave or lost_in_master or screwy_in_slave:
raise Exception("Test failures")
# Keys are the suite names (passed on the command line to smoke.py)
# Values are pairs: (filenames, <start mongod before running tests>)
-suiteGlobalConfig = {"js": ("[!_]*.js", True),
+suiteGlobalConfig = {"js": ("core/*.js", True),
"quota": ("quota/*.js", True),
"jsPerf": ("perf/*.js", True),
"disk": ("disk/*.js", True),
- "jsSlowNightly": ("slowNightly/*.js", True),
- "jsSlowWeekly": ("slowWeekly/*.js", False),
+ "noPassthroughWithMongod": ("noPassthroughWithMongod/*.js", True),
+ "noPassthrough": ("noPassthrough/*.js", False),
"parallel": ("parallel/*.js", True),
"clone": ("clone/*.js", False),
"repl": ("repl/*.js", False),
@@ -666,15 +846,88 @@ suiteGlobalConfig = {"js": ("[!_]*.js", True),
"aggregation": ("aggregation/*.js", True),
"multiVersion": ("multiVersion/*.js", True),
"failPoint": ("fail_point/*.js", False),
- "ssl": ("ssl/*.js", True)
+ "ssl": ("ssl/*.js", True),
+ "sslSpecial": ("sslSpecial/*.js", True),
+ "jsCore": ("core/*.js", True),
+ "gle": ("gle/*.js", True),
+ "slow1": ("slow1/*.js", True),
+ "slow2": ("slow2/*.js", True),
}
+def get_module_suites():
+ """Attempts to discover and return information about module test suites
+
+ Returns a dictionary of module suites in the format:
+
+ {
+ "<suite_name>" : "<full_path_to_suite_directory/[!_]*.js>",
+ ...
+ }
+
+ This means the values of this dictionary can be used as "glob"s to match all jstests in the
+ suite directory that don't start with an underscore
+
+ The module tests should be put in 'src/mongo/db/modules/<module_name>/<suite_name>/*.js'
+
+ NOTE: This assumes that if we have more than one module the suite names don't conflict
+ """
+ modules_directory = 'src/mongo/db/modules'
+ test_suites = {}
+
+ # Return no suites if we have no modules
+ if not os.path.exists(modules_directory) or not os.path.isdir(modules_directory):
+ return {}
+
+ module_directories = os.listdir(modules_directory)
+ for module_directory in module_directories:
+
+ test_directory = os.path.join(modules_directory, module_directory, "jstests")
+
+ # Skip this module if it has no "jstests" directory
+ if not os.path.exists(test_directory) or not os.path.isdir(test_directory):
+ continue
+
+ # Get all suites for this module
+ for test_suite in os.listdir(test_directory):
+ test_suites[test_suite] = os.path.join(test_directory, test_suite, "[!_]*.js")
+
+ return test_suites
+
def expand_suites(suites,expandUseDB=True):
+ """Takes a list of suites and expands to a list of tests according to a set of rules.
+
+ Keyword arguments:
+ suites -- list of suites specified by the user
+ expandUseDB -- expand globs (such as [!_]*.js) for tests that are run against a database
+ (default True)
+
+ This function handles expansion of globs (such as [!_]*.js), aliases (such as "client" and
+ "all"), detection of suites in the "modules" directory, and enumerating the test files in a
+ given suite. It returns a list of tests of the form (path_to_test, usedb), where the second
+ part of the tuple specifies whether the test is run against the database (see --nodb in the
+ mongo shell)
+
+ """
globstr = None
tests = []
+ module_suites = get_module_suites()
for suite in suites:
if suite == 'all':
- return expand_suites(['test', 'perf', 'client', 'js', 'jsPerf', 'jsSlowNightly', 'jsSlowWeekly', 'clone', 'parallel', 'repl', 'auth', 'sharding', 'tool'],expandUseDB=expandUseDB)
+ return expand_suites(['test',
+ 'perf',
+ 'jsCore',
+ 'jsPerf',
+ 'noPassthroughWithMongod',
+ 'noPassthrough',
+ 'clone',
+ 'parallel',
+ 'repl',
+ 'auth',
+ 'sharding',
+ 'slow1',
+ 'slow2',
+ 'tool'],
+ expandUseDB=expandUseDB)
if suite == 'test':
if os.sys.platform == "win32":
program = 'test.exe'
@@ -687,20 +940,6 @@ def expand_suites(suites,expandUseDB=True):
else:
program = 'perftest'
(globstr, usedb) = (program, False)
- elif suite == 'client':
- paths = ["firstExample", "secondExample", "whereExample", "authTest", "clientTest", "httpClientTest"]
- if os.sys.platform == "win32":
- paths = [path + '.exe' for path in paths]
-
- if not test_path:
- # If we are testing 'in-tree', then add any files of the same name from the
- # sharedclient directory. The out of tree client build doesn't have shared clients.
- scpaths = ["sharedclient/" + path for path in paths]
- scfiles = glob.glob("sharedclient/*")
- paths += [scfile for scfile in scfiles if scfile in scpaths]
-
- # hack
- tests += [(test_path and path or os.path.join(mongo_repo, path), False) for path in paths]
elif suite == 'mongosTest':
if os.sys.platform == "win32":
program = 'mongos.exe'
@@ -714,6 +953,13 @@ def expand_suites(suites,expandUseDB=True):
usedb = suiteGlobalConfig[name][1]
break
tests += [ ( os.path.join( mongo_repo , suite ) , usedb ) ]
+ elif suite in module_suites:
+ # Currently we connect to a database in all module tests since there's no mechanism yet
+ # to configure it independently
+ usedb = True
+ paths = glob.glob(module_suites[suite])
+ paths.sort()
+ tests += [(path, usedb) for path in paths]
else:
try:
globstr, usedb = suiteGlobalConfig[suite]
@@ -723,7 +969,7 @@ def expand_suites(suites,expandUseDB=True):
if globstr:
if usedb and not expandUseDB:
tests += [ (suite,False) ]
- else:
+ else:
if globstr.endswith('.js'):
loc = 'jstests/'
else:
@@ -742,13 +988,23 @@ def add_exe(e):
return e
def set_globals(options, tests):
- global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, small_oplog_rs
- global no_journal, no_preallocj, auth, authMechanism, keyFile, smoke_db_prefix, test_path, start_mongod
- global use_ssl
+ global mongod_executable, mongod_port, shell_executable, continue_on_failure
+ global small_oplog, small_oplog_rs
+ global no_journal, set_parameters, set_parameters_mongos, no_preallocj
+ global auth, authMechanism, keyFile, keyFileData, smoke_db_prefix, test_path, start_mongod
+ global use_ssl, use_x509
global file_of_commands_mode
+ global report_file, shell_write_mode, use_write_commands
+ global temp_path
+ global clean_every_n_tests
+ global clean_whole_dbroot
+
start_mongod = options.start_mongod
if hasattr(options, 'use_ssl'):
use_ssl = options.use_ssl
+ if hasattr(options, 'use_x509'):
+ use_x509 = options.use_x509
+ use_ssl = use_ssl or use_x509
#Careful, this can be called multiple times
test_path = options.test_path
@@ -768,27 +1024,38 @@ def set_globals(options, tests):
if hasattr(options, "small_oplog_rs"):
small_oplog_rs = options.small_oplog_rs
no_journal = options.no_journal
+ set_parameters = options.set_parameters
+ set_parameters_mongos = options.set_parameters_mongos
no_preallocj = options.no_preallocj
- if options.mode == 'suite' and tests == ['client']:
- # The client suite doesn't work with authentication
- if options.auth:
- print "Not running client suite with auth even though --auth was provided"
- auth = False;
- keyFile = False;
- authMechanism = None
- else:
- auth = options.auth
- authMechanism = options.authMechanism
- keyFile = options.keyFile
+ auth = options.auth
+ authMechanism = options.authMechanism
+ keyFile = options.keyFile
+
+ clean_every_n_tests = options.clean_every_n_tests
+ clean_whole_dbroot = options.with_cleanbb
if auth and not keyFile:
# if only --auth was given to smoke.py, load the
# default keyFile from jstests/libs/authTestsKey
keyFile = os.path.join(mongo_repo, 'jstests', 'libs', 'authTestsKey')
+ if keyFile:
+ f = open(keyFile, 'r')
+ keyFileData = re.sub(r'\s', '', f.read()) # Remove all whitespace
+ f.close()
+ os.chmod(keyFile, stat.S_IRUSR | stat.S_IWUSR)
+ else:
+ keyFileData = None
+
# if smoke.py is running a list of commands read from a
# file (or stdin) rather than running a suite of js tests
file_of_commands_mode = options.File and options.mode == 'files'
+ # generate json report
+ report_file = options.report_file
+ temp_path = options.temp_path
+
+ use_write_commands = options.use_write_commands
+ shell_write_mode = options.shell_write_mode
def file_version():
return md5(open(__file__, 'r').read()).hexdigest()
@@ -870,7 +1137,10 @@ def add_to_failfile(tests, options):
def main():
- global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog, no_journal, no_preallocj, auth, keyFile, smoke_db_prefix, test_path
+ global mongod_executable, mongod_port, shell_executable, continue_on_failure, small_oplog
+ global no_journal, set_parameters, set_parameters_mongos, no_preallocj, auth
+ global keyFile, smoke_db_prefix, test_path, use_write_commands
+
parser = OptionParser(usage="usage: smoke.py [OPTIONS] ARGS*")
parser.add_option('--mode', dest='mode', default='suite',
help='If "files", ARGS are filenames; if "suite", ARGS are sets of tests (%default)')
@@ -908,6 +1178,9 @@ def main():
parser.add_option('--auth', dest='auth', default=False,
action="store_true",
help='Run standalone mongods in tests with authentication enabled')
+ parser.add_option('--use-x509', dest='use_x509', default=False,
+ action="store_true",
+ help='Use x509 auth for internal cluster authentication')
parser.add_option('--authMechanism', dest='authMechanism', default='MONGODB-CR',
help='Use the given authentication mechanism, when --auth is used.')
parser.add_option('--keyFile', dest='keyFile', default=None,
@@ -920,25 +1193,43 @@ def main():
parser.add_option('--reset-old-fails', dest='reset_old_fails', default=False,
action="store_true",
help='Clear the failfile. Do this if all tests pass')
- parser.add_option('--with-cleanbb', dest='with_cleanbb', default=False,
- action="store_true",
- help='Clear database files from previous smoke.py runs')
- parser.add_option('--dont-start-mongod', dest='start_mongod', default=True,
+ parser.add_option('--with-cleanbb', dest='with_cleanbb', action="store_true",
+ default=False,
+ help='Clear database files before first test')
+ parser.add_option('--clean-every', dest='clean_every_n_tests', type='int',
+ default=20,
+ help='Clear database files every N tests [default %default]')
+ parser.add_option('--dont-start-mongod', dest='start_mongod', default=True,
action='store_false',
help='Do not start mongod before commencing test running')
parser.add_option('--use-ssl', dest='use_ssl', default=False,
action='store_true',
help='Run mongo shell and mongod instances with SSL encryption')
-
+ parser.add_option('--set-parameters', dest='set_parameters', default="",
+ help='Adds --setParameter to mongod for each passed in item in the csv list - ex. "param1=1,param2=foo" ')
+ parser.add_option('--set-parameters-mongos', dest='set_parameters_mongos', default="",
+ help='Adds --setParameter to mongos for each passed in item in the csv list - ex. "param1=1,param2=foo" ')
+ parser.add_option('--temp-path', dest='temp_path', default=None,
+ help='If present, passed as --tempPath to unittests and dbtests')
# Buildlogger invocation from command line
parser.add_option('--buildlogger-builder', dest='buildlogger_builder', default=None,
action="store", help='Set the "builder name" for buildlogger')
parser.add_option('--buildlogger-buildnum', dest='buildlogger_buildnum', default=None,
action="store", help='Set the "build number" for buildlogger')
+ parser.add_option('--buildlogger-url', dest='buildlogger_url', default=None,
+ action="store", help='Set the url root for the buildlogger service')
parser.add_option('--buildlogger-credentials', dest='buildlogger_credentials', default=None,
action="store", help='Path to Python file containing buildlogger credentials')
parser.add_option('--buildlogger-phase', dest='buildlogger_phase', default=None,
action="store", help='Set the "phase" for buildlogger (e.g. "core", "auth") for display in the webapp (optional)')
+ parser.add_option('--report-file', dest='report_file', default=None,
+ action='store',
+ help='Path to generate detailed json report containing all test details')
+ parser.add_option('--use-write-commands', dest='use_write_commands', default=False,
+ action='store_true',
+ help='Deprecated(use --shell-write-mode): Sets the shell to use write commands by default')
+ parser.add_option('--shell-write-mode', dest='shell_write_mode', default="legacy",
+ help='Sets the shell to use a specific write mode: commands/compatibility/legacy (default:legacy)')
global tests
(options, tests) = parser.parse_args()
@@ -957,6 +1248,9 @@ def main():
# some but not all of the required options were sete
raise Exception("you must set all of --buildlogger-builder, --buildlogger-buildnum, --buildlogger-credentials")
+ if options.buildlogger_url: #optional; if None, defaults to const in buildlogger.py
+ os.environ['BUILDLOGGER_URL'] = options.buildlogger_url
+
if options.File:
if options.File == '-':
tests = sys.stdin.readlines()
@@ -981,7 +1275,7 @@ def main():
if options.ignore_files != None :
ignore_patt = re.compile( options.ignore_files )
print "Ignoring files with pattern: ", ignore_patt
-
+
def ignore_test( test ):
if ignore_patt.search( test[0] ) != None:
print "Ignoring test ", test[0]
@@ -996,17 +1290,23 @@ def main():
return
if options.with_cleanbb:
- dbroot = os.path.join(options.smoke_db_prefix, 'data', 'db')
- call([utils.find_python(), "buildscripts/cleanbb.py", "--nokill", dbroot])
+ clean_dbroot(nokill=True)
+ test_report["start"] = time.time()
+ test_report["mongod_running_at_start"] = mongod().is_mongod_up(mongod_port)
try:
run_tests(tests)
finally:
add_to_failfile(fails, options)
- f = open( "smoke-last.json", "wb" )
- f.write( json.dumps( { "results" : all_test_results } ) )
- f.close()
+ test_report["end"] = time.time()
+ test_report["elapsed"] = test_report["end"] - test_report["start"]
+ test_report["failures"] = len(losers.keys())
+ test_report["mongod_running_at_end"] = mongod().is_mongod_up(mongod_port)
+ if report_file:
+ f = open( report_file, "wb" )
+ f.write( json.dumps( test_report, indent=4, separators=(',', ': ')) )
+ f.close()
report()
diff --git a/buildscripts/utils.py b/buildscripts/utils.py
index fe35f69eec3..68273ee69c8 100644
--- a/buildscripts/utils.py
+++ b/buildscripts/utils.py
@@ -36,7 +36,7 @@ def getAllSourceFiles( arr=None , prefix="." ):
def getGitBranch():
- if not os.path.exists( ".git" ):
+ if not os.path.exists( ".git" ) or not os.path.isdir(".git"):
return None
version = open( ".git/HEAD" ,'r' ).read().strip()
@@ -63,7 +63,7 @@ def getGitBranchString( prefix="" , postfix="" ):
return prefix + b + postfix
def getGitVersion():
- if not os.path.exists( ".git" ):
+ if not os.path.exists( ".git" ) or not os.path.isdir(".git"):
return "nogitversion"
version = open( ".git/HEAD" ,'r' ).read().strip()
@@ -87,7 +87,7 @@ def execsys( args ):
def getprocesslist():
raw = ""
try:
- raw = execsys( "/bin/ps -ax" )[0]
+ raw = execsys( "/bin/ps axww" )[0]
except Exception,e:
print( "can't get processlist: " + str( e ) )
diff --git a/buildscripts/vcxproj.header b/buildscripts/vcxproj.header
new file mode 100644
index 00000000000..4a90ec8ed25
--- /dev/null
+++ b/buildscripts/vcxproj.header
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!-- header for use with make_vcxproj.py
+
+ Note that once you generate the vcxproj file, if you change it in visual studio, when it
+ writes it back out you will use the comments and semi-neat formatting below.
+-->
+
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Win2008PlusDebug|Win32">
+ <Configuration>Win2008PlusDebug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Win2008PlusDebug|x64">
+ <Configuration>Win2008PlusDebug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Win2008PlusRelease|Win32">
+ <Configuration>Win2008PlusRelease</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Win2008PlusRelease|x64">
+ <Configuration>Win2008PlusRelease</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+
+ <PropertyGroup Label="Globals">
+ <ProjectName>_TARGET_</ProjectName>
+ <!-- <ProjectGuid>{215B2D68-0A70-4D10-8E75-B31010C62A91}</ProjectGuid> -->
+ <Keyword>Win32Proj</Keyword>
+ <RootNamespace>_TARGET_</RootNamespace>
+ </PropertyGroup>
+
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+
+ <PropertyGroup Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ <CharacterSet>Unicode</CharacterSet>
+ </PropertyGroup>
+
+<!-- given we do final official build with scons, it is reasonable to consider leaving this off for better
+ compile speed during development:
+-->
+<!--
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'" Label="Configuration">
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'" Label="Configuration">
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ </PropertyGroup>
+-->
+
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+
+ <PropertyGroup>
+ <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
+ <OutDir>$(ProjectDir)$(Platform)\$(Configuration)\</OutDir>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|Win32'">true</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|x64'">true</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'">false</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</LinkIncremental>
+ <LinkIncremental Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'">false</LinkIncremental>
+ <CodeAnalysisRuleSet >AllRules.ruleset</CodeAnalysisRuleSet>
+ </PropertyGroup>
+
+ <!-- GLOBAL SETTINGS ALL BUILD STYLES -->
+ <ItemDefinitionGroup>
+ <ClCompile>
+ <ObjectFileName>%(RelativeDir)/</ObjectFileName>
+ <AdditionalIncludeDirectories>src;src\mongo;src\third_party\v8\include;src\third_party\pcre-8.30;src\third_party\boost;src\third_party\snappy;src\third_party\s2;src\third_party\yaml-cpp-0.5.1\include</AdditionalIncludeDirectories>
+ <PreprocessorDefinitions>NTDDI_VERSION=0x06010000;_WIN32_WINNT=0x0601;WIN32;XP_WIN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <!-- above is temp - NTDDI etc. should be configurable, not finished yet, this gets us compiling some for now.
+ <PreprocessorDefinitions>WIN32;XP_WIN;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ -->
+ <DisableSpecificWarnings>4355;4800;4267;4244;4351</DisableSpecificWarnings>
+ <MultiProcessorCompilation>true</MultiProcessorCompilation>
+ <MinimalRebuild>No</MinimalRebuild>
+ <WarningLevel>Level3</WarningLevel>
+ <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ </ClCompile>
+ <Link>
+ <SubSystem>Console</SubSystem>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <IgnoreAllDefaultLibraries>false</IgnoreAllDefaultLibraries>
+ <AdditionalDependencies>winmm.lib;ws2_32.lib;psapi.lib;dbghelp.lib;%(AdditionalDependencies)</AdditionalDependencies>
+ </Link>
+ </ItemDefinitionGroup>
+
+ <!-- DEBUG -->
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug' Or '(Configuration)'=='Win2008PlusDebug'">
+ <ClCompile>
+ <Optimization>Disabled</Optimization>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary>
+ <PreprocessorDefinitions>_DEBUG;DEBUG;OBJECT_PRINT;ENABLE_DISASSEMBLER;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ <!--<DebugInformationFormat>EditAndContinue</DebugInformationFormat>-->
+ </ClCompile>
+ </ItemDefinitionGroup>
+
+ <!-- RELEASE -->
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Release' Or '(Configuration)'=='Win2008PlusRelease'">
+ <ClCompile>
+ <Optimization>MaxSpeed</Optimization>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
+ </ClCompile>
+ </ItemDefinitionGroup>
+
+ <!-- X64 -->
+ <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+ <ClCompile>
+ <PreprocessorDefinitions>V8_TARGET_ARCH_X64;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ </ItemDefinitionGroup>
+
+ <!-- 32 bit -->
+ <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+ <ClCompile>
+ <PreprocessorDefinitions>V8_TARGET_ARCH_IA32;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ <Link>
+ <TargetMachine>MachineX86</TargetMachine>
+ <LargeAddressAware>true</LargeAddressAware>
+ </Link>
+ </ItemDefinitionGroup>
+
+<!-- temp commented out, not working yet
+-->
+ <!-- SRW / Windows2008+ -->
+<!--
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Win2008PlusRelease' Or '(Configuration)'=='Win2008PlusDebug'">
+ <ClCompile>
+ <PreprocessorDefinitions>NTDDI_VERSION=0x06010000;_WIN32_WINNT=0x0601;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ </ItemDefinitionGroup>
+-->
+ <!-- legacy -->
+<!--
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Release' Or '(Configuration)'=='Debug'">
+ <ClCompile>
+ <PreprocessorDefinitions>NTDDI_VERSION=0x05020200;_WIN32_WINNT=0x0502;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+ </ClCompile>
+ </ItemDefinitionGroup>
+-->
+
+ <!-- SPECIFICS -->
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+ <Link>
+ <!-- this is likely unnecessary and should be removed, but was in the old vcxproj: -->
+ <IgnoreSpecificDefaultLibraries>msvcrtd;%(IgnoreSpecificDefaultLibraries)</IgnoreSpecificDefaultLibraries>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|Win32'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusDebug|x64'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|Win32'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Win2008PlusRelease|x64'">
+ </ItemDefinitionGroup>
+
+<!--
+ <ItemDefinitionGroup>
+ <PreBuildEvent>
+ <Command>cscript //Nologo "src\third_party\run_if_newer.js" /path:"$(ProjectDir)..\base" /input:"generate_error_codes.py,error_codes.err" /output:"error_codes.h,error_codes.cpp" /command:"python generate_error_codes.py error_codes.err error_codes.h error_codes.cpp"
+cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\auth" /input:"generate_action_types.py,action_types.txt" /output:"action_type.h,action_type.cpp" /command:"python generate_action_types.py action_types.txt action_type.h action_type.cpp"
+cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\v8" /input:"tools\js2c.py,src\proxy.js,src\collection.js,src\macros.py" /output:"src\experimental-libraries.cc" /command:"python tools\js2c.py src/experimental-libraries.cc EXPERIMENTAL off src/proxy.js src/collection.js src/macros.py"
+cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\..\third_party\v8" /input:"tools\js2c.py,src/runtime.js,src/v8natives.js,src/array.js,src/string.js,src/uri.js,src/math.js,src/messages.js,src/apinatives.js,src/date.js,src/regexp.js,src/json.js,src/liveedit-debugger.js,src/mirror-debugger.js,src/debug-debugger.js,src\macros.py" /output:"src\libraries.cc" /command:"python tools\js2c.py src/libraries.cc CORE off src/runtime.js src/v8natives.js src/array.js src/string.js src/uri.js src/math.js src/messages.js src/apinatives.js src/date.js src/regexp.js src/json.js src/liveedit-debugger.js src/mirror-debugger.js src/debug-debugger.js src\macros.py"
+cscript //Nologo "$(ProjectDir)..\..\third_party\run_if_newer.js" /path:"$(ProjectDir)..\db\fts" /input:"generate_stop_words.py,stop_words_danish.txt,stop_words_dutch.txt,stop_words_english.txt,stop_words_finnish.txt,stop_words_french.txt,stop_words_german.txt,stop_words_hungarian.txt,stop_words_italian.txt,stop_words_norwegian.txt,stop_words_portuguese.txt,stop_words_romanian.txt,stop_words_russian.txt,stop_words_spanish.txt,stop_words_swedish.txt,stop_words_turkish.txt" /output:"stop_words_list.h,stop_words_list.cpp" /command:"python generate_stop_words.py stop_words_danish.txt stop_words_dutch.txt stop_words_english.txt stop_words_finnish.txt stop_words_french.txt stop_words_german.txt stop_words_hungarian.txt stop_words_italian.txt stop_words_norwegian.txt stop_words_portuguese.txt stop_words_romanian.txt stop_words_russian.txt stop_words_spanish.txt stop_words_swedish.txt stop_words_turkish.txt stop_words_list.h stop_words_list.cpp"
+cscript //Nologo "$(ProjectDir)..\shell\createCPPfromJavaScriptFiles.js" "$(ProjectDir).."
+ </Command>
+ <Message>Run pre-build commands</Message>
+ </PreBuildEvent>
+ </ItemDefinitionGroup>
+-->
+
+