diff options
70 files changed, 3529 insertions, 485 deletions
diff --git a/buildscripts/cleanbb.py b/buildscripts/cleanbb.py index dd52020d7e4..589370e33ae 100644 --- a/buildscripts/cleanbb.py +++ b/buildscripts/cleanbb.py @@ -3,6 +3,7 @@ import sys import os, os.path import utils import time +import exceptions from optparse import OptionParser # set cwd to the root mongo dir, one level up from this @@ -14,7 +15,6 @@ if os.path.basename(cwd) == 'buildscripts': print( "cwd [" + cwd + "]" ) def shouldKill( c ): - if "smoke.py" in c: return False @@ -36,7 +36,6 @@ def shouldKill( c ): return False def killprocs( signal="" ): - killed = 0 if sys.platform == 'win32': @@ -55,7 +54,7 @@ def killprocs( signal="" ): x = x.lstrip() if not shouldKill( x ): continue - + pid = x.split( " " )[0] print( "killing: " + x ) utils.execsys( "/bin/kill " + signal + " " + pid ) @@ -65,7 +64,6 @@ def killprocs( signal="" ): def cleanup( root , nokill ): - if nokill: print "nokill requested, not killing anybody" else: @@ -76,11 +74,18 @@ def cleanup( root , nokill ): # delete all regular files, directories can stay # NOTE: if we delete directories later, we can't delete diskfulltest for ( dirpath , dirnames , filenames ) in os.walk( root , topdown=False ): - for x in filenames: + for x in filenames: foo = dirpath + "/" + x print( "removing: " + foo ) - os.remove( foo ) - + try: + os.remove(foo) + except exceptions.OSError, e: + # SERVER-10462 compensate for Windows file locking race + # We want to catch WindowsError but can't use that name on other platforms + print(repr(e)) + print("os.remove(%s) failed, retrying once." % foo) + time.sleep(1) + os.remove(foo) if __name__ == "__main__": parser = OptionParser(usage="read the script") @@ -90,5 +95,5 @@ if __name__ == "__main__": root = "/data/db/" if len(args) > 0: root = args[0] - + cleanup( root , options.nokill ) diff --git a/buildscripts/packager-enterprise.py b/buildscripts/packager-enterprise.py new file mode 100755 index 00000000000..4ccb262a0d8 --- /dev/null +++ b/buildscripts/packager-enterprise.py @@ -0,0 +1,997 @@ +#!/usr/bin/python + +# This program makes Debian and RPM repositories for MongoDB, by +# downloading our tarballs of statically linked executables and +# insinuating them into Linux packages. It must be run on a +# Debianoid, since Debian provides tools to make RPMs, but RPM-based +# systems don't provide debian packaging crud. + +# Notes: +# +# * Almost anything that you want to be able to influence about how a +# package construction must be embedded in some file that the +# packaging tool uses for input (e.g., debian/rules, debian/control, +# debian/changelog; or the RPM specfile), and the precise details are +# arbitrary and silly. So this program generates all the relevant +# inputs to the packaging tools. +# +# * Once a .deb or .rpm package is made, there's a separate layer of +# tools that makes a "repository" for use by the apt/yum layers of +# package tools. The layouts of these repositories are arbitrary and +# silly, too. +# +# * Before you run the program on a new host, these are the +# prerequisites: +# +# apt-get install dpkg-dev rpm debhelper fakeroot ia32-libs createrepo git-core libsnmp15 +# echo "Now put the dist gnupg signing keys in ~root/.gnupg" + +import errno +import getopt +from packager import httpget +import os +import re +import stat +import subprocess +import sys +import tempfile +import time +import urlparse + +# For the moment, this program runs on the host that also serves our +# repositories to the world, so the last thing the program does is +# move the repositories into place. Make this be the path where the +# web server will look for repositories. +REPOPATH="/var/www/repo" + +# The 10gen names for the architectures we support. +ARCHES=["x86_64"] + +# Made up names for the flavors of distribution we package for. +DISTROS=["debian-sysvinit", "ubuntu-upstart", "redhat"] + +# When we're preparing a directory containing packaging tool inputs +# and our binaries, use this relative subdirectory for placing the +# binaries. +BINARYDIR="BINARIES" + +sys.stderr.write("BINARYDIR: %s, REPOPATH: %s\n" % (BINARYDIR, REPOPATH)) + +class Spec(object): + def __init__(self, specstr): + tup = specstr.split(":") + self.ver = tup[0] + # Hack: the second item in the tuple is treated as a suffix if + # it lacks an equals sign; otherwise it's the start of named + # parameters. + self.suf = None + if len(tup) > 1 and tup[1].find("=") == -1: + self.suf = tup[1] + # Catch-all for any other parameters to the packaging. + i = 2 if self.suf else 1 + self.params = dict([s.split("=") for s in tup[i:]]) + for key in self.params.keys(): + assert(key in ["suffix", "revision"]) + + def version(self): + return self.ver + + def version_better_than(self, version_string): + # FIXME: this is wrong, but I'm in a hurry. + # e.g., "1.8.2" < "1.8.10", "1.8.2" < "1.8.2-rc1" + return self.ver > version_string + + def suffix(self): + # suffix is what we tack on after pkgbase. + if self.suf: + return self.suf + elif "suffix" in self.params: + return self.params["suffix"] + else: + return "-10gen-enterprise" if int(self.ver.split(".")[1])%2==0 else "-10gen-unstable" + + + def pversion(self, distro): + # Note: Debian packages have funny rules about dashes in + # version numbers, and RPM simply forbids dashes. pversion + # will be the package's version number (but we need to know + # our upstream version too). + if re.search("^(debian|ubuntu)", distro.name()): + return re.sub("-", "~", self.ver) + elif re.search("(redhat|fedora|centos)", distro.name()): + return re.sub("\\d+-", "", self.ver) + else: + raise Exception("BUG: unsupported platform?") + + def param(self, param): + if param in self.params: + return self.params[param] + return None + +class Distro(object): + def __init__(self, string): + self.n=string + + def name(self): + return self.n + + def pkgbase(self): + # pkgbase is the first part of the package's name on + # this distro. + return "mongo" if re.search("(redhat|fedora|centos)", self.n) else "mongodb" + + def archname(self, arch): + if re.search("^(debian|ubuntu)", self.n): + return "i386" if arch.endswith("86") else "amd64" + elif re.search("^(centos|redhat|fedora)", self.n): + return "i686" if arch.endswith("86") else "x86_64" + else: + raise Exception("BUG: unsupported platform?") + + def repodir(self, arch): + """Return the directory where we'll place the package files for + (distro, distro_version) in that distro's preferred repository + layout (as distinct from where that distro's packaging building + tools place the package files).""" + if re.search("^(debian|ubuntu)", self.n): + return "repo/%s/dists/dist/10gen/binary-%s/" % (self.n, self.archname(arch)) + elif re.search("(redhat|fedora|centos)", self.n): + return "repo/%s/os/%s/RPMS/" % (self.n, self.archname(arch)) + else: + raise Exception("BUG: unsupported platform?") + + def make_pkg(self, arch, spec, srcdir): + if re.search("^(debian|ubuntu)", self.n): + return make_deb(self, arch, spec, srcdir) + elif re.search("^(centos|redhat|fedora)", self.n): + return make_rpm(self, arch, spec, srcdir) + else: + raise Exception("BUG: unsupported platform?") + + def build_os(self): + """Return the build os label in the binary package to download ("rhel62" + for redhat, "ubuntu1204" for Ubuntu and Debian)""" + + if re.search("^(debian|ubuntu)", self.n): + return "ubuntu1204" + elif re.search("(redhat|fedora|centos)", self.n): + return "rhel62" + else: + raise Exception("BUG: unsupported platform?") +def main(argv): + (flags, specs) = parse_args(argv[1:]) + distros=[Distro(distro) for distro in DISTROS] + + oldcwd=os.getcwd() + srcdir=oldcwd+"/../" + + # We do all our work in a randomly-created directory. You can set + # TEMPDIR to influence where this program will do stuff. + prefix=tempfile.mkdtemp() + print "Working in directory %s" % prefix + + # This will be a list of directories where we put packages in + # "repository layout". + repos=[] + + os.chdir(prefix) + try: + # Download the binaries. + urlfmt="http://downloads.10gen.com/linux/mongodb-linux-%s-subscription-%s-%s.tgz" + + # Build a pacakge for each distro/spec/arch tuple, and + # accumulate the repository-layout directories. + for (distro, spec, arch) in crossproduct(distros, specs, ARCHES): + + httpget(urlfmt % (arch, distro.build_os(), spec.version()), ensure_dir(tarfile(distro, arch, spec))) + + repos.append(make_package(distro, arch, spec, srcdir)) + + # Build the repos' metadatas. + for repo in set(repos): + print repo + make_repo(repo) + + finally: + os.chdir(oldcwd) + if "-n" not in flags: + move_repos_into_place(prefix+"/repo", REPOPATH) + # FIXME: try shutil.rmtree some day. + sysassert(["rm", "-rv", prefix]) + + +def parse_args(args): + if len(args) == 0: + print """Usage: packager.py [OPTS] SPEC1 SPEC2 ... SPECn + +Options: + + -n: Just build the packages, don't publish them as a repo + or clean out the working directory + +Each SPEC is a mongodb version string optionally followed by a colon +and some parameters, of the form <paramname>=<value>. Supported +parameters: + + suffix -- suffix to append to the package's base name. (If + unsupplied, suffixes default based on the parity of the + middle number in the version.) + + revision -- least-order version number to packaging systems +""" + sys.exit(0) + + try: + (flags, args) = getopt.getopt(args, "n") + except getopt.GetoptError, err: + print str(err) + sys.exit(2) + flags=dict(flags) + specs=[Spec(arg) for arg in args] + return (flags, specs) + +def crossproduct(*seqs): + """A generator for iterating all the tuples consisting of elements + of seqs.""" + l = len(seqs) + if l == 0: + pass + elif l == 1: + for i in seqs[0]: + yield [i] + else: + for lst in crossproduct(*seqs[:-1]): + for i in seqs[-1]: + lst2=list(lst) + lst2.append(i) + yield lst2 + +def sysassert(argv): + """Run argv and assert that it exited with status 0.""" + print "In %s, running %s" % (os.getcwd(), " ".join(argv)) + sys.stdout.flush() + sys.stderr.flush() + assert(subprocess.Popen(argv).wait()==0) + +def backtick(argv): + """Run argv and return its output string.""" + print "In %s, running %s" % (os.getcwd(), " ".join(argv)) + sys.stdout.flush() + sys.stderr.flush() + return subprocess.Popen(argv, stdout=subprocess.PIPE).communicate()[0] + +def ensure_dir(filename): + """Make sure that the directory that's the dirname part of + filename exists, and return filename.""" + dirpart = os.path.dirname(filename) + try: + os.makedirs(dirpart) + except OSError: # as exc: # Python >2.5 + exc=sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + return filename + + +def tarfile(distro, arch, spec): + """Return the location where we store the downloaded tarball for + (arch, spec)""" + return "dl/mongodb-linux-%s-subscription-%s-%s.tar.gz" % (spec.version(), distro.build_os(), arch) + +def setupdir(distro, arch, spec): + # The setupdir will be a directory containing all inputs to the + # distro's packaging tools (e.g., package metadata files, init + # scripts, etc), along with the already-built binaries). In case + # the following format string is unclear, an example setupdir + # would be dst/x86_64/debian-sysvinit/mongodb-10gen-unstable/ + return "dst/%s/%s/%s%s-%s/" % (arch, distro.name(), distro.pkgbase(), spec.suffix(), spec.pversion(distro)) + +def unpack_binaries_into(distro, arch, spec, where): + """Unpack the tarfile for (distro, arch, spec) into directory where.""" + rootdir=os.getcwd() + ensure_dir(where) + # Note: POSIX tar doesn't require support for gtar's "-C" option, + # and Python's tarfile module prior to Python 2.7 doesn't have the + # features to make this detail easy. So we'll just do the dumb + # thing and chdir into where and run tar there. + os.chdir(where) + try: + sysassert(["tar", "xvzf", rootdir+"/"+tarfile(distro, arch, spec), "mongodb-linux-%s-subscription-%s-%s/bin" % (arch, distro.build_os(), spec.version())]) + os.rename("mongodb-linux-%s-subscription-%s-%s/bin" % (arch, distro.build_os(), spec.version()), "bin") + os.rmdir("mongodb-linux-%s-subscription-%s-%s" % (arch, distro.build_os(), spec.version())) + except Exception: + exc=sys.exc_value + os.chdir(rootdir) + raise exc + os.chdir(rootdir) + +def make_package(distro, arch, spec, srcdir): + """Construct the package for (arch, distro, spec), getting + packaging files from srcdir and any user-specified suffix from + suffixes""" + + sdir=setupdir(distro, arch, spec) + ensure_dir(sdir) + # Note that the RPM packages get their man pages from the debian + # directory, so the debian directory is needed in all cases (and + # innocuous in the debianoids' sdirs). + for pkgdir in ["debian", "rpm"]: + print "Copying packaging files from %s to %s" % ("%s/%s" % (srcdir, pkgdir), sdir) + # FIXME: sh-dash-cee is bad. See if tarfile can do this. + sysassert(["sh", "-c", "(cd \"%s\" && git archive r%s %s/ ) | (cd \"%s\" && tar xvf -)" % (srcdir, spec.version(), pkgdir, sdir)]) + # Splat the binaries under sdir. The "build" stages of the + # packaging infrastructure will move the binaries to wherever they + # need to go. + unpack_binaries_into(distro, arch, spec, sdir+("%s/usr/"%BINARYDIR)) + # Remove the mongosniff binary due to libpcap dynamic + # linkage. FIXME: this removal should go away + # eventually. + if os.path.exists(sdir+("%s/usr/bin/mongosniff"%BINARYDIR)): + os.unlink(sdir+("%s/usr/bin/mongosniff"%BINARYDIR)) + return distro.make_pkg(arch, spec, srcdir) + +def make_repo(repodir): + if re.search("(debian|ubuntu)", repodir): + make_deb_repo(repodir) + elif re.search("(centos|redhat|fedora)", repodir): + make_rpm_repo(repodir) + else: + raise Exception("BUG: unsupported platform?") + +def make_deb(distro, arch, spec, srcdir): + # I can't remember the details anymore, but the initscript/upstart + # job files' names must match the package name in some way; and + # see also the --name flag to dh_installinit in the generated + # debian/rules file. + suffix=spec.suffix() + sdir=setupdir(distro, arch, spec) + if re.search("sysvinit", distro.name()): + os.link(sdir+"debian/init.d", sdir+"debian/%s%s.mongodb.init" % (distro.pkgbase(), suffix)) + os.unlink(sdir+"debian/mongodb.upstart") + elif re.search("upstart", distro.name()): + os.link(sdir+"debian/mongodb.upstart", sdir+"debian/%s%s.upstart" % (distro.pkgbase(), suffix)) + os.unlink(sdir+"debian/init.d") + else: + raise Exception("unknown debianoid flavor: not sysvinit or upstart?") + # Rewrite the control and rules files + write_debian_control_file(sdir+"debian/control", spec) + write_debian_rules_file(sdir+"debian/rules", spec) + write_debian_changelog(sdir+"debian/changelog", spec, srcdir) + distro_arch=distro.archname(arch) + # Do the packaging. + oldcwd=os.getcwd() + try: + os.chdir(sdir) + sysassert(["dpkg-buildpackage", "-a"+distro_arch, "-k Richard Kreuter <richard@10gen.com>"]) + finally: + os.chdir(oldcwd) + r=distro.repodir(arch) + ensure_dir(r) + # FIXME: see if shutil.copyfile or something can do this without + # much pain. + sysassert(["cp", "-v", sdir+"../%s%s_%s%s_%s.deb"%(distro.pkgbase(), suffix, spec.pversion(distro), "-"+spec.param("revision") if spec.param("revision") else"", distro_arch), r]) + return r + +def make_deb_repo(repo): + # Note: the Debian repository Packages files must be generated + # very carefully in order to be usable. + oldpwd=os.getcwd() + os.chdir(repo+"../../../../") + try: + dirs=set([os.path.dirname(deb)[2:] for deb in backtick(["find", ".", "-name", "*.deb"]).split()]) + for d in dirs: + s=backtick(["dpkg-scanpackages", d, "/dev/null"]) + f=open(d+"/Packages", "w") + try: + f.write(s) + finally: + f.close() + b=backtick(["gzip", "-9c", d+"/Packages"]) + f=open(d+"/Packages.gz", "wb") + try: + f.write(b) + finally: + f.close() + finally: + os.chdir(oldpwd) + # Notes: the Release{,.gpg} files must live in a special place, + # and must be created after all the Packages.gz files have been + # done. + s=""" +Origin: 10gen +Label: 10gen +Suite: 10gen +Codename: %s +Version: %s +Architectures: i386 amd64 +Components: 10gen +Description: 10gen packages +""" % ("dist", "dist") + if os.path.exists(repo+"../../Release"): + os.unlink(repo+"../../Release") + if os.path.exists(repo+"../../Release.gpg"): + os.unlink(repo+"../../Release.gpg") + oldpwd=os.getcwd() + os.chdir(repo+"../../") + s2=backtick(["apt-ftparchive", "release", "."]) + try: + f=open("Release", 'w') + try: + f.write(s) + f.write(s2) + finally: + f.close() + + arg=None + for line in backtick(["gpg", "--list-keys"]).split("\n"): + tokens=line.split() + if len(tokens)>0 and tokens[0] == "uid": + arg=tokens[-1] + break + # Note: for some reason, I think --no-tty might be needed + # here, but maybe not. + sysassert(["gpg", "-r", arg, "--no-secmem-warning", "-abs", "--output", "Release.gpg", "Release"]) + finally: + os.chdir(oldpwd) + + +def move_repos_into_place(src, dst): + # Find all the stuff in src/*, move it to a freshly-created + # directory beside dst, then play some games with symlinks so that + # dst is a name the new stuff and dst+".old" names the previous + # one. This feels like a lot of hooey for something so trivial. + + # First, make a crispy fresh new directory to put the stuff in. + i=0 + while True: + date_suffix=time.strftime("%Y-%m-%d") + dname=dst+".%s.%d" % (date_suffix, i) + try: + os.mkdir(dname) + break + except OSError: + exc=sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + i=i+1 + + # Put the stuff in our new directory. + for r in os.listdir(src): + sysassert(["cp", "-rv", src + "/" + r, dname]) + + # Make a symlink to the new directory; the symlink will be renamed + # to dst shortly. + i=0 + while True: + tmpnam=dst+".TMP.%d" % i + try: + os.symlink(dname, tmpnam) + break + except OSError: # as exc: # Python >2.5 + exc=sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + i=i+1 + + # Make a symlink to the old directory; this symlink will be + # renamed shortly, too. + oldnam=None + if os.path.exists(dst): + i=0 + while True: + oldnam=dst+".old.%d" % i + try: + os.symlink(os.readlink(dst), oldnam) + break + except OSError: # as exc: # Python >2.5 + exc=sys.exc_value + if exc.errno == errno.EEXIST: + pass + else: + raise exc + + os.rename(tmpnam, dst) + if oldnam: + os.rename(oldnam, dst+".old") + + +def write_debian_changelog(path, spec, srcdir): + oldcwd=os.getcwd() + os.chdir(srcdir) + preamble="" + if spec.param("revision"): + preamble="""mongodb%s (%s-%s) unstable; urgency=low + + * Bump revision number + + -- Richard Kreuter <richard@10gen.com> %s + +""" % (spec.suffix(), spec.pversion(Distro("debian")), spec.param("revision"), time.strftime("%a, %d %b %Y %H:%m:%S %z")) + try: + s=preamble+backtick(["sh", "-c", "git archive r%s debian/changelog | tar xOf -" % spec.version()]) + finally: + os.chdir(oldcwd) + f=open(path, 'w') + lines=s.split("\n") + # If the first line starts with "mongodb", it's not a revision + # preamble, and so frob the version number. + lines[0]=re.sub("^mongodb \\(.*\\)", "mongodb (%s)" % (spec.pversion(Distro("debian"))), lines[0]) + # Rewrite every changelog entry starting in mongodb<space> + lines=[re.sub("^mongodb ", "mongodb%s " % (spec.suffix()), l) for l in lines] + lines=[re.sub("^ --", " --", l) for l in lines] + s="\n".join(lines) + try: + f.write(s) + finally: + f.close() + +def write_debian_control_file(path, spec): + s="""Source: @@PACKAGE_BASENAME@@ +Section: devel +Priority: optional +Maintainer: Richard Kreuter <richard@10gen.com> +Build-Depends: +Standards-Version: 3.8.0 +Homepage: http://www.mongodb.org + +Package: @@PACKAGE_BASENAME@@ +Conflicts: @@PACKAGE_CONFLICTS@@ +Architecture: any +Depends: libc6 (>= 2.3.2), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.1.1), libsnmp15, libsasl2-2, libssl1.0.0 +Description: An object/document-oriented database + MongoDB is a high-performance, open source, schema-free + document-oriented data store that's easy to deploy, manage + and use. It's network accessible, written in C++ and offers + the following features : + . + * Collection oriented storage - easy storage of object- + style data + * Full index support, including on inner objects + * Query profiling + * Replication and fail-over support + * Efficient storage of binary data including large + objects (e.g. videos) + * Auto-sharding for cloud-level scalability (Q209) + . + High performance, scalability, and reasonable depth of + functionality are the goals for the project. +""" + s=re.sub("@@PACKAGE_BASENAME@@", "mongodb%s" % spec.suffix(), s) + conflict_suffixes=["", "-stable", "-unstable", "-nightly", "-10gen", "-10gen-unstable", "-10gen-enterprise"] + conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()] + s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongodb"+suffix for suffix in conflict_suffixes] + [ "mongodb18"+suffix for suffix in conflict_suffixes] + ["mongodb20"+suffix for suffix in conflict_suffixes]), s) + + f=open(path, 'w') + try: + f.write(s) + finally: + f.close() + +def write_debian_rules_file(path, spec): + # Note debian/rules is a makefile, so for visual disambiguation we + # make all tabs here \t. + s="""#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + + +configure: configure-stamp +configure-stamp: +\tdh_testdir + # Add here commands to configure the package. + +\ttouch configure-stamp + + +build: build-stamp + +build-stamp: configure-stamp +\tdh_testdir + + # Add here commands to compile the package. +# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED. +\t# scons + #docbook-to-man debian/mongodb.sgml > mongodb.1 +\tls debian/*.1 > debian/@@PACKAGE_NAME@@.manpages + +\ttouch $@ + +clean: +\tdh_testdir +\tdh_testroot +\trm -f build-stamp configure-stamp + +\t# FIXME: scons freaks out at the presence of target files +\t# under debian/mongodb. +\t#scons -c +\trm -rf $(CURDIR)/debian/@@PACKAGE_NAME@@ +\trm -f config.log +\trm -f mongo +\trm -f mongod +\trm -f mongoimportjson +\trm -f mongoexport +\trm -f mongorestore +\trm -f mongodump +\trm -f mongofiles +\trm -f .sconsign.dblite +\trm -f libmongoclient.a +\trm -rf client/*.o +\trm -rf tools/*.o +\trm -rf shell/*.o +\trm -rf .sconf_temp +\trm -f buildscripts/*.pyc +\trm -f *.pyc +\trm -f buildinfo.cpp +\tdh_clean debian/files + +install: build +\tdh_testdir +\tdh_testroot +\tdh_prep +\tdh_installdirs + +# THE FOLLOWING LINE IS INTENTIONALLY COMMENTED. +\t# scons --prefix=$(CURDIR)/debian/mongodb/usr install +\tcp -v $(CURDIR)/@@BINARYDIR@@/usr/bin/* $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/bin +\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/etc +\tcp $(CURDIR)/debian/mongodb.conf $(CURDIR)/debian/@@PACKAGE_NAME@@/etc/mongodb.conf + +\tmkdir -p $(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/ +\tinstall -m 644 $(CURDIR)/debian/lintian-overrides \ +\t\t$(CURDIR)/debian/@@PACKAGE_NAME@@/usr/share/lintian/overrides/@@PACKAGE_NAME@@ + +# Build architecture-independent files here. +binary-indep: build install +# We have nothing to do by default. + +# Build architecture-dependent files here. +binary-arch: build install +\tdh_testdir +\tdh_testroot +\tdh_installchangelogs +\tdh_installdocs +\tdh_installexamples +#\tdh_install +#\tdh_installmenu +#\tdh_installdebconf\t +#\tdh_installlogrotate +#\tdh_installemacsen +#\tdh_installpam +#\tdh_installmime +\tdh_installinit --name=@@PACKAGE_BASENAME@@ +#\tdh_installinfo +\tdh_installman +\tdh_link +# Appears to be broken on Ubuntu 11.10...? +#\tdh_strip +\tdh_compress +\tdh_fixperms +\tdh_installdeb +\tdh_shlibdeps +\tdh_gencontrol +\tdh_md5sums +\tdh_builddeb + +binary: binary-indep binary-arch +.PHONY: build clean binary-indep binary-arch binary install configure +""" + s=re.sub("@@PACKAGE_NAME@@", "mongodb%s" % spec.suffix(), s) + s=re.sub("@@PACKAGE_BASENAME@@", "mongodb", s) + s=re.sub("@@BINARYDIR@@", BINARYDIR, s) + f=open(path, 'w') + try: + f.write(s) + finally: + f.close() + # FIXME: some versions of debianoids seem to + # need the rules file to be 755? + os.chmod(path, stat.S_IXUSR|stat.S_IWUSR|stat.S_IRUSR|stat.S_IXGRP|stat.S_IRGRP|stat.S_IXOTH|stat.S_IWOTH) + +def make_rpm(distro, arch, spec, srcdir): + # Create the specfile. + suffix=spec.suffix() + sdir=setupdir(distro, arch, spec) + specfile=sdir+"rpm/mongo%s.spec" % suffix + write_rpm_spec_file(specfile, spec) + topdir=ensure_dir(os.getcwd()+'/rpmbuild/') + for subdir in ["BUILD", "RPMS", "SOURCES", "SPECS", "SRPMS"]: + ensure_dir("%s/%s/" % (topdir, subdir)) + distro_arch=distro.archname(arch) + # RPM tools take these macro files that define variables in + # RPMland. Unfortunately, there's no way to tell RPM tools to use + # a given file *in addition* to the files that it would already + # load, so we have to figure out what it would normally load, + # augment that list, and tell RPM to use the augmented list. To + # figure out what macrofiles ordinarily get loaded, older RPM + # versions had a parameter called "macrofiles" that could be + # extracted from "rpm --showrc". But newer RPM versions don't + # have this. To tell RPM what macros to use, older versions of + # RPM have a --macros option that doesn't work; on these versions, + # you can put a "macrofiles" parameter into an rpmrc file. But + # that "macrofiles" setting doesn't do anything for newer RPM + # versions, where you have to use the --macros flag instead. And + # all of this is to let us do our work with some guarantee that + # we're not clobbering anything that doesn't belong to us. Why is + # RPM so braindamaged? + macrofiles=[l for l in backtick(["rpm", "--showrc"]).split("\n") if l.startswith("macrofiles")] + flags=[] + macropath=os.getcwd()+"/macros" + write_rpm_macros_file(macropath, topdir) + if len(macrofiles)>0: + macrofiles=macrofiles[0]+":"+macropath + rcfile=os.getcwd()+"/rpmrc" + write_rpmrc_file(rcfile, macrofiles) + flags=["--rpmrc", rcfile] + else: + # This hard-coded hooey came from some box running RPM + # 4.4.2.3. It may not work over time, but RPM isn't sanely + # configurable. + flags=["--macros", "/usr/lib/rpm/macros:/usr/lib/rpm/%s-linux/macros:/etc/rpm/macros.*:/etc/rpm/macros:/etc/rpm/%s-linux/macros:~/.rpmmacros:%s" % (distro_arch, distro_arch, macropath)] + # Put the specfile and the tar'd up binaries and stuff in + # place. FIXME: see if shutil.copyfile can do this without too + # much hassle. + sysassert(["cp", "-v", specfile, topdir+"SPECS/"]) + oldcwd=os.getcwd() + os.chdir(sdir+"/../") + try: + sysassert(["tar", "-cpzf", topdir+"SOURCES/mongo%s-%s.tar.gz" % (suffix, spec.pversion(distro)), os.path.basename(os.path.dirname(sdir))]) + finally: + os.chdir(oldcwd) + # Do the build. + sysassert(["rpmbuild", "-ba", "--target", distro_arch] + flags + ["%s/SPECS/mongo%s.spec" % (topdir, suffix)]) + r=distro.repodir(arch) + ensure_dir(r) + # FIXME: see if some combination of shutil.copy<hoohah> and glob + # can do this without shelling out. + sysassert(["sh", "-c", "cp -v \"%s/RPMS/%s/\"*.rpm \"%s\""%(topdir, distro_arch, r)]) + return r + +def make_rpm_repo(repo): + oldpwd=os.getcwd() + os.chdir(repo+"../") + try: + sysassert(["createrepo", "."]) + finally: + os.chdir(oldpwd) + + +def write_rpmrc_file(path, string): + f=open(path, 'w') + try: + f.write(string) + finally: + f.close() + +def write_rpm_macros_file(path, topdir): + f=open(path, 'w') + try: + f.write("%%_topdir %s" % topdir) + finally: + f.close() + +def write_rpm_spec_file(path, spec): + s="""Name: @@PACKAGE_BASENAME@@ +Conflicts: @@PACKAGE_CONFLICTS@@ +Obsoletes: @@PACKAGE_OBSOLETES@@ +Version: @@PACKAGE_VERSION@@ +Release: mongodb_@@PACKAGE_REVISION@@%{?dist} +Summary: mongo client shell and tools +License: AGPL 3.0 +URL: http://www.mongodb.org +Group: Applications/Databases +Requires: cyrus-sasl, net-snmp-libs + +Source0: %{name}-%{version}.tar.gz +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root + +%description +Mongo (from "huMONGOus") is a schema-free document-oriented database. +It features dynamic profileable queries, full indexing, replication +and fail-over support, efficient storage of large binary data objects, +and auto-sharding. + +This package provides the mongo shell, import/export tools, and other +client utilities. + +%package server +Summary: mongo server, sharding server, and support scripts +Group: Applications/Databases +Requires: @@PACKAGE_BASENAME@@ + +%description server +Mongo (from "huMONGOus") is a schema-free document-oriented database. + +This package provides the mongo server software, mongo sharding server +softwware, default configuration files, and init.d scripts. + +%package devel +Summary: Headers and libraries for mongo development. +Group: Applications/Databases + +%description devel +Mongo (from "huMONGOus") is a schema-free document-oriented database. + +This package provides the mongo static library and header files needed +to develop mongo client software. + +%prep +%setup + +%build +#scons --prefix=$RPM_BUILD_ROOT/usr all +# XXX really should have shared library here + +%install +#scons --prefix=$RPM_BUILD_ROOT/usr install +mkdir -p $RPM_BUILD_ROOT/usr +cp -rv @@BINARYDIR@@/usr/bin $RPM_BUILD_ROOT/usr +mkdir -p $RPM_BUILD_ROOT/usr/share/man/man1 +cp debian/*.1 $RPM_BUILD_ROOT/usr/share/man/man1/ +# FIXME: remove this rm when mongosniff is back in the package +rm -v $RPM_BUILD_ROOT/usr/share/man/man1/mongosniff.1* +mkdir -p $RPM_BUILD_ROOT/etc/rc.d/init.d +cp -v rpm/init.d-mongod $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod +chmod a+x $RPM_BUILD_ROOT/etc/rc.d/init.d/mongod +mkdir -p $RPM_BUILD_ROOT/etc +cp -v rpm/mongod.conf $RPM_BUILD_ROOT/etc/mongod.conf +mkdir -p $RPM_BUILD_ROOT/etc/sysconfig +cp -v rpm/mongod.sysconfig $RPM_BUILD_ROOT/etc/sysconfig/mongod +mkdir -p $RPM_BUILD_ROOT/var/lib/mongo +mkdir -p $RPM_BUILD_ROOT/var/log/mongo +touch $RPM_BUILD_ROOT/var/log/mongo/mongod.log + +%clean +#scons -c +rm -rf $RPM_BUILD_ROOT + +%pre server +if ! /usr/bin/id -g mongod &>/dev/null; then + /usr/sbin/groupadd -r mongod +fi +if ! /usr/bin/id mongod &>/dev/null; then + /usr/sbin/useradd -M -r -g mongod -d /var/lib/mongo -s /bin/false \ + -c mongod mongod > /dev/null 2>&1 +fi + +%post server +if test $1 = 1 +then + /sbin/chkconfig --add mongod +fi + +%preun server +if test $1 = 0 +then + /sbin/chkconfig --del mongod +fi + +%postun server +if test $1 -ge 1 +then + /sbin/service mongod condrestart >/dev/null 2>&1 || : +fi + +%files +%defattr(-,root,root,-) +#%doc README GNU-AGPL-3.0.txt + +%{_bindir}/bsondump +%{_bindir}/mongo +%{_bindir}/mongodump +%{_bindir}/mongoexport +#@@VERSION!=2.1.0@@%{_bindir}/mongofiles +%{_bindir}/mongoimport +#@@VERSION>=2.1.0@@%{_bindir}/mongooplog +#@@VERSION>=2.1.0@@%{_bindir}/mongoperf +%{_bindir}/mongorestore +#@@VERSION>1.9@@%{_bindir}/mongotop +%{_bindir}/mongostat +# FIXME: uncomment when mongosniff is back in the package +#%{_bindir}/mongosniff + +# FIXME: uncomment this when there's a stable release whose source +# tree contains a bsondump man page. +#@@VERSION>1.9@@%{_mandir}/man1/bsondump.1* +%{_mandir}/man1/mongo.1* +%{_mandir}/man1/mongodump.1* +%{_mandir}/man1/mongoexport.1* +%{_mandir}/man1/mongofiles.1* +%{_mandir}/man1/mongoimport.1* +%{_mandir}/man1/mongorestore.1* +%{_mandir}/man1/mongostat.1* +# FIXME: uncomment when mongosniff is back in the package +#%{_mandir}/man1/mongosniff.1* +#@@VERSION>=2.4.0@@%{_mandir}/man1/mongotop.1* +#@@VERSION>=2.4.0@@%{_mandir}/man1/mongoperf.1* +#@@VERSION>=2.4.0@@%{_mandir}/man1/mongooplog.1* + +%files server +%defattr(-,root,root,-) +%config(noreplace) /etc/mongod.conf +%{_bindir}/mongod +%{_bindir}/mongos +%{_mandir}/man1/mongod.1* +%{_mandir}/man1/mongos.1* +/etc/rc.d/init.d/mongod +/etc/sysconfig/mongod +#/etc/rc.d/init.d/mongos +%attr(0755,mongod,mongod) %dir /var/lib/mongo +%attr(0755,mongod,mongod) %dir /var/log/mongo +%attr(0640,mongod,mongod) %config(noreplace) %verify(not md5 size mtime) /var/log/mongo/mongod.log + +%changelog +* Thu Jan 28 2010 Richard M Kreuter <richard@10gen.com> +- Minor fixes. + +* Sat Oct 24 2009 Joe Miklojcik <jmiklojcik@shopwiki.com> - +- Wrote mongo.spec. +""" + suffix=spec.suffix() + s=re.sub("@@PACKAGE_BASENAME@@", "mongo%s" % suffix, s) + s=re.sub("@@PACKAGE_VERSION@@", spec.pversion(Distro("redhat")), s) + # FIXME, maybe: the RPM guide says that Release numbers ought to + # be integers starting at 1, but we use "mongodb_1{%dist}", + # whatever the hell that means. + s=re.sub("@@PACKAGE_REVISION@@", str(int(spec.param("revision"))+1) if spec.param("revision") else "1", s) + s=re.sub("@@BINARYDIR@@", BINARYDIR, s) + conflict_suffixes=["", "-10gen", "-10gen-unstable", "-10gen-enterprise"] + conflict_suffixes = [suff for suff in conflict_suffixes if suff != spec.suffix()] + s=re.sub("@@PACKAGE_CONFLICTS@@", ", ".join(["mongo"+suffix for suffix in conflict_suffixes] + [ "mongo18"+suffix for suffix in conflict_suffixes] + ["mongo20"+suffix for suffix in conflict_suffixes]), s) + if suffix.endswith("-10gen"): + s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-stable", s) + s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-stable", s) + elif suffix == "-10gen-unstable": + s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-unstable", s) + s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-unstable", s) + elif suffix == "-10gen-enterprise": + s=re.sub("@@PACKAGE_PROVIDES@@", "mongo-enterprise", s) + s=re.sub("@@PACKAGE_OBSOLETES@@", "mongo-enterprise", s) + else: + raise Exception("BUG: unknown suffix %s" % suffix) + + lines=[] + for line in s.split("\n"): + m = re.search("@@VERSION(>|>=|!=)(\d.*)@@(.*)", line) + if m: + op = m.group(1) + ver = m.group(2) + fn = m.group(3) + if op == '>': + if spec.version_better_than(ver): + lines.append(fn) + elif op == '>=': + if spec.version() == ver or spec.version_better_than(ver): + lines.append(fn) + elif op == '!=': + if spec.version() != ver: + lines.append(fn) + else: + # Since we're inventing our own template system for RPM + # specfiles here, we oughtn't use template syntax we don't + # support. + raise Exception("BUG: probable bug in packager script: %s, %s, %s" % (m.group(1), m.group(2), m.group(3))) + else: + lines.append(line) + s="\n".join(lines) + + f=open(path, 'w') + try: + f.write(s) + finally: + f.close() + +if __name__ == "__main__": + main(sys.argv) diff --git a/doxygenConfig b/doxygenConfig index 1eebb6c10b9..90b26608766 100644 --- a/doxygenConfig +++ b/doxygenConfig @@ -3,7 +3,7 @@ #--------------------------------------------------------------------------- DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = MongoDB -PROJECT_NUMBER = 2.4.6 +PROJECT_NUMBER = 2.4.9 OUTPUT_DIRECTORY = docs/doxygen CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/jstests/aggregation/mongos_slaveok.js b/jstests/aggregation/mongos_slaveok.js new file mode 100644 index 00000000000..e5d0f6692ae --- /dev/null +++ b/jstests/aggregation/mongos_slaveok.js @@ -0,0 +1,38 @@ +/** + * Tests aggregate command against mongos with slaveOk. For more tests on read preference, + * please refer to jstests/sharding/read_pref_cmd.js. + */ + +var NODES = 2; + +var doTest = function(st, doSharded) { +var testDB = st.s.getDB('test'); + +if (doSharded) { + testDB.adminCommand({ enableSharding: 'test' }); + testDB.adminCommand({ shardCollection: 'test.user', key: { x: 1 }}); +} + +testDB.user.insert({ x: 10 }); +testDB.runCommand({ getLastError: 1, w: NODES }); +testDB.setSlaveOk(true); + +var secNode = st.rs0.getSecondary(); +secNode.getDB('test').setProfilingLevel(2); + +var res = testDB.runCommand({ aggregate: 'user', pipeline: [{ $project: { x: 1 }}]}); +assert(res.ok, 'aggregate command failed: ' + tojson(res)); + +var profileQuery = { op: 'command', ns: 'test.$cmd', 'command.aggregate': 'user' }; +var profileDoc = secNode.getDB('test').system.profile.findOne(profileQuery); + +assert(profileDoc != null); +testDB.dropDatabase(); +}; + +var st = new ShardingTest({ shards: { rs0: { oplogSize: 10, verbose: 1, nodes: NODES }}}); + +doTest(st, false); +doTest(st, true); + +st.stop(); diff --git a/jstests/auth/js_scope_leak.js b/jstests/auth/js_scope_leak.js new file mode 100644 index 00000000000..c8c19b61aef --- /dev/null +++ b/jstests/auth/js_scope_leak.js @@ -0,0 +1,139 @@ +// Test for SERVER-9129 +// Verify global scope data does not persist past logout or auth. +// NOTE: Each test case covers 3 state transitions: +// no auth -> auth user 'a' +// auth user 'a' -> auth user 'b' +// auth user 'b' -> logout +// +// These transitions are tested for dbEval, $where, MapReduce and $group + +var conn = MongoRunner.runMongod({ auth: "", smallfiles: ""}); +var test = conn.getDB("test"); + +// insert a single document and add two test users +test.foo.insert({a:1}); +test.getLastError(); +assert.eq(1, test.foo.findOne().a); +test.addUser('a', 'a'); +test.addUser('b', 'b'); + +function missingOrEquals(string) { + return 'function() { ' + + 'var global = function(){return this;}.call();' + // Uncomment the next line when debugging. + // + 'print(global.hasOwnProperty("someGlobal") ? someGlobal : "MISSING" );' + + 'return !global.hasOwnProperty("someGlobal")' + + ' || someGlobal == unescape("' + escape(string) + '");' + +'}()' +} + +function testDbEval() { + // set the global variable 'someGlobal' before authenticating + test.eval('someGlobal = "noUsers";'); + + // test new user auth causes scope to be cleared + test.auth('a', 'a'); + assert(test.eval('return ' + missingOrEquals('a')), "dbEval: Auth user 'a'"); + + // test auth as another user causes scope to be cleared + test.eval('someGlobal = "a";'); + test.auth('b', 'b'); + assert(test.eval('return ' + missingOrEquals('a&b')), "dbEval: Auth user 'b'"); + + // test user logout causes scope to be cleared + test.eval('someGlobal = "a&b";'); + test.logout(); + assert(test.eval('return ' + missingOrEquals('noUsers')), "dbEval: log out"); +} +testDbEval(); +testDbEval(); + +// test $where +function testWhere() { + // set the global variable 'someGlobal' before authenticating + test.foo.findOne({$where:'someGlobal = "noUsers";'}); + + // test new user auth causes scope to be cleared + test.auth('a', 'a'); + assert.eq(1, + test.foo.count({$where: 'return ' + missingOrEquals('a')}), + "$where: Auth user 'a"); + + // test auth as another user causes scope to be cleared + test.foo.findOne({$where:'someGlobal = "a";'}); + test.auth('b', 'b'); + assert(test.foo.count({$where: 'return ' + missingOrEquals('a&b')}), "$where: Auth user 'b'"); + // test user logout causes scope to be cleared + test.foo.findOne({$where:'someGlobal = "a&b";'}); + test.logout(); + assert(test.foo.count({$where: 'return ' + missingOrEquals('noUsers')}), "$where: log out"); +} +testWhere(); +testWhere(); + +function testMapReduce() { + var mapSet = function(string) { return Function('someGlobal = "' + string + '"'); } + var mapGet = function(string) { return Function('assert(' + missingOrEquals(string) +')'); } + var reduce = function(k, v) { } + var setGlobalInMap = function(string) { + test.foo.mapReduce(mapSet(string), reduce, {out:{inline:1}}); + } + var getGlobalFromMap = function(string) { + test.foo.mapReduce(mapGet(string), reduce, {out:{inline:1}}); + } + + // set the global variable 'someGlobal' before authenticating + setGlobalInMap('noUsers'); + + // test new user auth causes scope to be cleared + test.auth('a', 'a'); + getGlobalFromMap('a'); // throws on fail + + // test auth as another user causes scope to be cleared + setGlobalInMap('a'); + test.auth('b', 'b'); + getGlobalFromMap('a&b'); // throws on fail + + // test user logout causes scope to be cleared + setGlobalInMap('a&b'); + test.logout(); + getGlobalFromMap('noUsers'); // throws on fail +} +testMapReduce(); +testMapReduce(); + +function testGroup() { + var setGlobalInGroup = function(string) { + return test.foo.group({key: 'a', + reduce: Function('doc1', 'agg', + 'someGlobal = "' + string + '"'), + initial:{}}); + } + var getGlobalFromGroup = function(string) { + return test.foo.group({key: 'a', + reduce: Function('doc1', 'agg', + 'assert(' + missingOrEquals(string) +')'), + initial:{}}); + } + + // set the global variable 'someGlobal' before authenticating + setGlobalInGroup('noUsers'); + + // test new user auth causes scope to be cleared + test.auth('a', 'a'); + getGlobalFromGroup('a'); // throws on fail + + // test auth as another user causes scope to be cleared + setGlobalInGroup('a'); + test.auth('b', 'b'); + getGlobalFromGroup('a&b'); // throws on fail + + // test user logout causes scope to be cleared + setGlobalInGroup('a&b'); + test.logout(); + getGlobalFromGroup('noUsers'); // throws on fail +} +testGroup(); +testGroup(); + + diff --git a/jstests/dbhash2.js b/jstests/dbhash2.js new file mode 100644 index 00000000000..ac491291c2b --- /dev/null +++ b/jstests/dbhash2.js @@ -0,0 +1,22 @@ + +mydb = db.getSisterDB( "config" ); + +t = mydb.foo; +t.drop(); + +t.insert( { x : 1 } ); +res1 = mydb.runCommand( "dbhash" ); +assert( res1.fromCache.indexOf( "config.foo" ) == -1 ); + +res2 = mydb.runCommand( "dbhash" ); +assert( res2.fromCache.indexOf( "config.foo" ) >= 0 ); +assert.eq( res1.collections.foo, res2.collections.foo ); + +t.insert( { x : 2 } ); +res3 = mydb.runCommand( "dbhash" ); +assert( res3.fromCache.indexOf( "config.foo" ) < 0 ); +assert.neq( res1.collections.foo, res3.collections.foo ); + + + + diff --git a/jstests/dropdb.js b/jstests/dropdb.js index 0b838846bde..6b5fa6e2fe5 100644 --- a/jstests/dropdb.js +++ b/jstests/dropdb.js @@ -6,12 +6,21 @@ m = db.getMongo(); baseName = "jstests_dropdb"; ddb = db.getSisterDB( baseName ); +print("initial dbs: " + tojson(m.getDBNames())); + +function check(shouldExist) { + var dbs = m.getDBNames(); + assert.eq(Array.contains(dbs, baseName), shouldExist, + "DB " + baseName + " should " + (shouldExist ? "" : "not ") + "exist." + + " dbs: " + tojson(dbs)); +} + ddb.c.save( {} ); ddb.getLastError(); -assert.neq( -1, m.getDBNames().indexOf( baseName ) ); +check(true); ddb.dropDatabase(); -assert.eq( -1, m.getDBNames().indexOf( baseName ) ); +check(false); ddb.dropDatabase(); -assert.eq( -1, m.getDBNames().indexOf( baseName ) ); +check(false); diff --git a/jstests/eval2.js b/jstests/eval2.js index c3a7499ae47..6e39bb4a7bd 100644 --- a/jstests/eval2.js +++ b/jstests/eval2.js @@ -1,12 +1,12 @@ -t = db.test; +t = db.eval2; t.drop(); t.save({a:1}); t.save({a:1}); var f = db.group( { - ns: "test", + ns: t.getName(), key: { a:true}, cond: { a:1 }, reduce: function(obj,prev) { prev.csum++; } , diff --git a/jstests/geo_circle2a.js b/jstests/geo_circle2a.js index 1597033c01b..67a6ba17243 100644 --- a/jstests/geo_circle2a.js +++ b/jstests/geo_circle2a.js @@ -2,35 +2,36 @@ // Tests to make sure that nested multi-key indexing works for geo indexes and is not used for direct position // lookups -db.test.drop() -db.test.insert({ p : [1112,3473], t : [{ k : 'a', v : 'b' }, { k : 'c', v : 'd' }] }) -db.test.ensureIndex({ p : '2d', 't.k' : 1 }, { min : 0, max : 10000 }) +var coll = db.geo_circle2a; +coll.drop(); +coll.insert({ p : [1112,3473], t : [{ k : 'a', v : 'b' }, { k : 'c', v : 'd' }] }) +coll.ensureIndex({ p : '2d', 't.k' : 1 }, { min : 0, max : 10000 }) // Succeeds, since on direct lookup should not use the index -assert(1 == db.test.find({p:[1112,3473],'t.k':'a'}).count(), "A") +assert(1 == coll.find({p:[1112,3473],'t.k':'a'}).count(), "A") // Succeeds and uses the geo index -assert(1 == db.test.find({p:{$within:{$box:[[1111,3472],[1113,3475]]}}, 't.k' : 'a' }).count(), "B") +assert(1 == coll.find({p:{$within:{$box:[[1111,3472],[1113,3475]]}}, 't.k' : 'a' }).count(), "B") -db.test.drop() -db.test.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' }, { k : 'key2', v : 123 } ] }) -db.test.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' } ] }) +coll.drop() +coll.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' }, { k : 'key2', v : 123 } ] }) +coll.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' } ] }) -db.test.ensureIndex({ point : "2d" , "tags.k" : 1, "tags.v" : 1 }) +coll.ensureIndex({ point : "2d" , "tags.k" : 1, "tags.v" : 1 }) // Succeeds, since should now lookup multi-keys correctly -assert(2 == db.test.find({ point : { $within : { $box : [[0,0],[12,12]] } } }).count(), "C") +assert(2 == coll.find({ point : { $within : { $box : [[0,0],[12,12]] } } }).count(), "C") // Succeeds, and should not use geoindex -assert(2 == db.test.find({ point : [1, 10] }).count(), "D") -assert(2 == db.test.find({ point : [1, 10], "tags.v" : "value" }).count(), "E") -assert(1 == db.test.find({ point : [1, 10], "tags.v" : 123 }).count(), "F") +assert(2 == coll.find({ point : [1, 10] }).count(), "D") +assert(2 == coll.find({ point : [1, 10], "tags.v" : "value" }).count(), "E") +assert(1 == coll.find({ point : [1, 10], "tags.v" : 123 }).count(), "F") -db.test.drop() -db.test.insert({ point:[ 1, 10 ], tags : [ { k : { 'hello' : 'world'}, v : 'value' }, { k : 'key2', v : 123 } ] }) -db.test.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' } ] }) +coll.drop() +coll.insert({ point:[ 1, 10 ], tags : [ { k : { 'hello' : 'world'}, v : 'value' }, { k : 'key2', v : 123 } ] }) +coll.insert({ point:[ 1, 10 ], tags : [ { k : 'key', v : 'value' } ] }) -db.test.ensureIndex({ point : "2d" , "tags.k" : 1, "tags.v" : 1 }) +coll.ensureIndex({ point : "2d" , "tags.k" : 1, "tags.v" : 1 }) // Succeeds, should be able to look up the complex element -assert(1 == db.test.find({ point : { $within : { $box : [[0,0],[12,12]] } }, 'tags.k' : { 'hello' : 'world' } }).count(), "G")
\ No newline at end of file +assert(1 == coll.find({ point : { $within : { $box : [[0,0],[12,12]] } }, 'tags.k' : { 'hello' : 'world' } }).count(), "G")
\ No newline at end of file diff --git a/jstests/replsets/gle_explicit_optime.js b/jstests/replsets/gle_explicit_optime.js new file mode 100644 index 00000000000..77c526e981d --- /dev/null +++ b/jstests/replsets/gle_explicit_optime.js @@ -0,0 +1,57 @@ +// +// Tests the use of the wOpTime option in getLastError +// + +var rst = new ReplSetTest({ nodes : 2 }); +rst.startSet(); +rst.initiate(); + +var primary = rst.getPrimary(); +var secondary = rst.getSecondary(); + +var coll = primary.getCollection( "foo.bar" ); + +// Insert a doc and replicate it to two servers +coll.insert({ some : "doc" }); +var gleObj = coll.getDB().getLastErrorObj( 2 ); // w : 2 +assert.eq( null, gleObj.err ); +var opTimeBeforeFailure = gleObj.lastOp; + +// Lock the secondary +secondary.getDB("admin").fsyncLock(); + +// Insert a doc and replicate it to the primary only +coll.insert({ some : "doc" }); +gleObj = coll.getDB().getLastErrorObj( 1 ); // w : 1 +assert.eq( null, gleObj.err ); +var opTimeAfterFailure = gleObj.lastOp; + +printjson(opTimeBeforeFailure); +printjson(opTimeAfterFailure); +printjson( primary.getDB("admin").runCommand({ replSetGetStatus : true }) ); + +// Create a new connection with new client and no opTime +var newClientConn = new Mongo( primary.host ); + +// New client has no set opTime, so w : 2 has no impact +gleObj = newClientConn.getCollection( coll.toString() ).getDB().getLastErrorObj( 2 ); // w : 2 +assert.eq( null, gleObj.err ); + +// Using an explicit optime on the new client should work if the optime is earlier than the +// secondary was locked +var gleOpTimeBefore = { getLastError : true, w : 2, wOpTime : opTimeBeforeFailure }; +gleObj = newClientConn.getCollection( coll.toString() ).getDB().runCommand( gleOpTimeBefore ); +assert.eq( null, gleObj.err ); + +// Using an explicit optime on the new client should not work if the optime is later than the +// secondary was locked +var gleOpTimeAfter = { getLastError : true, w : 2, wtimeout : 1000, wOpTime : opTimeAfterFailure }; +gleObj = newClientConn.getCollection( coll.toString() ).getDB().runCommand( gleOpTimeAfter ); +assert.neq( null, gleObj.err ); +assert( gleObj.wtimeout ); + +jsTest.log("DONE!"); + +// Unlock the secondary +secondary.getDB("admin").fsyncUnlock(); +rst.stopSet(); diff --git a/jstests/sharding/dbhash_cache.js b/jstests/sharding/dbhash_cache.js new file mode 100644 index 00000000000..7ba95361cf2 --- /dev/null +++ b/jstests/sharding/dbhash_cache.js @@ -0,0 +1,38 @@ +/** +* Test that split/move chunk update the dbhash on the config server +*/ + +st = new ShardingTest({ name: "dbhash", shards : 2, mongos : 2, verbose : 2, other: {separateConfig : 1 } }); +st.stopBalancer(); + +var mongos = st.s0; +var shards = mongos.getCollection( "config.shards" ).find().toArray(); +var admin = mongos.getDB( "admin" ); +var configs = st._configServers + +assert(admin.runCommand({ enablesharding : "test" }).ok); +printjson(admin.runCommand({ movePrimary : "test", to : shards[0]._id })); +assert(admin.runCommand({ shardcollection : "test.foo" , key : { x : 1 } }).ok); + +mongos.getCollection("test.foo").insert({x:1}); +assert.eq(1, st.config.chunks.count(), "there should only be 1 chunk") + +var dbhash1 = configs[0].getDB("config").runCommand( "dbhash"); +printjson("dbhash before split and move is " + dbhash1.collections.chunks); + +// split the chunk and move one chunk to the non-primary shard +assert(admin.runCommand({ split : "test.foo", middle : { x : 0 } }).ok); +assert( admin.runCommand({ moveChunk : "test.foo", + find : { x : 0 }, + to : shards[1]._id, + _waitForDelete : true }).ok ); + +st.printShardingStatus(); +assert.eq(2, st.config.chunks.count(), "there should be 2 chunks") + +var dbhash2 = configs[0].getDB("config").runCommand("dbhash"); +printjson("dbhash after split and move is " + dbhash2.collections.chunks); + +assert.neq(dbhash1.collections.chunks, dbhash2.collections.chunks, "The hash should be different after split and move." ) + +st.stop(); diff --git a/jstests/sharding/forget_mr_temp_ns.js b/jstests/sharding/forget_mr_temp_ns.js new file mode 100644 index 00000000000..54eeb88d9b5 --- /dev/null +++ b/jstests/sharding/forget_mr_temp_ns.js @@ -0,0 +1,46 @@ +// +// Tests whether we forget M/R's temporary namespaces for sharded output +// + +var options = { separateConfig : true }; + +var st = new ShardingTest({ shards : 1, mongos : 1, other : options }); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var coll = mongos.getCollection( "foo.bar" ); +var outputColl = mongos.getCollection( (coll.getDB() + "") + ".mrOutput" ); + +for ( var i = 0; i < 10; i++ ) { + coll.insert({ _id : i, even : (i % 2 == 0) }); +} +assert.eq( null, coll.getDB().getLastError() ); + +var map = function() { emit( this.even, 1 ); }; +var reduce = function( key, values ) { return Array.sum(values); }; + +out = coll.mapReduce( map, reduce, { out: { reduce : outputColl.getName(), sharded: true } } ); + +printjson( out ); +printjson( outputColl.find().toArray() ); + +var mongodThreadStats = st.shard0.getDB( "admin" ).runCommand({ shardConnPoolStats : 1 }).threads; +var mongosThreadStats = admin.runCommand({ shardConnPoolStats : 1 }).threads; + +printjson( mongodThreadStats ); +printjson( mongosThreadStats ); + +var checkForSeenNS = function( threadStats, regex ) { + for ( var i = 0; i < threadStats.length; i++ ) { + var seenNSes = threadStats[i].seenNS; + for ( var j = 0; j < seenNSes.length; j++ ) { + assert( !( regex.test( seenNSes ) ) ); + } + } +} + +checkForSeenNS( mongodThreadStats, /^foo.tmp/ ); +checkForSeenNS( mongosThreadStats, /^foo.tmp/ ); + +st.stop(); + diff --git a/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js new file mode 100644 index 00000000000..007f60f6d2e --- /dev/null +++ b/jstests/sharding/mongos_rs_auth_shard_failure_tolerance.js @@ -0,0 +1,341 @@ +// +// Tests mongos's failure tolerance for authenticated replica set shards and slaveOk queries +// +// Sets up a cluster with three shards, the first shard of which has an unsharded collection and +// half a sharded collection. The second shard has the second half of the sharded collection, and +// the third shard has nothing. The primary of the unsharded database is on the first shard, and +// the primary of the sharded database is on the second shard. Progressively shuts down the shards +// to test the impact on the cluster with user authentication. +// +// Three different connection states are tested - active (connection is active through whole +// sequence), idle (connection is connected but not used before a shard change), and new +// (connection connected after shard change). +// + +var options = { separateConfig : true, + rs : true, + rsOptions : { nodes : 2 }, + keyFile : "jstests/libs/key1" }; + +var st = new ShardingTest({shards : 3, mongos : 1, other : options}); +st.stopBalancer(); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getDB( "config" ).shards.find().toArray(); + +assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, authOnPrimaryOnly : false }) ); + +var collSharded = mongos.getCollection( "fooSharded.barSharded" ); +var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" ); + +// Create the unsharded database with shard0 primary +collUnsharded.insert({ some : "doc" }); +assert.eq( null, collUnsharded.getDB().getLastError() ); +collUnsharded.remove({}); +assert.eq( null, collUnsharded.getDB().getLastError() ); +printjson( admin.runCommand({ movePrimary : collUnsharded.getDB().toString(), to : shards[0]._id }) ); + +// Create the sharded database with shard1 primary +assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB().toString() }) ); +printjson( admin.runCommand({ movePrimary : collSharded.getDB().toString(), to : shards[1]._id }) ); +assert.commandWorked( admin.runCommand({ shardCollection : collSharded.toString(), + key : { _id : 1 } }) ); +assert.commandWorked( admin.runCommand({ split : collSharded.toString(), middle : { _id : 0 } }) ); +assert.commandWorked( admin.runCommand({ moveChunk : collSharded.toString(), + find : { _id : -1 }, + to : shards[0]._id }) ); + +st.printShardingStatus(); + +var adminUser = "adminUser"; +var shardedDBUser = "shardedDBUser"; +var unshardedDBUser = "unshardedDBUser"; +var password = "password"; + +jsTest.log("Setting up initial admin user..."); + +// Create a user +admin.addUser({ user : adminUser, pwd : password, roles: [ "userAdminAnyDatabase" ] }); + +// There's an admin user now, so we need to login to do anything + +// Login as admin user +admin.auth(adminUser, password); + +jsTest.log("Setting up database users..."); + +// Create db users +collSharded.getDB().addUser({ user : shardedDBUser, + pwd : password, roles : [ "readWrite" ] }); +collUnsharded.getDB().addUser({ user : unshardedDBUser, + pwd : password, roles : [ "readWrite" ] }); + +admin.logout(); + +function authDBUsers( conn ) { + conn.getDB( collSharded.getDB().toString() ).auth(shardedDBUser, password); + conn.getDB( collUnsharded.getDB().toString() ).auth(unshardedDBUser, password); + return conn; +} + +function authUnshardedUser( conn ) { + conn.getDB( collUnsharded.getDB().toString() ).auth(unshardedDBUser, password); + return conn; +} + +// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we +// don't care if this happens in this test, we only care that we did not get "write succeeded". +// Depending on the connection pool state, we could get either. +function gleErrorOrThrow(database, msg) { + var gle; + try { + gle = database.getLastErrorObj(); + } + catch (ex) { + return; + } + if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg); + return; +}; + +// +// Setup is complete +// + +jsTest.log("Inserting initial data..."); + +var mongosConnActive = authDBUsers( new Mongo( mongos.host ) ); +authDBUsers(mongosConnActive); +var mongosConnIdle = null; +var mongosConnNew = null; + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -1 }); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collSharded.toString() ).getDB().getLastError()); + +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collUnsharded.toString() ).getDB().getLastError()); + +jsTest.log("Stopping primary of third shard..."); + +mongosConnIdle = authDBUsers( new Mongo( mongos.host ) ); + +st.rs2.stop(st.rs2.getPrimary(), true ); // wait for stop + +jsTest.log("Testing active connection with third primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with third primary down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with third primary down..."); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping primary of second shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = authDBUsers( new Mongo( mongos.host ) ); +mongosConnIdle.setSlaveOk(); + +// Need to save this node for later +var rs1Secondary = st.rs1.getSecondary(); + +st.rs1.stop(st.rs1.getPrimary(), true ); // wait for stop + +jsTest.log("Testing active connection with second primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -5 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 5 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 5 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with second primary down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 6 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with second primary down..."); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -7 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 7 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 7 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping primary of first shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = authDBUsers( new Mongo( mongos.host ) ); +mongosConnIdle.setSlaveOk(); + +st.rs0.stop(st.rs0.getPrimary(), true ); // wait for stop + +jsTest.log("Testing active connection with first primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with first primary down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with first primary down..."); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = authDBUsers( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping second shard..."); + +mongosConnActive.setSlaveOk(); +mongosConnIdle = authDBUsers( new Mongo( mongos.host ) ); +mongosConnIdle.setSlaveOk(); + +st.rs1.stop(rs1Secondary, true ); // wait for stop + +jsTest.log("Testing active connection with second shard down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with second shard down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with second shard down..."); + +// Note that this would fail for the sharded database with a primary on the second shard + +mongosConnNew = authUnshardedUser( new Mongo( mongos.host ) ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = authUnshardedUser( new Mongo( mongos.host ) ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("DONE!"); +st.stop(); diff --git a/jstests/sharding/mongos_rs_shard_failure_tolerance.js b/jstests/sharding/mongos_rs_shard_failure_tolerance.js new file mode 100644 index 00000000000..61743d1b450 --- /dev/null +++ b/jstests/sharding/mongos_rs_shard_failure_tolerance.js @@ -0,0 +1,460 @@ +// +// Tests mongos's failure tolerance for replica set shards and read preference queries +// +// Sets up a cluster with three shards, the first shard of which has an unsharded collection and +// half a sharded collection. The second shard has the second half of the sharded collection, and +// the third shard has nothing. Progressively shuts down the primary of each shard to see the +// impact on the cluster. +// +// Three different connection states are tested - active (connection is active through whole +// sequence), idle (connection is connected but not used before a shard change), and new +// (connection connected after shard change). +// + +var options = {separateConfig : true, + rs : true, + rsOptions : { nodes : 2 }}; + +var st = new ShardingTest({shards : 3, mongos : 1, other : options}); +st.stopBalancer(); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getDB( "config" ).shards.find().toArray(); + +assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, authOnPrimaryOnly : false }) ); + +var collSharded = mongos.getCollection( "fooSharded.barSharded" ); +var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" ); + +// Create the unsharded database +collUnsharded.insert({ some : "doc" }); +assert.eq( null, collUnsharded.getDB().getLastError() ); +collUnsharded.remove({}); +assert.eq( null, collUnsharded.getDB().getLastError() ); +printjson( admin.runCommand({ movePrimary : collUnsharded.getDB().toString(), + to : shards[0]._id }) ); + +// Create the sharded database +assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB().toString() }) ); +printjson( admin.runCommand({ movePrimary : collSharded.getDB().toString(), to : shards[0]._id }) ); +assert.commandWorked( admin.runCommand({ shardCollection : collSharded.toString(), + key : { _id : 1 } }) ); +assert.commandWorked( admin.runCommand({ split : collSharded.toString(), middle : { _id : 0 } }) ); +assert.commandWorked( admin.runCommand({ moveChunk : collSharded.toString(), + find : { _id : 0 }, + to : shards[1]._id }) ); + +st.printShardingStatus(); + +// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we +// don't care if this happens in this test, we only care that we did not get "write succeeded". +// Depending on the connection pool state, we could get either. +function gleErrorOrThrow(database, msg) { + var gle; + try { + gle = database.getLastErrorObj(); + } + catch (ex) { + return; + } + if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg); + return; +}; + +// +// Setup is complete +// + +jsTest.log("Inserting initial data..."); + +var mongosConnActive = new Mongo( mongos.host ); +var mongosConnIdle = null; +var mongosConnNew = null; + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -1 }); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collSharded.toString() ).getDB().getLastError()); + +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collUnsharded.toString() ).getDB().getLastError()); + +jsTest.log("Stopping primary of third shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +st.rs2.stop(st.rs2.getPrimary(), true /*wait for stop*/ ); + +jsTest.log("Testing active connection with third primary down..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with third primary down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with third primary down..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping primary of second shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +// Need to save this node for later +var rs1Secondary = st.rs1.getSecondary(); + +st.rs1.stop(st.rs1.getPrimary(), true /* wait for stop */); + +jsTest.log("Testing active connection with second primary down..."); + +// Reads with read prefs +mongosConnActive.setSlaveOk(); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); +mongosConnActive.setSlaveOk(false); + +mongosConnActive.setReadPref("primary"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.throws(function() { + mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +//Ensure read prefs override slaveOK +mongosConnActive.setSlaveOk(); +mongosConnActive.setReadPref("primary"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.throws(function() { + mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); +mongosConnActive.setSlaveOk(false); + +mongosConnActive.setReadPref("secondary"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.setReadPref("primaryPreferred"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.setReadPref("secondaryPreferred"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.setReadPref("nearest"); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +// Writes +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -5 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 5 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 5 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with second primary down..."); + +// Writes +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 6 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +// Reads with read prefs +mongosConnIdle.setSlaveOk(); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); +mongosConnIdle.setSlaveOk(false); + +mongosConnIdle.setReadPref("primary"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.throws(function() { + mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +// Ensure read prefs override slaveOK +mongosConnIdle.setSlaveOk(); +mongosConnIdle.setReadPref("primary"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.throws(function() { + mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); +mongosConnIdle.setSlaveOk(false); + +mongosConnIdle.setReadPref("secondary"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnIdle.setReadPref("primaryPreferred"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnIdle.setReadPref("secondaryPreferred"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnIdle.setReadPref("nearest"); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +jsTest.log("Testing new connections with second primary down..."); + +// Reads with read prefs +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primary"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primary"); +assert.throws(function() { + mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primary"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +// Ensure read prefs override slaveok +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +mongosConnNew.setReadPref("primary"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +mongosConnNew.setReadPref("primary"); +assert.throws(function() { + mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }); +}); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +mongosConnNew.setReadPref("primary"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondary"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondary"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondary"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("primaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("secondaryPreferred"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("nearest"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("nearest"); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setReadPref("nearest"); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +// Writes +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -7 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 7 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 7 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping primary of first shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +st.rs0.stop(st.rs0.getPrimary(), true /*wait for stop*/ ); + +jsTest.log("Testing active connection with first primary down..."); + +mongosConnActive.setSlaveOk(); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 8 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with first primary down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 9 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +mongosConnIdle.setSlaveOk(); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with first primary down..."); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 10 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping second shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +st.rs1.stop(rs1Secondary, true /* wait for stop */); + +jsTest.log("Testing active connection with second shard down..."); + +mongosConnActive.setSlaveOk(); +assert.neq(null, mongosConnActive.getCollection( collSharded.toString() ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded.toString() ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : -11 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collSharded.toString() ).insert({ _id : 11 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded.toString() ).getDB()); +mongosConnActive.getCollection( collUnsharded.toString() ).insert({ _id : 11 }); +gleErrorOrThrow(mongosConnActive.getCollection( collUnsharded.toString() ).getDB()); + +jsTest.log("Testing idle connection with second shard down..."); + +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : -12 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collSharded.toString() ).insert({ _id : 12 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded.toString() ).getDB()); +mongosConnIdle.getCollection( collUnsharded.toString() ).insert({ _id : 12 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collUnsharded.toString() ).getDB()); + +mongosConnIdle.setSlaveOk(); +assert.neq(null, mongosConnIdle.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections with second shard down..."); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collSharded.toString() ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.setSlaveOk(); +assert.neq(null, mongosConnNew.getCollection( collUnsharded.toString() ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : -13 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded.toString() ).insert({ _id : 13 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded.toString() ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded.toString() ).insert({ _id : 13 }); +gleErrorOrThrow(mongosConnNew.getCollection( collUnsharded.toString() ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("DONE!"); +st.stop(); diff --git a/jstests/sharding/mongos_shard_failure_tolerance.js b/jstests/sharding/mongos_shard_failure_tolerance.js new file mode 100644 index 00000000000..9f5ffefca28 --- /dev/null +++ b/jstests/sharding/mongos_shard_failure_tolerance.js @@ -0,0 +1,186 @@ +// +// Tests mongos's failure tolerance for single-node shards +// +// +// Sets up a cluster with three shards, the first shard of which has an unsharded collection and +// half a sharded collection. The second shard has the second half of the sharded collection, and +// the third shard has nothing. Progressively shuts down each shard to see the impact on the +// cluster. +// +// Three different connection states are tested - active (connection is active through whole +// sequence), idle (connection is connected but not used before a shard change), and new +// (connection connected after shard change). +// + +var options = {separateConfig : true}; +var st = new ShardingTest({shards : 3, mongos : 1, other : options}); +st.stopBalancer(); + +var mongos = st.s0; +var admin = mongos.getDB( "admin" ); +var shards = mongos.getDB( "config" ).shards.find().toArray(); + +assert.commandWorked( admin.runCommand({ setParameter : 1, traceExceptions : true }) ); +assert.commandWorked( admin.runCommand({ setParameter : 1, ignoreInitialVersionFailure : true }) ); + +var collSharded = mongos.getCollection( "fooSharded.barSharded" ); +var collUnsharded = mongos.getCollection( "fooUnsharded.barUnsharded" ); + +assert.commandWorked( admin.runCommand({ enableSharding : collSharded.getDB() + "" }) ); +printjson( admin.runCommand({ movePrimary : collSharded.getDB() + "", to : shards[0]._id }) ); +assert.commandWorked( admin.runCommand({ shardCollection : collSharded + "", key : { _id : 1 } }) ); +assert.commandWorked( admin.runCommand({ split : collSharded + "", middle : { _id : 0 } }) ); +assert.commandWorked( admin.runCommand({ moveChunk : collSharded + "", + find : { _id : 0 }, + to : shards[1]._id }) ); + +// Create the unsharded database +collUnsharded.insert({ some : "doc" }); +assert.eq( null, collUnsharded.getDB().getLastError() ); +collUnsharded.remove({}); +assert.eq( null, collUnsharded.getDB().getLastError() ); +printjson( admin.runCommand({ movePrimary : collUnsharded.getDB() + "", to : shards[0]._id }) ); + +st.printShardingStatus(); + +// Needed b/c the GLE command itself can fail if the shard is down ("write result unknown") - we +// don't care if this happens in this test, we only care that we did not get "write succeeded". +// Depending on the connection pool state, we could get either. +function gleErrorOrThrow(database, msg) { + var gle; + try { + gle = database.getLastErrorObj(); + } + catch (ex) { + return; + } + if (!gle.err) doassert("getLastError is null: " + tojson(gle) + " :" + msg); + return; +}; + +// +// Setup is complete +// + +jsTest.log("Inserting initial data..."); + +var mongosConnActive = new Mongo( mongos.host ); +var mongosConnIdle = null; +var mongosConnNew = null; + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -1 }); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collSharded + "" ).getDB().getLastError()); + +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 1 }); +assert.eq(null, mongosConnActive.getCollection( collUnsharded + "" ).getDB().getLastError()); + +jsTest.log("Stopping third shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +MongoRunner.stopMongod( st.shard2 ); + +jsTest.log("Testing active connection..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 })); +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : 1 })); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 })); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 2 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 3 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : 1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 4 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("Stopping second shard..."); + +mongosConnIdle = new Mongo( mongos.host ); + +MongoRunner.stopMongod( st.shard1 ); + +jsTest.log("Testing active connection..."); + +assert.neq(null, mongosConnActive.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnActive.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : -5 }); +assert.gleSuccess(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collSharded + "" ).insert({ _id : 5 }); +gleErrorOrThrow(mongosConnActive.getCollection( collSharded + "" ).getDB()); +mongosConnActive.getCollection( collUnsharded + "" ).insert({ _id : 5 }); +assert.gleSuccess(mongosConnActive.getCollection( collUnsharded + "" ).getDB()); + +jsTest.log("Testing idle connection..."); + +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : -6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collSharded + "" ).insert({ _id : 6 }); +gleErrorOrThrow(mongosConnIdle.getCollection( collSharded + "" ).getDB()); +mongosConnIdle.getCollection( collUnsharded + "" ).insert({ _id : 6 }); +assert.gleSuccess(mongosConnIdle.getCollection( collUnsharded + "" ).getDB()); + +assert.neq(null, mongosConnIdle.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +assert.neq(null, mongosConnIdle.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +jsTest.log("Testing new connections..."); + +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collSharded + "" ).findOne({ _id : -1 }) ); +mongosConnNew = new Mongo( mongos.host ); +assert.neq(null, mongosConnNew.getCollection( collUnsharded + "" ).findOne({ _id : 1 }) ); + +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : -7 }); +assert.gleSuccess(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collSharded + "" ).insert({ _id : 7 }); +gleErrorOrThrow(mongosConnNew.getCollection( collSharded + "" ).getDB()); +mongosConnNew = new Mongo( mongos.host ); +mongosConnNew.getCollection( collUnsharded + "" ).insert({ _id : 7 }); +assert.gleSuccess(mongosConnNew.getCollection( collUnsharded + "" ).getDB()); + +gc(); // Clean up new connections + +jsTest.log("DONE!"); +st.stop(); + + + + + diff --git a/jstests/slowNightly/sharding_passthrough.js b/jstests/slowNightly/sharding_passthrough.js index 56f60d23a51..67db88b7b0d 100644 --- a/jstests/slowNightly/sharding_passthrough.js +++ b/jstests/slowNightly/sharding_passthrough.js @@ -70,6 +70,7 @@ files.forEach(function(x) { 'copydb-auth|' + 'profile\\d*|' + 'dbhash|' + + 'dbhash2|' + 'median|' + 'apitest_dbcollection|' + 'evalb|' + diff --git a/jstests/slowNightly/ttl1.js b/jstests/slowNightly/ttl1.js index f795d8bc0ff..50dae5670e3 100644 --- a/jstests/slowNightly/ttl1.js +++ b/jstests/slowNightly/ttl1.js @@ -1,21 +1,33 @@ /** * Part 1: Simple test of TTL. Create a new collection with 24 docs, with timestamps at one hour * intervals, from now-minus-23 hours ago until now. Also add some docs with non-date - * values. Then create a TTL index that expires all docs older than ~5.5 hours (20000 - * seconds). Wait 70 seconds (TTL monitor runs every 60) and check that 18 docs deleted. - * Part 2: Add a second TTL index on an identical field. The second index expires docs older than + * values. Then create a TTL index that expires all docs older than a string. Wait 70 + * seconds (TTL monitor runs every 60) and check that no documents were deleted. + * Part 2: Add a second TTL index that expires all docs older than ~5.5 hours (20000 + * seconds). Wait 70 seconds and check that 18 docs deleted. + * Part 3: Add a third TTL index on an identical field. The second index expires docs older than * ~2.8 hours (10000 seconds). Wait 70 seconds and check that 3 more docs deleted. */ +assertEntryMatches = function(array, regex) { + var found = false; + for (i=0; i<array.length; i++) { + if (regex.test(array[i])) { + found = true; + } + } + assert(found, + "The regex: " + regex + " did not match any entries in the array: " + array.join('\n')); +} // Part 1 var t = db.ttl1; t.drop(); var now = (new Date()).getTime(); -for ( i=0; i<24; i++ ){ - var past = new Date( now - ( 3600 * 1000 * i ) ); - t.insert( { x : past , y : past } ); +for (i=0; i<24; i++) { + var past = new Date(now - (3600 * 1000 * i)); + t.insert({x: past, y: past, z: past}); } t.insert( { a : 1 } ) //no x value t.insert( { x: null } ) //non-date value @@ -27,6 +39,18 @@ db.getLastError(); assert.eq( 30 , t.count() ); +t.ensureIndex( { z : 1 } , { expireAfterSeconds : "20000" } ); + +sleep(70 * 1000); + +assert.eq(t.count(), 30); + +var loggedWarning = false; +var log = db.adminCommand({getLog: "global"}).log; +var msg = RegExp("ttl indexes require the expireAfterSeconds" + + " field to be numeric but received a type of:"); +assertEntryMatches(log, msg); +// Part 2 t.ensureIndex( { x : 1 } , { expireAfterSeconds : 20000 } ); assert.soon( @@ -41,7 +65,7 @@ assert.eq( 12 , t.count() ); assert.lte( 18, db.serverStatus().metrics.ttl.deletedDocuments ); assert.lte( 1, db.serverStatus().metrics.ttl.passes ); -// Part 2 +// Part 3 t.ensureIndex( { y : 1 } , { expireAfterSeconds : 10000 } ); assert.soon( diff --git a/rpm/mongo.spec b/rpm/mongo.spec index 4b8185a0055..d8a9a121bed 100755 --- a/rpm/mongo.spec +++ b/rpm/mongo.spec @@ -1,7 +1,7 @@ Name: mongo-10gen Conflicts: mongo, mongo-10gen-unstable Obsoletes: mongo-stable -Version: 2.4.6 +Version: 2.4.8 Release: mongodb_1%{?dist} Summary: mongo client shell and tools License: AGPL 3.0 diff --git a/src/mongo/SConscript b/src/mongo/SConscript index 816a6bfacf8..265cd4800e0 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -433,6 +433,7 @@ serverOnlyFiles = [ "db/curop.cpp", "db/dbcommands_admin.cpp", # most commands are only for mongod + "db/commands/dbhash.cpp", "db/commands/fsync.cpp", "db/commands/distinct.cpp", "db/commands/find_and_modify.cpp", diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err index 1dfa1702b64..5fa8cde17e2 100644 --- a/src/mongo/base/error_codes.err +++ b/src/mongo/base/error_codes.err @@ -27,4 +27,7 @@ error_code("AlreadyInitialized", 23) error_code("LockTimeout", 24) error_code("RemoteValidationError", 25) +# Non-sequential error codes (for compatibility only) +error_code("NodeNotFound", 74) + error_class("NetworkError", ["HostUnreachable", "HostNotFound"]) diff --git a/src/mongo/client/dbclient_rs.cpp b/src/mongo/client/dbclient_rs.cpp index 21a5b0810f4..cc82c2aa948 100644 --- a/src/mongo/client/dbclient_rs.cpp +++ b/src/mongo/client/dbclient_rs.cpp @@ -126,54 +126,6 @@ namespace mongo { } _populateReadPrefSecOkCmdList; /** - * @param ns the namespace of the query. - * @param queryOptionFlags the flags for the query. - * @param queryObj the query object to check. - * - * @return true if the given query can be sent to a secondary node without taking the - * slaveOk flag into account. - */ - bool _isQueryOkToSecondary(const string& ns, int queryOptionFlags, const BSONObj& queryObj) { - if (queryOptionFlags & QueryOption_SlaveOk) { - return true; - } - - if (!Query::hasReadPreference(queryObj)) { - return false; - } - - if (ns.find(".$cmd") == string::npos) { - return true; - } - - BSONObj actualQueryObj; - if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { - actualQueryObj = queryObj["query"].embeddedObject(); - } - else { - actualQueryObj = queryObj; - } - - const string cmdName = actualQueryObj.firstElementFieldName(); - if (_secOkCmdList.count(cmdName) == 1) { - return true; - } - - if (cmdName == "mapReduce" || cmdName == "mapreduce") { - if (!actualQueryObj.hasField("out")) { - return false; - } - - BSONElement outElem(actualQueryObj["out"]); - if (outElem.isABSONObj() && outElem["inline"].trueValue()) { - return true; - } - } - - return false; - } - - /** * Selects the right node given the nodes to pick from and the preference. * This method does strict tag matching, and will not implicitly fallback * to matching anything. @@ -275,15 +227,18 @@ namespace mongo { * * @param query the raw query document * - * @return the read preference setting. If the tags field was not present, it will contain one - * empty tag document {} which matches any tag. + * @return the read preference setting if a read preference exists, otherwise the default read + * preference of Primary_Only. If the tags field was not present, it will contain one + * empty tag document {} which matches any tag. * * @throws AssertionException if the read preference object is malformed */ - ReadPreferenceSetting* _extractReadPref(const BSONObj& query) { - ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; + ReadPreferenceSetting* _extractReadPref(const BSONObj& query, int queryOptions) { if (Query::hasReadPreference(query)) { + + ReadPreference pref = mongo::ReadPreference_SecondaryPreferred; + BSONElement readPrefElement; if (query.hasField(Query::ReadPrefField.name())) { @@ -334,9 +289,17 @@ namespace mongo { return new ReadPreferenceSetting(pref, tags); } + else { + TagSet tags(BSON_ARRAY(BSONObj())); + return new ReadPreferenceSetting(pref, tags); + } } + // Default read pref is primary only or secondary preferred with slaveOK TagSet tags(BSON_ARRAY(BSONObj())); + ReadPreference pref = + queryOptions & QueryOption_SlaveOk ? + mongo::ReadPreference_SecondaryPreferred : mongo::ReadPreference_PrimaryOnly; return new ReadPreferenceSetting(pref, tags); } @@ -405,7 +368,6 @@ namespace mongo { // delete ReplicaSetMonitors from ReplicaSetMonitor::remove. ReplicaSetMonitor::~ReplicaSetMonitor() { scoped_lock lk ( _lock ); - log() << "deleting replica set monitor for: " << _getServerAddress_inlock() << endl; _cacheServerAddresses_inlock(); pool.removeHost( _getServerAddress_inlock() ); _nodes.clear(); @@ -1501,6 +1463,55 @@ namespace mongo { return rsm->getServerAddress(); } + // Internal implementation of isSecondaryQuery, takes previously-parsed read preference + static bool _isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + const ReadPreferenceSetting& readPref ) { + + // If the read pref is primary only, this is not a secondary query + if (readPref.pref == ReadPreference_PrimaryOnly) return false; + + if (ns.find(".$cmd") == string::npos) { + return true; + } + + // This is a command with secondary-possible read pref + // Only certain commands are supported for secondary operation. + + BSONObj actualQueryObj; + if (strcmp(queryObj.firstElement().fieldName(), "query") == 0) { + actualQueryObj = queryObj["query"].embeddedObject(); + } + else { + actualQueryObj = queryObj; + } + + const string cmdName = actualQueryObj.firstElementFieldName(); + if (_secOkCmdList.count(cmdName) == 1) { + return true; + } + + if (cmdName == "mapReduce" || cmdName == "mapreduce") { + if (!actualQueryObj.hasField("out")) { + return false; + } + + BSONElement outElem(actualQueryObj["out"]); + if (outElem.isABSONObj() && outElem["inline"].trueValue()) { + return true; + } + } + + return false; + } + + bool DBClientReplicaSet::isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ) { + auto_ptr<ReadPreferenceSetting> readPref( _extractReadPref( queryObj, queryOptions ) ); + return _isSecondaryQuery( ns, queryObj, *readPref ); + } + DBClientConnection * DBClientReplicaSet::checkMaster() { ReplicaSetMonitorPtr monitor = _getMonitor(); HostAndPort h = monitor->getMaster(); @@ -1593,6 +1604,18 @@ namespace mongo { return _getMonitor()->isAnyNodeOk(); } + void DBClientReplicaSet::authPrimary(const BSONObj& params) { + _auth(params); + } + + bool DBClientReplicaSet::authPrimary( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + return auth( dbname, username, password_text, errmsg, digestPassword ); + } + void DBClientReplicaSet::_auth(const BSONObj& params) { DBClientConnection * m = checkMaster(); @@ -1620,6 +1643,98 @@ namespace mongo { _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); } + bool DBClientReplicaSet::authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ) { + try { + authAny(BSON(saslCommandMechanismFieldName << "MONGODB-CR" << + saslCommandPrincipalSourceFieldName << dbname << + saslCommandPrincipalFieldName << username << + saslCommandPasswordFieldName << password_text << + saslCommandDigestPasswordFieldName << digestPassword)); + return true; + } catch(const UserException& ex) { + if (ex.getCode() != ErrorCodes::AuthenticationFailed) + throw; + errmsg = ex.what(); + return false; + } + } + + static bool isAuthenticationException( const DBException& ex ) { + return ex.getCode() == ErrorCodes::AuthenticationFailed; + } + + void DBClientReplicaSet::authAny( const BSONObj& params ) { + + // We prefer to authenticate against a primary, but otherwise a secondary is ok too + // Empty tag matches every secondary + TagSet tags(BSON_ARRAY(BSONObj())); + shared_ptr<ReadPreferenceSetting> readPref( + new ReadPreferenceSetting( ReadPreference_PrimaryPreferred, tags ) ); + + LOG(3) << "dbclient_rs authentication of " << _getMonitor()->getName() << endl; + + // NOTE that we retry MAX_RETRY + 1 times, since we're always primary preferred we don't + // fallback to the primary. + Status lastNodeStatus = Status::OK(); + for ( size_t retry = 0; retry < MAX_RETRY + 1; retry++ ) { + try { + DBClientConnection* conn = selectNodeUsingTags( readPref ); + + if ( conn == NULL ) { + break; + } + + conn->auth( params ); + + // Cache the new auth information since we've now validated it's good + _auths[params[saslCommandPrincipalSourceFieldName].str()] = params.getOwned(); + + // Ensure the only child connection open is the one we authenticated against - other + // child connections may not have full authentication information. + // NOTE: _lastSlaveOkConn may or may not be the same as _master + dassert(_lastSlaveOkConn.get() == conn || _master.get() == conn); + if ( conn != _lastSlaveOkConn.get() ) { + _lastSlaveOkHost = HostAndPort(); + _lastSlaveOkConn.reset(); + } + if ( conn != _master.get() ) { + _masterHost = HostAndPort(); + _master.reset(); + } + + return; + } + catch ( const DBException &ex ) { + + // We care if we can't authenticate (i.e. bad password) in credential params. + if ( isAuthenticationException( ex ) ) { + throw; + } + + StringBuilder errMsgB; + errMsgB << "can't authenticate against replica set node " + << _lastSlaveOkHost.toString(); + lastNodeStatus = ex.toStatus( errMsgB.str() ); + + LOG(1) << lastNodeStatus.reason() << endl; + invalidateLastSlaveOkCache(); + } + } + + if ( lastNodeStatus.isOK() ) { + StringBuilder assertMsgB; + assertMsgB << "Failed to authenticate, no good nodes in " << _getMonitor()->getName(); + uasserted( ErrorCodes::NodeNotFound, assertMsgB.str() ); + } + else { + uasserted( lastNodeStatus.code(), lastNodeStatus.reason() ); + } + } + void DBClientReplicaSet::logout(const string &dbname, BSONObj& info) { DBClientConnection* priConn = checkMaster(); @@ -1668,9 +1783,8 @@ namespace mongo { int queryOptions, int batchSize) { - if ( _isQueryOkToSecondary( ns, queryOptions, query.obj ) ) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs query using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1718,9 +1832,9 @@ namespace mongo { const Query& query, const BSONObj *fieldsToReturn, int queryOptions) { - if (_isQueryOkToSecondary(ns, queryOptions, query.obj)) { - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(query.obj)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( query.obj, queryOptions ) ); + if ( _isSecondaryQuery( ns, query.obj, *readPref ) ) { LOG( 3 ) << "dbclient_rs findOne using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1872,17 +1986,15 @@ namespace mongo { _lazyState = LazyState(); const int lastOp = toSend.operation(); - bool slaveOk = false; if (lastOp == dbQuery) { // TODO: might be possible to do this faster by changing api DbMessage dm(toSend); QueryMessage qm(dm); - const bool slaveOk = qm.queryOptions & QueryOption_SlaveOk; - if (_isQueryOkToSecondary(qm.ns, qm.queryOptions, qm.query)) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( qm.ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs say using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " @@ -1910,7 +2022,7 @@ namespace mongo { conn->say(toSend); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._isSecondaryQuery = true; _lazyState._lastClient = conn; } catch ( const DBException& DBExcep ) { @@ -1936,7 +2048,7 @@ namespace mongo { *actualServer = master->getServerAddress(); _lazyState._lastOp = lastOp; - _lazyState._slaveOk = slaveOk; + _lazyState._isSecondaryQuery = false; // Don't retry requests to primary since there is only one host to try _lazyState._retries = MAX_RETRY; _lazyState._lastClient = master; @@ -1981,7 +2093,7 @@ namespace mongo { if( nReturned == 1 ) dataObj = BSONObj( data ); // Check if we should retry here - if( _lazyState._lastOp == dbQuery && _lazyState._slaveOk ){ + if( _lazyState._lastOp == dbQuery && _lazyState._isSecondaryQuery ){ // Check the error code for a slave not secondary error if( nReturned == -1 || @@ -2036,9 +2148,9 @@ namespace mongo { QueryMessage qm(dm); ns = qm.ns; - if (_isQueryOkToSecondary(ns, qm.queryOptions, qm.query)) { - - shared_ptr<ReadPreferenceSetting> readPref(_extractReadPref(qm.query)); + shared_ptr<ReadPreferenceSetting> readPref( _extractReadPref( qm.query, + qm.queryOptions ) ); + if ( _isSecondaryQuery( ns, qm.query, *readPref ) ) { LOG( 3 ) << "dbclient_rs call using secondary or tagged node selection in " << _getMonitor()->getName() << ", read pref is " diff --git a/src/mongo/client/dbclient_rs.h b/src/mongo/client/dbclient_rs.h index 05f7faa9b2b..d4c9512576f 100644 --- a/src/mongo/client/dbclient_rs.h +++ b/src/mongo/client/dbclient_rs.h @@ -518,10 +518,65 @@ namespace mongo { virtual bool call( Message &toSend, Message &response, bool assertOk=true , string * actualServer = 0 ); virtual bool callRead( Message& toSend , Message& response ) { return checkMaster()->callRead( toSend , response ); } + /** + * Authenticate using supplied credentials. Authenticates against the primary node, fails + * if node is down. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + * + * This is the default authentication mode for DBClientReplicaSet connections. + */ + void authPrimary(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authPrimary(const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword); + + /** + * Authenticate using supplied credentials. Prefers authentication against the primary + * node, will fall back to a secondary and retry if the primary node is down but + * secondaries are still available. + * Credentials are cached for future connections. + * + * See DBClientWithCommands::auth() for more details. + */ + void authAny(const BSONObj& params); + + /** + * Same as above, but authorizes access to a particular database. + * + * See DBClientWithCommands::auth() for more details. + */ + bool authAny( const string &dbname, + const string &username, + const string &password_text, + string& errmsg, + bool digestPassword ); + + /* + * Returns whether a query or command can be sent to secondaries based on the query object + * and options. + * + * @param ns the namespace of the query. + * @param queryObj the query object to check. + * @param queryOptions the query options + * + * @return true if the query/cmd could potentially be sent to a secondary, false otherwise + */ + static bool isSecondaryQuery( const string& ns, + const BSONObj& queryObj, + int queryOptions ); protected: - /** Authorize. Authorizes all nodes as needed - */ + virtual void _auth(const BSONObj& params); virtual void sayPiggyBack( Message &toSend ) { checkMaster()->say( toSend ); } @@ -606,10 +661,12 @@ namespace mongo { */ class LazyState { public: - LazyState() : _lastClient( NULL ), _lastOp( -1 ), _slaveOk( false ), _retries( 0 ) {} + LazyState() : + _lastClient( NULL ), _lastOp( -1 ), _isSecondaryQuery( false ), _retries( 0 ) { + } DBClientConnection* _lastClient; int _lastOp; - bool _slaveOk; + bool _isSecondaryQuery; int _retries; } _lazyState; diff --git a/src/mongo/client/parallel.cpp b/src/mongo/client/parallel.cpp index 18649d9c6b9..7dcc7573df0 100644 --- a/src/mongo/client/parallel.cpp +++ b/src/mongo/client/parallel.cpp @@ -715,26 +715,28 @@ namespace mongo { } const DBClientBase* rawConn = state->conn->getRawConn(); - if (( _options & QueryOption_SlaveOk ) && - rawConn->type() == ConnectionString::SET && - rawConn->isFailed() ) { - /* A side effect of this short circuiting is this will not be - * able figure out that the primary is now up on it's own and - * has to rely on other threads to refresh the node states. - */ + bool allowShardVersionFailure = + rawConn->type() == ConnectionString::SET && + DBClientReplicaSet::isSecondaryQuery( _qSpec.ns(), _qSpec.query(), _qSpec.options() ); + + if ( allowShardVersionFailure && rawConn->isFailed() ) { + + state->conn->donotCheckVersion(); + + // A side effect of this short circuiting is the mongos will not be able figure out that + // the primary is now up on it's own and has to rely on other threads to refresh node + // states. OCCASIONALLY { - const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( rawConn ); + const DBClientReplicaSet* repl = dynamic_cast<const DBClientReplicaSet*>( rawConn ); + dassert(repl); warning() << "Primary for " << repl->getServerAddress() << " was down before, bypassing setShardVersion." - << " Local config view can be stale." << endl; + << " The local replica set view and targeting may be stale." << endl; } - } else { + } + else { try { - /* TODO: Undo SERVER-5797. This try-catch is a temporary hack until - * secondaries can properly handle shard versioning - */ if ( state->conn->setVersion() ) { // It's actually okay if we set the version here, since either the // manager will be verified as compatible, or if the manager doesn't @@ -742,19 +744,20 @@ namespace mongo { LOG( pc ) << "needed to set remote version on connection to value " << "compatible with " << vinfo << endl; } - } catch ( const DBException& dbEx ) { - if ( (dbEx.getCode() == 10009 /* no master */ && - ( _options & QueryOption_SlaveOk )) ) { + } + catch ( const DBException& dbEx ) { + if ( allowShardVersionFailure ) { + + // It's okay if we don't set the version when talking to a secondary, we can + // be stale in any case. OCCASIONALLY { const DBClientReplicaSet* repl = - dynamic_cast<const DBClientReplicaSet*>( - state->conn->getRawConn() ); - - warning() << "Cannot contact primary for " - << repl->getServerAddress() - << " to check shard version. " - << "SlaveOk query can be sent to the wrong shard." + dynamic_cast<const DBClientReplicaSet*>( state->conn->getRawConn() ); + dassert(repl); + warning() << "Cannot contact primary for " << repl->getServerAddress() + << " to check shard version." + << " The local replica set view and targeting may be stale." << endl; } } diff --git a/src/mongo/db/auth/auth_external_state_s.cpp b/src/mongo/db/auth/auth_external_state_s.cpp index b7167cef509..7e890636cdf 100644 --- a/src/mongo/db/auth/auth_external_state_s.cpp +++ b/src/mongo/db/auth/auth_external_state_s.cpp @@ -49,9 +49,11 @@ namespace mongo { } bool AuthExternalStateMongos::_findUser(const string& usersNamespace, - const BSONObj& query, + const BSONObj& queryDoc, BSONObj* result) const { scoped_ptr<ScopedDbConnection> conn(getConnectionForUsersCollection(usersNamespace)); + Query query(queryDoc); + query.readPref(ReadPreference_PrimaryPreferred, BSONArray()); *result = conn->get()->findOne(usersNamespace, query).getOwned(); conn->done(); return !result->isEmpty(); diff --git a/src/mongo/db/auth/authorization_manager.cpp b/src/mongo/db/auth/authorization_manager.cpp index 1cf8efede39..f30443eda1b 100644 --- a/src/mongo/db/auth/authorization_manager.cpp +++ b/src/mongo/db/auth/authorization_manager.cpp @@ -443,6 +443,18 @@ namespace { return _authenticatedPrincipals.getNames(); } + std::string AuthorizationManager::getAuthenticatedPrincipalNamesToken() { + std::string ret; + for (PrincipalSet::NameIterator nameIter = getAuthenticatedPrincipalNames(); + nameIter.more(); + nameIter.next()) { + ret += '\0'; // Using a NUL byte which isn't valid in usernames to separate them. + ret += nameIter->getFullName(); + } + + return ret; + } + Status AuthorizationManager::acquirePrivilege(const Privilege& privilege, const PrincipalName& authorizingPrincipal) { if (!_authenticatedPrincipals.lookup(authorizingPrincipal)) { diff --git a/src/mongo/db/auth/authorization_manager.h b/src/mongo/db/auth/authorization_manager.h index a32710557dd..7131d7624a6 100644 --- a/src/mongo/db/auth/authorization_manager.h +++ b/src/mongo/db/auth/authorization_manager.h @@ -91,6 +91,10 @@ namespace mongo { // Gets an iterator over the names of all authenticated principals stored in this manager. PrincipalSet::NameIterator getAuthenticatedPrincipalNames(); + // Returns a string representing all logged-in principals on the current session. + // WARNING: this string will contain NUL bytes so don't call c_str()! + std::string getAuthenticatedPrincipalNamesToken(); + // Removes any authenticated principals whose authorization credentials came from the given // database, and revokes any privileges that were granted via that principal. void logoutDatabase(const std::string& dbname); diff --git a/src/mongo/db/cmdline.cpp b/src/mongo/db/cmdline.cpp index 0dea50faeef..c14cc603b56 100644 --- a/src/mongo/db/cmdline.cpp +++ b/src/mongo/db/cmdline.cpp @@ -151,9 +151,14 @@ namespace { if ( s.find( "FASTSYNC" ) != string::npos ) cout << "warning \"fastsync\" should not be put in your configuration file" << endl; - if ( s.c_str()[0] == '#' ) { - // skipping commented line - } else if ( s.find( "=FALSE" ) == string::npos ) { + // skip commented lines + if ( s.c_str()[0] == '#' ) { + // In this block, we copy the actual line into our intermediate buffer to actually be + // parsed later only if the string does not contain the substring "=FALSE" OR the option + // is a setParameter option. Note that this is done after we call boost::to_upper + // above. + } else if ( s.find( "=FALSE" ) == string::npos || + s.find( "SETPARAMETER" ) == 0 ) { ss << line << endl; } else { cout << "warning: remove or comment out this line by starting it with \'#\', skipping now : " << line << endl; diff --git a/src/mongo/db/commands/dbhash.cpp b/src/mongo/db/commands/dbhash.cpp new file mode 100644 index 00000000000..38a07cace56 --- /dev/null +++ b/src/mongo/db/commands/dbhash.cpp @@ -0,0 +1,216 @@ +// dbhash.cpp + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#include "mongo/db/commands/dbhash.h" + +#include "mongo/db/btreecursor.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/database.h" +#include "mongo/db/pdfile.h" +#include "mongo/util/md5.hpp" +#include "mongo/util/timer.h" + +namespace mongo { + + DBHashCmd dbhashCmd; + + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt ) { + dbhashCmd.wipeCacheForCollection( ns ); + } + + // ---- + + DBHashCmd::DBHashCmd() + : Command( "dbHash", false, "dbhash" ), + _cachedHashedMutex( "_cachedHashedMutex" ){ + } + + void DBHashCmd::addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector<Privilege>* out) { + ActionSet actions; + actions.addAction(ActionType::dbHash); + out->push_back(Privilege(dbname, actions)); + } + + string DBHashCmd::hashCollection( const string& fullCollectionName, bool* fromCache ) { + + scoped_ptr<scoped_lock> cachedHashedLock; + + if ( isCachable( fullCollectionName ) ) { + cachedHashedLock.reset( new scoped_lock( _cachedHashedMutex ) ); + string hash = _cachedHashed[fullCollectionName]; + if ( hash.size() > 0 ) { + *fromCache = true; + return hash; + } + } + + *fromCache = false; + NamespaceDetails * nsd = nsdetails( fullCollectionName ); + verify( nsd ); + + // debug SERVER-761 + NamespaceDetails::IndexIterator ii = nsd->ii(); + while( ii.more() ) { + const IndexDetails &idx = ii.next(); + if ( !idx.head.isValid() || !idx.info.isValid() ) { + log() << "invalid index for ns: " << fullCollectionName << " " << idx.head << " " << idx.info; + if ( idx.info.isValid() ) + log() << " " << idx.info.obj(); + log() << endl; + } + } + + int idNum = nsd->findIdIndex(); + + shared_ptr<Cursor> cursor; + + if ( idNum >= 0 ) { + cursor.reset( BtreeCursor::make( nsd, + nsd->idx( idNum ), + BSONObj(), + BSONObj(), + false, + 1 ) ); + } + else if ( nsd->isCapped() ) { + cursor = findTableScan( fullCollectionName.c_str() , BSONObj() ); + } + else { + log() << "can't find _id index for: " << fullCollectionName << endl; + return "no _id _index"; + } + + md5_state_t st; + md5_init(&st); + + long long n = 0; + + while ( cursor->ok() ) { + BSONObj c = cursor->current(); + md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); + n++; + cursor->advance(); + } + + md5digest d; + md5_finish(&st, d); + string hash = digestToString( d ); + + if ( cachedHashedLock.get() ) { + _cachedHashed[fullCollectionName] = hash; + } + + return hash; + } + + bool DBHashCmd::run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { + Timer timer; + + set<string> desiredCollections; + if ( cmdObj["collections"].type() == Array ) { + BSONObjIterator i( cmdObj["collections"].Obj() ); + while ( i.more() ) { + BSONElement e = i.next(); + if ( e.type() != String ) { + errmsg = "collections entries have to be strings"; + return false; + } + desiredCollections.insert( e.String() ); + } + } + + list<string> colls; + Database* db = cc().database(); + if ( db ) + db->namespaceIndex.getNamespaces( colls ); + colls.sort(); + + result.appendNumber( "numCollections" , (long long)colls.size() ); + result.append( "host" , prettyHostName() ); + + md5_state_t globalState; + md5_init(&globalState); + + vector<string> cached; + + BSONObjBuilder bb( result.subobjStart( "collections" ) ); + for ( list<string>::iterator i=colls.begin(); i != colls.end(); i++ ) { + string fullCollectionName = *i; + string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); + + if ( shortCollectionName.find( "system." ) == 0 ) + continue; + + if ( desiredCollections.size() > 0 && + desiredCollections.count( shortCollectionName ) == 0 ) + continue; + + bool fromCache = false; + string hash = hashCollection( fullCollectionName, &fromCache ); + + bb.append( shortCollectionName, hash ); + + md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); + if ( fromCache ) + cached.push_back( fullCollectionName ); + } + bb.done(); + + md5digest d; + md5_finish(&globalState, d); + string hash = digestToString( d ); + + result.append( "md5" , hash ); + result.appendNumber( "timeMillis", timer.millis() ); + + result.append( "fromCache", cached ); + + return 1; + } + + void DBHashCmd::wipeCacheForCollection( const StringData& ns ) { + if ( !isCachable( ns ) ) + return; + scoped_lock lk( _cachedHashedMutex ); + _cachedHashed.erase( ns.toString() ); + } + + bool DBHashCmd::isCachable( const StringData& ns ) const { + return ns.startsWith( "config." ); + } + +} diff --git a/src/mongo/db/commands/dbhash.h b/src/mongo/db/commands/dbhash.h new file mode 100644 index 00000000000..262c6609868 --- /dev/null +++ b/src/mongo/db/commands/dbhash.h @@ -0,0 +1,67 @@ +// dbhash.h + +/** +* Copyright (C) 2013 10gen Inc. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License, version 3, +* as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see <http://www.gnu.org/licenses/>. +* +* As a special exception, the copyright holders give permission to link the +* code of portions of this program with the OpenSSL library under certain +* conditions as described in each individual source file and distribute +* linked combinations including the program with the OpenSSL library. You +* must comply with the GNU Affero General Public License in all respects for +* all of the code used other than as permitted herein. If you modify file(s) +* with this exception, you may extend this exception to your version of the +* file(s), but you are not obligated to do so. If you do not wish to do so, +* delete this exception statement from your version. If you delete this +* exception statement from all source files in the program, then also delete +* it in the license file. +*/ + +#pragma once + +#include "mongo/db/commands.h" + +namespace mongo { + + void logOpForDbHash( const char* opstr, + const char* ns, + const BSONObj& obj, + BSONObj* patt ); + + class DBHashCmd : public Command { + public: + DBHashCmd(); + + virtual bool slaveOk() const { return true; } + virtual LockType locktype() const { return READ; } + virtual void addRequiredPrivileges(const std::string& dbname, + const BSONObj& cmdObj, + std::vector<Privilege>* out); + + virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool); + + void wipeCacheForCollection( const StringData& ns ); + + private: + + bool isCachable( const StringData& ns ) const; + + string hashCollection( const string& fullCollectionName, bool* fromCache ); + + map<string,string> _cachedHashed; + mutex _cachedHashedMutex; + + }; + +} diff --git a/src/mongo/db/commands/group.cpp b/src/mongo/db/commands/group.cpp index 441a1192905..20481521e91 100644 --- a/src/mongo/db/commands/group.cpp +++ b/src/mongo/db/commands/group.cpp @@ -20,9 +20,11 @@ #include <vector> +#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/action_set.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" +#include "mongo/db/client_basic.h" #include "mongo/db/commands.h" #include "mongo/db/instance.h" #include "mongo/scripting/engine.h" @@ -72,7 +74,9 @@ namespace mongo { string& errmsg, BSONObjBuilder& result ) { - auto_ptr<Scope> s = globalScriptEngine->getPooledScope( realdbname, "group"); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + auto_ptr<Scope> s = globalScriptEngine->getPooledScope(realdbname, "group" + userToken); if ( reduceScope ) s->init( reduceScope ); diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp index 9528e495ded..1b237d3ed7f 100644 --- a/src/mongo/db/commands/mr.cpp +++ b/src/mongo/db/commands/mr.cpp @@ -22,6 +22,7 @@ #include "mongo/client/connpool.h" #include "mongo/client/parallel.h" +#include "mongo/db/auth/authorization_manager.h" #include "mongo/db/clientcursor.h" #include "mongo/db/commands.h" #include "mongo/db/db.h" @@ -300,8 +301,13 @@ namespace mongo { */ void State::dropTempCollections() { _db.dropCollection(_config.tempNamespace); - if (_useIncremental) + // Always forget about temporary namespaces, so we don't cache lots of them + ShardConnection::forgetNS( _config.tempNamespace ); + if (_useIncremental) { _db.dropCollection(_config.incLong); + ShardConnection::forgetNS( _config.incLong ); + } + } /** @@ -622,7 +628,10 @@ namespace mongo { */ void State::init() { // setup js - _scope.reset(globalScriptEngine->getPooledScope( _config.dbname, "mapreduce" ).release() ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + _scope.reset(globalScriptEngine->getPooledScope( + _config.dbname, "mapreduce" + userToken).release()); if ( ! _config.scopeSetup.isEmpty() ) _scope->init( &_config.scopeSetup ); @@ -1165,9 +1174,23 @@ namespace mongo { state.init(); state.prepTempCollection(); ON_BLOCK_EXIT_OBJ(state, &State::dropTempCollections); - ProgressMeterHolder pm(op->setMessage("m/r: (1/3) emit phase", - "M/R: (1/3) Emit Progress", - state.incomingDocuments())); + + int progressTotal = 0; + bool showTotal = true; + if ( state.config().filter.isEmpty() ) { + progressTotal = state.incomingDocuments(); + } + else { + showTotal = false; + // Set an arbitrary total > 0 so the meter will be activated. + progressTotal = 1; + } + + ProgressMeter& progress( op->setMessage("m/r: (1/3) emit phase", + "M/R: (1/3) Emit Progress", + progressTotal )); + progress.showTotal(showTotal); + ProgressMeterHolder pm(progress); wassert( config.limit < 0x4000000 ); // see case on next line to 32 bit unsigned long long mapTime = 0; @@ -1457,6 +1480,9 @@ namespace mongo { break; } + // Forget temporary input collection, if output is sharded collection + ShardConnection::forgetNS( inputNS ); + result.append( "chunkSizes" , chunkSizes.arr() ); long long outputCount = state.postProcessCollection(op, pm); diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2361ec2a945..ec337c89885 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -232,7 +232,6 @@ namespace mongo { virtual void disconnected( AbstractMessagingPort* p ) { Client * c = currentClient.get(); if( c ) c->shutdown(); - globalScriptEngine->threadDone(); } }; diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index 33b77f455d6..fb6be132171 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -178,9 +178,9 @@ namespace mongo { } } - if ( err ) { + if ( err && cmdObj["wOpTime"].eoo() ) { // doesn't make sense to wait for replication - // if there was an error + // if there was an error and we aren't explicitly waiting for another wOpTime return true; } @@ -200,7 +200,14 @@ namespace mongo { long long passes = 0; char buf[32]; - OpTime op(c.getLastOp()); + + OpTime op; + if ( cmdObj["wOpTime"].type() == Timestamp ) { + op = OpTime( cmdObj["wOpTime"].date() ); + } + else { + op = c.getLastOp(); + } if ( op.isNull() ) { if ( anyReplEnabled() ) { @@ -1689,122 +1696,6 @@ namespace mongo { } return Status::OK(); } - - class DBHashCmd : public Command { - public: - DBHashCmd() : Command( "dbHash", false, "dbhash" ) {} - virtual bool slaveOk() const { return true; } - virtual LockType locktype() const { return READ; } - virtual void addRequiredPrivileges(const std::string& dbname, - const BSONObj& cmdObj, - std::vector<Privilege>* out) { - ActionSet actions; - actions.addAction(ActionType::dbHash); - out->push_back(Privilege(dbname, actions)); - } - virtual bool run(const string& dbname , BSONObj& cmdObj, int, string& errmsg, BSONObjBuilder& result, bool) { - Timer timer; - - set<string> desiredCollections; - if ( cmdObj["collections"].type() == Array ) { - BSONObjIterator i( cmdObj["collections"].Obj() ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( e.type() != String ) { - errmsg = "collections entries have to be strings"; - return false; - } - desiredCollections.insert( e.String() ); - } - } - - list<string> colls; - Database* db = cc().database(); - if ( db ) - db->namespaceIndex.getNamespaces( colls ); - colls.sort(); - - result.appendNumber( "numCollections" , (long long)colls.size() ); - result.append( "host" , prettyHostName() ); - - md5_state_t globalState; - md5_init(&globalState); - - BSONObjBuilder bb( result.subobjStart( "collections" ) ); - for ( list<string>::iterator i=colls.begin(); i != colls.end(); i++ ) { - string fullCollectionName = *i; - string shortCollectionName = fullCollectionName.substr( dbname.size() + 1 ); - - if ( shortCollectionName.find( "system." ) == 0 ) - continue; - - if ( desiredCollections.size() > 0 && - desiredCollections.count( shortCollectionName ) == 0 ) - continue; - - shared_ptr<Cursor> cursor; - - NamespaceDetails * nsd = nsdetails( fullCollectionName ); - - // debug SERVER-761 - NamespaceDetails::IndexIterator ii = nsd->ii(); - while( ii.more() ) { - const IndexDetails &idx = ii.next(); - if ( !idx.head.isValid() || !idx.info.isValid() ) { - log() << "invalid index for ns: " << fullCollectionName << " " << idx.head << " " << idx.info; - if ( idx.info.isValid() ) - log() << " " << idx.info.obj(); - log() << endl; - } - } - - int idNum = nsd->findIdIndex(); - if ( idNum >= 0 ) { - cursor.reset( BtreeCursor::make( nsd, - nsd->idx( idNum ), - BSONObj(), - BSONObj(), - false, - 1 ) ); - } - else if ( nsd->isCapped() ) { - cursor = findTableScan( fullCollectionName.c_str() , BSONObj() ); - } - else { - log() << "can't find _id index for: " << fullCollectionName << endl; - continue; - } - - md5_state_t st; - md5_init(&st); - - long long n = 0; - while ( cursor->ok() ) { - BSONObj c = cursor->current(); - md5_append( &st , (const md5_byte_t*)c.objdata() , c.objsize() ); - n++; - cursor->advance(); - } - md5digest d; - md5_finish(&st, d); - string hash = digestToString( d ); - - bb.append( shortCollectionName, hash ); - - md5_append( &globalState , (const md5_byte_t*)hash.c_str() , hash.size() ); - } - bb.done(); - - md5digest d; - md5_finish(&globalState, d); - string hash = digestToString( d ); - - result.append( "md5" , hash ); - result.appendNumber( "timeMillis", timer.millis() ); - return 1; - } - - } dbhashCmd; /* for diagnostic / testing purposes. Enabled via command line. */ class CmdSleep : public Command { diff --git a/src/mongo/db/dbeval.cpp b/src/mongo/db/dbeval.cpp index 5a6cc464c34..9e6a3360a49 100644 --- a/src/mongo/db/dbeval.cpp +++ b/src/mongo/db/dbeval.cpp @@ -57,7 +57,9 @@ namespace mongo { return false; } - auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" ); + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); + auto_ptr<Scope> s = globalScriptEngine->getPooledScope( dbName, "dbeval" + userToken ); ScriptingFunction f = s->createFunction(code); if ( f == 0 ) { errmsg = (string)"compile failed: " + s->getError(); diff --git a/src/mongo/db/fts/fts_spec.cpp b/src/mongo/db/fts/fts_spec.cpp index eb534f10a3e..f1829104796 100644 --- a/src/mongo/db/fts/fts_spec.cpp +++ b/src/mongo/db/fts/fts_spec.cpp @@ -30,10 +30,25 @@ namespace mongo { const double MAX_WEIGHT = 1000000000.0; const double MAX_WORD_WEIGHT = MAX_WEIGHT / 10000; + const int TEXT_INDEX_VERSION = 1; FTSSpec::FTSSpec( const BSONObj& indexInfo ) { - massert( 16739, "found invalid spec for text index", + // indexInfo is a text index spec. Text index specs pass through fixSpec() before + // being saved to the system.indexes collection. fixSpec() enforces a schema, such that + // required fields must exist and be of the correct type (e.g. weights, + // textIndexVersion). + massert( 16739, + "found invalid spec for text index, expected object for weights", indexInfo["weights"].isABSONObj() ); + BSONElement textIndexVersionElt = indexInfo["textIndexVersion"]; + massert( 17287, + "found invalid spec for text index, expected number for textIndexVersion", + textIndexVersionElt.isNumber() ); + massert( 17288, + str::stream() << "attempt to use unsupported textIndexVersion " << + textIndexVersionElt.numberInt() << ", only textIndexVersion " << + TEXT_INDEX_VERSION << " supported", + textIndexVersionElt.numberInt() == TEXT_INDEX_VERSION ); _defaultLanguage = indexInfo["default_language"].valuestrsafe(); _languageOverrideField = indexInfo["language_override"].valuestrsafe(); @@ -357,7 +372,7 @@ namespace mongo { language_override = "language"; int version = -1; - int textIndexVersion = 1; + int textIndexVersion = TEXT_INDEX_VERSION; BSONObjBuilder b; BSONObjIterator i( spec ); @@ -385,7 +400,7 @@ namespace mongo { textIndexVersion = e.numberInt(); uassert( 16730, str::stream() << "bad textIndexVersion: " << textIndexVersion, - textIndexVersion == 1 ); + textIndexVersion == TEXT_INDEX_VERSION ); } else { b.append( e ); diff --git a/src/mongo/db/fts/fts_spec_test.cpp b/src/mongo/db/fts/fts_spec_test.cpp index df4ff719f02..00c95261207 100644 --- a/src/mongo/db/fts/fts_spec_test.cpp +++ b/src/mongo/db/fts/fts_spec_test.cpp @@ -24,6 +24,36 @@ namespace mongo { namespace fts { + const BSONObj makeFixedSpec( int textIndexVersion ) { + return BSON( "v" << 1 << + "key" << BSON( "_fts" << "text" << + "_ftsx" << 1 ) << + "name" << "a_text" << + "ns" << "test.foo" << + "weights" << BSON( "a" << 1 ) << + "default_language" << "english" << + "language_override" << "language" << + "textIndexVersion" << textIndexVersion ); + } + + TEST( FTSSpec, TextIndexVersionCheck1 ) { + const int currentVersion = 1; + const int unsupportedVersion = 2; + + // Constructing an FTSSpec with the current textIndexVersion should succeed. + BSONObj validTextSpec = makeFixedSpec( currentVersion ); + try { + FTSSpec spec( validTextSpec ); + } + catch ( DBException& e ) { + ASSERT( false ); + } + + // Constructing an FTSSpec with an unsupported textIndexVersion should fail. + BSONObj invalidTextSpec = makeFixedSpec( unsupportedVersion ); + ASSERT_THROWS( FTSSpec spec( invalidTextSpec ), DBException ); + } + TEST( FTSSpec, Fix1 ) { BSONObj user = BSON( "key" << BSON( "title" << "fts" << "text" << "fts" ) << diff --git a/src/mongo/db/geo/geoparser.cpp b/src/mongo/db/geo/geoparser.cpp index 3de520be0be..ed86a2c3fa9 100644 --- a/src/mongo/db/geo/geoparser.cpp +++ b/src/mongo/db/geo/geoparser.cpp @@ -39,8 +39,15 @@ namespace mongo { static const string GEOJSON_COORDINATES = "coordinates"; //// Utility functions used by GeoParser functions below. - static S2Point coordToPoint(double p0, double p1) { - return S2LatLng::FromDegrees(p1, p0).Normalized().ToPoint(); + static S2Point coordToPoint(double lng, double lat) { + // Note that it's (lat, lng) for S2 but (lng, lat) for MongoDB. + S2LatLng ll = S2LatLng::FromDegrees(lat, lng).Normalized(); + if (!ll.is_valid()) { + stringstream ss; + ss << "coords invalid after normalization, lng = " << lng << " lat = " << lat << endl; + uasserted(17125, ss.str()); + } + return ll.ToPoint(); } static S2Point coordsToPoint(const vector<BSONElement>& coordElt) { diff --git a/src/mongo/db/matcher.cpp b/src/mongo/db/matcher.cpp index 2e679591f2c..532c18e1fe0 100644 --- a/src/mongo/db/matcher.cpp +++ b/src/mongo/db/matcher.cpp @@ -27,6 +27,7 @@ #include "db.h" #include "queryutil.h" #include "client.h" +#include "mongo/db/auth/authorization_manager.h" #include "pdfile.h" @@ -74,8 +75,10 @@ namespace mongo { return; _initCalled = true; + const string userToken = ClientBasic::getCurrent()->getAuthorizationManager() + ->getAuthenticatedPrincipalNamesToken(); NamespaceString ns( _ns ); - _scope = globalScriptEngine->getPooledScope( ns.db.c_str(), "where" ); + _scope = globalScriptEngine->getPooledScope( ns.db.c_str(), "where" + userToken ); massert( 10341 , "code has to be set first!" , ! _jsCode.empty() ); @@ -429,6 +432,8 @@ namespace mongo { uassert( 10066 , "$where may only appear once in query", _where == 0 ); uassert( 10067 , "$where query, but no script engine", globalScriptEngine ); massert( 13089 , "no current client needed for $where" , haveClient() ); + uassert( 17126 , "no valid context found for $where", cc().getContext()); + _where = new Where( cc().ns() ); if ( e.type() == CodeWScope ) { diff --git a/src/mongo/db/namespace_details.cpp b/src/mongo/db/namespace_details.cpp index 94adac276a6..0fc7bdf5eb5 100644 --- a/src/mongo/db/namespace_details.cpp +++ b/src/mongo/db/namespace_details.cpp @@ -976,6 +976,10 @@ namespace mongo { bool legalClientSystemNS( const string& ns , bool write ) { if( ns == "local.system.replset" ) return true; + if( ns == "admin.system.version" ) return true; + if( ns == "admin.system.roles" ) return true; + if( ns == "admin.system.new_users" ) return true; + if( ns == "admin.system.backup_users" ) return true; if ( ns.find( ".system.users" ) != string::npos ) return true; diff --git a/src/mongo/db/oplog.cpp b/src/mongo/db/oplog.cpp index 12f4742c256..b88ad66ac03 100644 --- a/src/mongo/db/oplog.cpp +++ b/src/mongo/db/oplog.cpp @@ -26,6 +26,7 @@ #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/dbhash.h" #include "mongo/db/index_update.h" #include "mongo/db/instance.h" #include "mongo/db/namespacestring.h" @@ -334,6 +335,7 @@ namespace mongo { } logOpForSharding( opstr , ns , obj , patt ); + logOpForDbHash( opstr , ns , obj , patt ); } void createOplog() { @@ -977,13 +979,17 @@ namespace mongo { BSONElement e = i.next(); const BSONObj& temp = e.Obj(); - Client::Context ctx(temp["ns"].String()); + string ns = temp["ns"].String(); + Client::Context ctx(ns); + bool failed = applyOperation_inlock(temp, false, alwaysUpsert); ab.append(!failed); if ( failed ) errors++; num++; + + logOpForDbHash( "u", ns.c_str(), BSONObj(), NULL ); } result.append( "applied" , num ); diff --git a/src/mongo/db/pdfile.cpp b/src/mongo/db/pdfile.cpp index 928b8560ced..4085dfd2b3f 100644 --- a/src/mongo/db/pdfile.cpp +++ b/src/mongo/db/pdfile.cpp @@ -267,7 +267,7 @@ namespace mongo { BSONElement e = options.getField("size"); if ( e.isNumber() ) { size = e.numberLong(); - uassert( 10083 , "create collection invalid size spec", size > 0 ); + uassert( 10083 , "create collection invalid size spec", size >= 0 ); size += 0xff; size &= 0xffffffffffffff00LL; diff --git a/src/mongo/db/pipeline/document.h b/src/mongo/db/pipeline/document.h index b51efa91497..5e7cf3f0579 100644 --- a/src/mongo/db/pipeline/document.h +++ b/src/mongo/db/pipeline/document.h @@ -162,6 +162,7 @@ namespace mongo { friend class FieldIterator; friend class ValueStorage; friend class MutableDocument; + friend class MutableValue; const DocumentStorage& storage() const { return (_storage ? *_storage : DocumentStorage::emptyDoc()); @@ -194,8 +195,17 @@ namespace mongo { /// Used by MutableDocument(MutableValue) const RefCountable*& getDocPtr() { - if (_val.getType() != Object) - *this = Value(Document()); + if (_val.getType() != Object || _val._storage.genericRCPtr == NULL) { + // If the current value isn't an object we replace it with a Object-typed Value. + // Note that we can't just use Document() here because that is a NULL pointer and + // Value doesn't refcount NULL pointers. This led to a memory leak (SERVER-10554) + // because MutableDocument::newStorage() would set a non-NULL pointer into the Value + // without setting the refCounter bit. While allocating a DocumentStorage here could + // result in an allocation where none is needed, in practice this is only called + // when we are about to add a field to the sub-document so this just changes where + // the allocation is done. + _val = Value(Document(new DocumentStorage())); + } return _val._storage.genericRCPtr; } diff --git a/src/mongo/db/pipeline/value.cpp b/src/mongo/db/pipeline/value.cpp index 5561cb41a59..207dc4e37b8 100644 --- a/src/mongo/db/pipeline/value.cpp +++ b/src/mongo/db/pipeline/value.cpp @@ -28,6 +28,48 @@ namespace mongo { using namespace mongoutils; + void ValueStorage::verifyRefCountingIfShould() const { + switch (type) { + case MinKey: + case MaxKey: + case jstOID: + case Date: + case Timestamp: + case EOO: + case jstNULL: + case Undefined: + case Bool: + case NumberInt: + case NumberLong: + case NumberDouble: + // the above types never reference external data + verify(!refCounter); + break; + + case String: + case RegEx: + case Code: + case Symbol: + // the above types reference data when not using short-string optimization + verify(refCounter == !shortStr); + break; + + case BinData: // TODO this should probably support short-string optimization + case Array: // TODO this should probably support empty-is-NULL optimization + case DBRef: + case CodeWScope: + // the above types always reference external data. + verify(refCounter); + verify(bool(genericRCPtr)); + break; + + case Object: + // Objects either hold a NULL ptr or should be ref-counting + verify(refCounter == bool(genericRCPtr)); + break; + } + } + void ValueStorage::putString(const StringData& s) { // Note: this also stores data portion of BinData const size_t sizeNoNUL = s.size(); diff --git a/src/mongo/db/pipeline/value_internal.h b/src/mongo/db/pipeline/value_internal.h index e3481951f9e..ffc7f42a7f7 100644 --- a/src/mongo/db/pipeline/value_internal.h +++ b/src/mongo/db/pipeline/value_internal.h @@ -89,6 +89,7 @@ namespace mongo { } ~ValueStorage() { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_release(genericRCPtr); DEV memset(this, 0xee, sizeof(*this)); @@ -109,6 +110,7 @@ namespace mongo { /// Call this after memcpying to update ref counts if needed void memcpyed() const { + DEV verifyRefCountingIfShould(); if (refCounter) intrusive_ptr_add_ref(genericRCPtr); } @@ -140,6 +142,7 @@ namespace mongo { intrusive_ptr_add_ref(genericRCPtr); refCounter = true; } + DEV verifyRefCountingIfShould(); } StringData getString() const { @@ -191,6 +194,8 @@ namespace mongo { && i64[1] == other.i64[1]); } + void verifyRefCountingIfShould() const; + // This data is public because this should only be used by Value which would be a friend union { struct { diff --git a/src/mongo/db/repl/consensus.cpp b/src/mongo/db/repl/consensus.cpp index f056befb605..dcb31408c11 100644 --- a/src/mongo/db/repl/consensus.cpp +++ b/src/mongo/db/repl/consensus.cpp @@ -61,7 +61,9 @@ namespace mongo { return true; } - if (primary && primary->hbinfo().opTime >= hopeful->hbinfo().opTime) { + if (primary && + (hopeful->hbinfo().id() != primary->hbinfo().id()) && + (primary->hbinfo().opTime >= hopeful->hbinfo().opTime)) { // other members might be aware of more up-to-date nodes errmsg = str::stream() << hopeful->fullName() << " is trying to elect itself but " << primary->fullName() << diff --git a/src/mongo/db/repl/heartbeat.cpp b/src/mongo/db/repl/heartbeat.cpp index f1bc18168f6..95bde6d185b 100644 --- a/src/mongo/db/repl/heartbeat.cpp +++ b/src/mongo/db/repl/heartbeat.cpp @@ -264,11 +264,13 @@ namespace mongo { down(mem, info.getStringField("errmsg")); } } - catch(DBException& e) { + catch (const DBException& e) { + log() << "replSet health poll task caught a DBException: " << e.what(); down(mem, e.what()); } - catch(...) { - down(mem, "replSet unexpected exception in ReplSetHealthPollTask"); + catch (const std::exception& e) { + log() << "replSet health poll task caught an exception: " << e.what(); + down(mem, e.what()); } m = mem; diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp index 68b7fc17529..841dcf7d898 100644 --- a/src/mongo/db/repl/rs_rollback.cpp +++ b/src/mongo/db/repl/rs_rollback.cpp @@ -177,6 +177,16 @@ namespace mongo { log() << "replSet " << o.toString() << rsLog; throw rsfatal(); } + else if( cmdname == "collMod" ) { + if ( o.nFields() == 2 && + o["usePowerOf2Sizes"].type() == Bool ) { + log() << "replSet not rolling back change of usePowerOf2Sizes: " << o; + } + else { + log() << "replSet error cannot rollback a collMod command: " << o; + throw rsfatal(); + } + } else { log() << "replSet error can't rollback this command yet: " << o.toString() << rsLog; log() << "replSet cmdname=" << cmdname << rsLog; diff --git a/src/mongo/db/ttl.cpp b/src/mongo/db/ttl.cpp index 66c0ee7ee7f..4019a550b2b 100644 --- a/src/mongo/db/ttl.cpp +++ b/src/mongo/db/ttl.cpp @@ -81,6 +81,12 @@ namespace mongo { error() << "key for ttl index can only have 1 field" << endl; continue; } + if (!idx[secondsExpireField].isNumber()) { + error() << "ttl indexes require the " << secondsExpireField << " field to be " + << "numeric but received a type of: " + << typeName(idx[secondsExpireField].type()) << endl; + continue; + } BSONObj query; { diff --git a/src/mongo/dbtests/documenttests.cpp b/src/mongo/dbtests/documenttests.cpp index 57bf7f8b55d..56e51176f1d 100644 --- a/src/mongo/dbtests/documenttests.cpp +++ b/src/mongo/dbtests/documenttests.cpp @@ -170,6 +170,22 @@ namespace DocumentTests { ASSERT( md.peek().getValue( "c" ).missing() ); assertRoundTrips( md.peek() ); + // Set a nested field using [] + md["x"]["y"]["z"] = Value("nested"); + ASSERT_EQUALS(md.peek()["x"]["y"]["z"], Value("nested")); + + // Set a nested field using setNestedField + FieldPath xxyyzz = string("xx.yy.zz"); + md.setNestedField(xxyyzz, Value("nested")); + ASSERT_EQUALS(md.peek().getNestedField(xxyyzz), Value("nested") ); + + // Set a nested fields through an existing empty document + md["xxx"] = Value(Document()); + md["xxx"]["yyy"] = Value(Document()); + FieldPath xxxyyyzzz = string("xxx.yyy.zzz"); + md.setNestedField(xxxyyyzzz, Value("nested")); + ASSERT_EQUALS(md.peek().getNestedField(xxxyyyzzz), Value("nested") ); + // Make sure nothing moved ASSERT_EQUALS(apos, md.peek().positionOf("a")); ASSERT_EQUALS(bpos, md.peek().positionOf("c")); diff --git a/src/mongo/s/client_info.cpp b/src/mongo/s/client_info.cpp index b373e54c010..7dc83aff003 100644 --- a/src/mongo/s/client_info.cpp +++ b/src/mongo/s/client_info.cpp @@ -268,41 +268,39 @@ namespace mongo { } clearSinceLastGetError(); - LOG(4) << "checking " << writebacks.size() << " writebacks for" - << " gle (" << theShard << ")" << endl; - - if ( writebacks.size() ){ - vector<BSONObj> v = _handleWriteBacks( writebacks , fromWriteBackListener ); - if ( v.size() == 0 && fromWriteBackListener ) { - // ok + // We never need to handle writebacks if we're coming from the wbl itself + if ( writebacks.size() && !fromWriteBackListener ){ + + LOG(4) << "checking " << writebacks.size() << " writebacks for" + << " gle (" << theShard << ")" << endl; + + vector<BSONObj> v = _handleWriteBacks( writebacks , false ); + + // this will usually be 1 + // it can be greater than 1 if a write to a different shard + // than the last write op had a writeback + // all we're going to report is the first + // since that's the current write + // but we block for all + verify( v.size() >= 1 ); + + if ( res["writebackSince"].numberInt() > 0 ) { + // got writeback from older op + // ignore the result from it, just needed to wait + result.appendElements( res ); + } + else if ( writebacks[0].fromLastOperation ) { + result.appendElements( v[0] ); + result.appendElementsUnique( res ); + result.append( "writebackGLE" , v[0] ); + result.append( "initialGLEHost" , theShard ); + result.append( "initialGLE", res ); } else { - // this will usually be 1 - // it can be greater than 1 if a write to a different shard - // than the last write op had a writeback - // all we're going to report is the first - // since that's the current write - // but we block for all - verify( v.size() >= 1 ); - - if ( res["writebackSince"].numberInt() > 0 ) { - // got writeback from older op - // ignore the result from it, just needed to wait - result.appendElements( res ); - } - else if ( writebacks[0].fromLastOperation ) { - result.appendElements( v[0] ); - result.appendElementsUnique( res ); - result.append( "writebackGLE" , v[0] ); - result.append( "initialGLEHost" , theShard ); - result.append( "initialGLE", res ); - } - else { - // there was a writeback - // but its from an old operations - // so all that's important is that we block, not that we return stats - result.appendElements( res ); - } + // there was a writeback + // but its from an old operations + // so all that's important is that we block, not that we return stats + result.appendElements( res ); } } else { @@ -406,6 +404,10 @@ namespace mongo { LOG(4) << "checking " << writebacks.size() << " writebacks for" << " gle (" << shards->size() << " shards)" << endl; + // Multi-shard results from the writeback listener implicitly means that: + // A) no versioning was used (multi-update/delete) + // B) internal GLE was used (bulk insert) + if ( errors.size() == 0 ) { result.appendNull( "err" ); _handleWriteBacks( writebacks , fromWriteBackListener ); diff --git a/src/mongo/s/commands_public.cpp b/src/mongo/s/commands_public.cpp index 1c2ff52364a..f69ef9fe1bb 100644 --- a/src/mongo/s/commands_public.cpp +++ b/src/mongo/s/commands_public.cpp @@ -1771,6 +1771,8 @@ namespace mongo { int options, string &errmsg, BSONObjBuilder &result, bool fromRepl); + virtual bool passOptions() const { return true; } + private: }; @@ -1815,7 +1817,7 @@ namespace mongo { */ DBConfigPtr conf(grid.getDBConfig(dbName , false)); if (!conf || !conf->isShardingEnabled() || !conf->isSharded(fullns)) - return passthrough(conf, cmdObj, result); + return passthrough(conf, cmdObj, options, result); /* split the pipeline into pieces for mongods and this mongos */ intrusive_ptr<Pipeline> pShardPipeline( diff --git a/src/mongo/s/shard.cpp b/src/mongo/s/shard.cpp index df620865603..28b281f0cbd 100644 --- a/src/mongo/s/shard.cpp +++ b/src/mongo/s/shard.cpp @@ -30,6 +30,7 @@ #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/jsobj.h" +#include "mongo/db/server_parameters.h" #include "mongo/s/client_info.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -39,6 +40,8 @@ namespace mongo { + MONGO_EXPORT_SERVER_PARAMETER(authOnPrimaryOnly, bool, true); + class StaticShardInfo { public: StaticShardInfo() : _mutex("StaticShardInfo"), _rsMutex("RSNameMap") { } @@ -411,14 +414,26 @@ namespace mongo { void ShardingConnectionHook::onCreate( DBClientBase * conn ) { if( !noauth ) { + bool result; string err; LOG(2) << "calling onCreate auth for " << conn->toString() << endl; - bool result = conn->auth( "local", - internalSecurity.user, - internalSecurity.pwd, - err, - false ); + if ( conn->type() == ConnectionString::SET && !authOnPrimaryOnly ) { + DBClientReplicaSet* setConn = dynamic_cast<DBClientReplicaSet*>(conn); + verify(setConn); + result = setConn->authAny( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } + else { + result = conn->auth( "local", + internalSecurity.user, + internalSecurity.pwd, + err, + false ); + } uassert( 15847, str::stream() << "can't authenticate to server " << conn->getServerAddress() << causedBy( err ), result ); diff --git a/src/mongo/s/shard.h b/src/mongo/s/shard.h index e917baa0986..41f819d3ae9 100644 --- a/src/mongo/s/shard.h +++ b/src/mongo/s/shard.h @@ -290,8 +290,12 @@ namespace mongo { */ bool runCommand( const string& db , const BSONObj& cmd , BSONObj& res ); + // Whether or not we release connections from the thread-local cache after a read static bool releaseConnectionsAfterResponse; + // Controls whether we throw on initially failing to set a version + static bool ignoreInitialVersionFailure; + /** checks all of my thread local connections for the version of this ns */ static void checkMyConnectionVersions( const string & ns ); @@ -307,6 +311,11 @@ namespace mongo { */ static void clearPool(); + /** + * Forgets a namespace to prevent future versioning. + */ + static void forgetNS( const string& ns ); + private: void _init(); void _finishInit(); diff --git a/src/mongo/s/shardconnection.cpp b/src/mongo/s/shardconnection.cpp index 61b47904147..66a105f05e8 100644 --- a/src/mongo/s/shardconnection.cpp +++ b/src/mongo/s/shardconnection.cpp @@ -22,6 +22,7 @@ #include "mongo/db/client.h" #include "mongo/db/commands.h" +#include "mongo/db/lasterror.h" #include "mongo/db/server_parameters.h" #include "mongo/s/config.h" #include "mongo/s/request.h" @@ -33,6 +34,14 @@ namespace mongo { + bool ShardConnection::ignoreInitialVersionFailure( false ); + ExportedServerParameter<bool> + _ignoreInitialVersionFailure( ServerParameterSet::getGlobal(), + "ignoreInitialVersionFailure", + &ShardConnection::ignoreInitialVersionFailure, + true, + true ); + DBConnectionPool shardConnectionPool; class ClientConnections; @@ -235,6 +244,12 @@ namespace mongo { vector<Shard> all; Shard::getAllShards( all ); + scoped_ptr<LastError::Disabled> ignoreForGLE; + if ( ShardConnection::ignoreInitialVersionFailure ) { + // Don't report exceptions here as errors in GetLastError if ignoring failures + ignoreForGLE.reset( new LastError::Disabled( lastError.get( false ) ) ); + } + // Now only check top-level shard connections for ( unsigned i=0; i<all.size(); i++ ) { @@ -250,11 +265,18 @@ namespace mongo { versionManager.checkShardVersionCB( s->avail, ns, false, 1 ); } - catch ( const std::exception& e ) { + catch ( const DBException& ex ) { - warning() << "problem while initially checking shard versions on" - << " " << shard.getName() << causedBy(e) << endl; - throw; + warning() << "problem while initially checking shard versions on " + << shard.getName() << causedBy( ex ) << endl; + + if ( !ShardConnection::ignoreInitialVersionFailure ) { + throw; + } + else { + // We swallow the error here, checking shard version here is a heuristic to + // prevent later stale config exceptions, not required for correctness. + } } } } @@ -322,6 +344,11 @@ namespace mongo { _hosts.clear(); } + void forgetNS( const string& ns ) { + scoped_spinlock lock( _lock ); + _seenNS.erase( ns ); + } + // ----- static thread_specific_ptr<ClientConnections> _perThread; @@ -487,4 +514,8 @@ namespace mongo { shardConnectionPool.clear(); ClientConnections::threadInstance()->clearPool(); } + + void ShardConnection::forgetNS( const string& ns ) { + ClientConnections::threadInstance()->forgetNS( ns ); + } } diff --git a/src/mongo/s/strategy_shard.cpp b/src/mongo/s/strategy_shard.cpp index 0d4adcef155..560a4ac72d4 100644 --- a/src/mongo/s/strategy_shard.cpp +++ b/src/mongo/s/strategy_shard.cpp @@ -192,15 +192,10 @@ namespace mongo { string host = cursorCache.getRef( id ); if( host.size() == 0 ){ - - // - // Match legacy behavior here by throwing an exception when we can't find - // the cursor, but make the exception more informative - // - - uasserted( 16336, - str::stream() << "could not find cursor in cache for id " << id - << " over collection " << ns ); + LOG(3) << "could not find cursor in cache for id " << id + << " over collection " << ns << endl; + replyToQuery( ResultFlag_CursorNotFound , r.p() , r.m() , 0 , 0 , 0 ); + return; } // we used ScopedDbConnection because we don't get about config versions @@ -227,11 +222,9 @@ namespace mongo { int ntoreturn = r.d().pullInt(); long long id = r.d().pullInt64(); - LOG(6) << "want cursor : " << id << endl; - ShardedClientCursorPtr cursor = cursorCache.get( id ); if ( ! cursor ) { - LOG(6) << "\t invalid cursor :(" << endl; + LOG(3) << "Invalid cursor:" << id << endl; replyToQuery( ResultFlag_CursorNotFound , r.p() , r.m() , 0 , 0 , 0 ); return; } diff --git a/src/mongo/s/type_shard.cpp b/src/mongo/s/type_shard.cpp index 9b6fd2fdb78..a8f4236aa5d 100644 --- a/src/mongo/s/type_shard.cpp +++ b/src/mongo/s/type_shard.cpp @@ -87,7 +87,7 @@ namespace mongo { if (fieldState == FieldParser::FIELD_INVALID) return false; _isDrainingSet = fieldState == FieldParser::FIELD_SET; - fieldState = FieldParser::extract(source, maxSize, &_maxSize, errMsg); + fieldState = FieldParser::extractNumber(source, maxSize, &_maxSize, errMsg); if (fieldState == FieldParser::FIELD_INVALID) return false; _isMaxSizeSet = fieldState == FieldParser::FIELD_SET; diff --git a/src/mongo/s/type_shard_test.cpp b/src/mongo/s/type_shard_test.cpp index dda55a5daaf..c75db61ec26 100644 --- a/src/mongo/s/type_shard_test.cpp +++ b/src/mongo/s/type_shard_test.cpp @@ -74,6 +74,17 @@ namespace { ASSERT_TRUE(shard.isValid(NULL)); } + TEST(Validity, MaxSizeAsFloat) { + ShardType shard; + BSONObj obj = BSON(ShardType::name("shard0000") << + ShardType::host("localhost:27017") << + ShardType::maxSize() << 100.0); + string errMsg; + ASSERT(shard.parseBSON(obj, &errMsg)); + ASSERT_EQUALS(errMsg, ""); + ASSERT_TRUE(shard.isValid(NULL)); + } + TEST(Validity, BadType) { ShardType shard; BSONObj obj = BSON(ShardType::name() << 0); diff --git a/src/mongo/s/version_manager.cpp b/src/mongo/s/version_manager.cpp index 1496ebc1407..3ba6254813b 100644 --- a/src/mongo/s/version_manager.cpp +++ b/src/mongo/s/version_manager.cpp @@ -103,23 +103,46 @@ namespace mongo { WriteBackListener::init( *conn_in ); - DBClientBase* conn = getVersionable( conn_in ); - verify( conn ); // errors thrown above + bool ok; + DBClientBase* conn = NULL; + try { + // May throw if replica set primary is down + conn = getVersionable( conn_in ); + dassert( conn ); // errors thrown above + + BSONObjBuilder cmdBuilder; + + cmdBuilder.append( "setShardVersion" , "" ); + cmdBuilder.appendBool( "init", true ); + cmdBuilder.append( "configdb" , configServer.modelServer() ); + cmdBuilder.appendOID( "serverID" , &serverID ); + cmdBuilder.appendBool( "authoritative" , true ); - BSONObjBuilder cmdBuilder; + BSONObj cmd = cmdBuilder.obj(); - cmdBuilder.append( "setShardVersion" , "" ); - cmdBuilder.appendBool( "init", true ); - cmdBuilder.append( "configdb" , configServer.modelServer() ); - cmdBuilder.appendOID( "serverID" , &serverID ); - cmdBuilder.appendBool( "authoritative" , true ); + LOG(1) << "initializing shard connection to " << conn->toString() << endl; + LOG(2) << "initial sharding settings : " << cmd << endl; + + ok = conn->runCommand("admin", cmd, result, 0); + } + catch( const DBException& ex ) { - BSONObj cmd = cmdBuilder.obj(); + bool ignoreFailure = ShardConnection::ignoreInitialVersionFailure + && conn_in->type() == ConnectionString::SET; + if ( !ignoreFailure ) + throw; - LOG(1) << "initializing shard connection to " << conn->toString() << endl; - LOG(2) << "initial sharding settings : " << cmd << endl; + // Using initShardVersion is not strictly required when talking to replica sets - it is + // preferred to do so because it registers mongos early with the mongod. This info is + // also sent by checkShardVersion before a connection is used for a write or read. - bool ok = conn->runCommand("admin", cmd, result, 0); + OCCASIONALLY { + warning() << "failed to initialize new replica set connection version, " + << "will initialize on first use" << endl; + } + + return true; + } // HACK for backwards compatibility with v1.8.x, v2.0.0 and v2.0.1 // Result is false, but will still initialize serverID and configdb diff --git a/src/mongo/s/writeback_listener.cpp b/src/mongo/s/writeback_listener.cpp index 255ab0160e5..0695c4cec56 100644 --- a/src/mongo/s/writeback_listener.cpp +++ b/src/mongo/s/writeback_listener.cpp @@ -345,6 +345,9 @@ namespace mongo { gle = b.obj(); } + dassert( !gle.isEmpty() ); + verify( !gle.isEmpty() ); + if ( gle["code"].numberInt() == 9517 ) { log() << "new version change detected, " diff --git a/src/mongo/scripting/engine.cpp b/src/mongo/scripting/engine.cpp index fc4d042c2af..e6bb8be819e 100644 --- a/src/mongo/scripting/engine.cpp +++ b/src/mongo/scripting/engine.cpp @@ -39,7 +39,7 @@ namespace mongo { Scope::Scope() : _localDBName(""), _loadedVersion(0), - _numTimeUsed(0), + _numTimesUsed(0), _lastRetIsNativeCode(false) { } @@ -259,90 +259,80 @@ namespace mongo { injectNative("benchFinish", BenchRunner::benchFinish); } - typedef map<string, list<Scope*> > PoolToScopes; - +namespace { class ScopeCache { public: - ScopeCache() : _mutex("ScopeCache") { - } + ScopeCache() : _mutex("ScopeCache") {} - ~ScopeCache() { - if (inShutdown()) - return; - clear(); - } - - void done(const string& pool, Scope* s) { + void release(const string& poolName, const boost::shared_ptr<Scope>& scope) { scoped_lock lk(_mutex); - list<Scope*>& l = _pools[pool]; - bool oom = s->hasOutOfMemoryException(); - // do not keep too many contexts, or use them for too long - if (l.size() > 10 || s->getTimeUsed() > 10 || oom || !s->getError().empty()) { - delete s; - } - else { - l.push_back(s); - s->reset(); + if (scope->hasOutOfMemoryException()) { + // make some room + log() << "Clearing all idle JS contexts due to out of memory" << endl; + _pools.clear(); + return; } - if (oom) { - // out of mem, make some room - log() << "Clearing all idle JS contexts due to out of memory" << endl; - clear(); + if (scope->getTimesUsed() > kMaxScopeReuse) + return; // used too many times to save + + if (!scope->getError().empty()) + return; // not saving errored scopes + + if (_pools.size() >= kMaxPoolSize) { + // prefer to keep recently-used scopes + _pools.pop_back(); } + + ScopeAndPool toStore = {scope, poolName}; + _pools.push_front(toStore); } - Scope* get(const string& pool) { + boost::shared_ptr<Scope> tryAcquire(const string& poolName) { scoped_lock lk(_mutex); - list<Scope*>& l = _pools[pool]; - if (l.size() == 0) - return 0; - - Scope* s = l.back(); - l.pop_back(); - s->reset(); - s->incTimeUsed(); - return s; - } - void clear() { - set<Scope*> seen; - for (PoolToScopes::iterator i = _pools.begin(); i != _pools.end(); ++i) { - for (list<Scope*>::iterator j = i->second.begin(); j != i->second.end(); ++j) { - Scope* s = *j; - fassert(16652, seen.insert(s).second); - delete s; + for (Pools::iterator it = _pools.begin(); it != _pools.end(); ++it) { + if (it->poolName == poolName) { + boost::shared_ptr<Scope> scope = it->scope; + _pools.erase(it); + scope->incTimesUsed(); + scope->reset(); + return scope; } } - _pools.clear(); + + return boost::shared_ptr<Scope>(); } private: - PoolToScopes _pools; + struct ScopeAndPool { + boost::shared_ptr<Scope> scope; + string poolName; + }; + + // Note: if these numbers change, reconsider choice of datastructure for _pools + static const unsigned kMaxPoolSize = 10; + static const int kMaxScopeReuse = 10; + + typedef deque<ScopeAndPool> Pools; // More-recently used Scopes are kept at the front. + Pools _pools; // protected by _mutex mongo::mutex _mutex; }; - thread_specific_ptr<ScopeCache> scopeCache; + ScopeCache scopeCache; +} // anonymous namespace class PooledScope : public Scope { public: - PooledScope(const std::string& pool, Scope* real) : _pool(pool), _real(real) { + PooledScope(const std::string& pool, const boost::shared_ptr<Scope>& real) + : _pool(pool) + , _real(real) { _real->loadStored(true); - }; + } + virtual ~PooledScope() { - ScopeCache* sc = scopeCache.get(); - if (sc) { - sc->done(_pool, _real); - _real = NULL; - } - else { - // this means that the Scope was killed from a different thread - // for example a cursor got timed out that has a $where clause - LOG(3) << "warning: scopeCache is empty!" << endl; - delete _real; - _real = 0; - } + scopeCache.release(_pool, _real); } // wrappers for the derived (_real) scope @@ -404,31 +394,24 @@ namespace mongo { private: string _pool; - Scope* _real; + boost::shared_ptr<Scope> _real; }; /** Get a scope from the pool of scopes matching the supplied pool name */ - auto_ptr<Scope> ScriptEngine::getPooledScope(const string& pool, const string& scopeType) { - if (!scopeCache.get()) - scopeCache.reset(new ScopeCache()); - - Scope* s = scopeCache->get(pool + scopeType); - if (!s) - s = newScope(); + auto_ptr<Scope> ScriptEngine::getPooledScope(const string& db, const string& scopeType) { + const string fullPoolName = db + scopeType; + boost::shared_ptr<Scope> s = scopeCache.tryAcquire(fullPoolName); + if (!s) { + s.reset(newScope()); + } auto_ptr<Scope> p; - p.reset(new PooledScope(pool + scopeType, s)); - p->setLocalDB(pool); + p.reset(new PooledScope(fullPoolName, s)); + p->setLocalDB(db); p->loadStored(true); return p; } - void ScriptEngine::threadDone() { - ScopeCache* sc = scopeCache.get(); - if (sc) - sc->clear(); - } - void (*ScriptEngine::_connectCallback)(DBClientWithCommands&) = 0; const char* (*ScriptEngine::_checkInterruptCallback)() = 0; unsigned (*ScriptEngine::_getCurrentOpIdCallback)() = 0; diff --git a/src/mongo/scripting/engine.h b/src/mongo/scripting/engine.h index 8233128e736..e0a3486a6ae 100644 --- a/src/mongo/scripting/engine.h +++ b/src/mongo/scripting/engine.h @@ -135,10 +135,10 @@ namespace mongo { static void validateObjectIdString(const string& str); /** increments the number of times a scope was used */ - void incTimeUsed() { ++_numTimeUsed; } + void incTimesUsed() { ++_numTimesUsed; } /** gets the number of times a scope was used */ - int getTimeUsed() { return _numTimeUsed; } + int getTimesUsed() { return _numTimesUsed; } /** return true if last invoke() return'd native code */ virtual bool isLastRetNativeCode() { return _lastRetIsNativeCode; } @@ -168,7 +168,7 @@ namespace mongo { set<string> _storedNames; static long long _lastVersion; FunctionCacheMap _cachedFunctions; - int _numTimeUsed; + int _numTimesUsed; bool _lastRetIsNativeCode; // v8 only: set to true if eval'd script returns a native func }; @@ -188,15 +188,12 @@ namespace mongo { static void setup(); /** gets a scope from the pool or a new one if pool is empty - * @param pool An identifier for the pool, usually the db name + * @param db The db name + * @param scopeType A unique id to limit scope sharing. + * This must include authenticated users. * @return the scope */ - auto_ptr<Scope> getPooledScope(const string& pool, const string& scopeType); - - /** - * call this method to release some JS resources when a thread is done - */ - void threadDone(); + auto_ptr<Scope> getPooledScope(const string& db, const string& scopeType); void setScopeInitCallback(void (*func)(Scope&)) { _scopeInitCallback = func; } static void setConnectCallback(void (*func)(DBClientWithCommands&)) { diff --git a/src/mongo/shell/assert.js b/src/mongo/shell/assert.js index b1279028307..0e84c52d11c 100644 --- a/src/mongo/shell/assert.js +++ b/src/mongo/shell/assert.js @@ -225,3 +225,41 @@ assert.close = function(a, b, msg, places){ doassert(a + " is not equal to " + b + " within " + places + " places, diff: " + (a-b) + " : " + msg); }; + +assert.gleSuccess = function(db, msg) { + var gle = db.getLastErrorObj(); + if (gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null:" + tojson(gle) + " :" + msg); + } +} + +assert.gleError = function(db, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null: " + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorCode = function(db, code, msg) { + var gle = db.getLastErrorObj(); + if (gle.err && (gle.code == code)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError not null or missing code( " + code + "): " + + tojson(gle) + " :" + msg); + } +} + +assert.gleErrorRegex = function(db, regex, msg) { + var gle = db.getLastErrorObj(); + if (!gle.err || !regex.test(gle.err)) { + if (typeof(msg) == "function") + msg = msg(gle); + doassert("getLastError is null or doesn't match regex (" + regex + "): " + + tojson(gle) + " :" + msg); + } +} diff --git a/src/mongo/shell/dbshell.cpp b/src/mongo/shell/dbshell.cpp index 5934e862076..6a790ee2adb 100644 --- a/src/mongo/shell/dbshell.cpp +++ b/src/mongo/shell/dbshell.cpp @@ -75,7 +75,9 @@ void generateCompletions( const string& prefix , vector<string>& all ) { try { BSONObj args = BSON( "0" << prefix ); - shellMainScope->invokeSafe( "function callShellAutocomplete(x) {shellAutocomplete(x)}", &args, 0, 1000 ); + shellMainScope->invokeSafe("function callShellAutocomplete(x) {shellAutocomplete(x)}", + &args, + NULL); BSONObjBuilder b; shellMainScope->append( b , "" , "__autocomplete__" ); BSONObj res = b.obj(); diff --git a/src/mongo/tools/restore.cpp b/src/mongo/tools/restore.cpp index abd7f868afe..625902224ef 100644 --- a/src/mongo/tools/restore.cpp +++ b/src/mongo/tools/restore.cpp @@ -484,34 +484,29 @@ private: return nfields == obj2.nFields(); } - void createCollectionWithOptions(BSONObj cmdObj) { + void createCollectionWithOptions(BSONObj obj) { + BSONObjIterator i(obj); - // Create a new cmdObj to skip undefined fields and fix collection name + // Rebuild obj as a command object for the "create" command. + // - {create: <name>} comes first, where <name> is the new name for the collection + // - elements with type Undefined get skipped over BSONObjBuilder bo; - - // Add a "create" field if it doesn't exist - if (!cmdObj.hasField("create")) { - bo.append("create", _curcoll); - } - - BSONObjIterator i(cmdObj); - while ( i.more() ) { + bo.append("create", _curcoll); + while (i.more()) { BSONElement e = i.next(); - // Replace the "create" field with the name of the collection we are actually creating if (strcmp(e.fieldName(), "create") == 0) { - bo.append("create", _curcoll); + continue; } - else { - if (e.type() == Undefined) { - log() << _curns << ": skipping undefined field: " << e.fieldName() << endl; - } - else { - bo.append(e); - } + + if (e.type() == Undefined) { + log() << _curns << ": skipping undefined field: " << e.fieldName() << endl; + continue; } + + bo.append(e); } - cmdObj = bo.obj(); + obj = bo.obj(); BSONObj fields = BSON("options" << 1); scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", Query(BSON("name" << _curns)), 0, 0, &fields)); @@ -519,8 +514,8 @@ private: bool createColl = true; if (cursor->more()) { createColl = false; - BSONObj obj = cursor->next(); - if (!obj.hasField("options") || !optionsSame(cmdObj, obj["options"].Obj())) { + BSONObj nsObj = cursor->next(); + if (!nsObj.hasField("options") || !optionsSame(obj, nsObj["options"].Obj())) { log() << "WARNING: collection " << _curns << " exists with different options than are in the metadata.json file and not using --drop. Options in the metadata file will be ignored." << endl; } } @@ -530,10 +525,10 @@ private: } BSONObj info; - if (!conn().runCommand(_curdb, cmdObj, info)) { + if (!conn().runCommand(_curdb, obj, info)) { uasserted(15936, "Creating collection " + _curns + " failed. Errmsg: " + info["errmsg"].String()); } else { - log() << "\tCreated collection " << _curns << " with options: " << cmdObj.jsonString() << endl; + log() << "\tCreated collection " << _curns << " with options: " << obj.jsonString() << endl; } } diff --git a/src/mongo/util/net/listen.cpp b/src/mongo/util/net/listen.cpp index 4ba94d19c90..b92a3266a14 100644 --- a/src/mongo/util/net/listen.cpp +++ b/src/mongo/util/net/listen.cpp @@ -262,10 +262,14 @@ namespace mongo { int s = accept(*it, from.raw(), &from.addressSize); if ( s < 0 ) { int x = errno; // so no global issues - if ( x == ECONNABORTED || x == EBADF ) { - log() << "Listener on port " << _port << " aborted" << endl; + if (x == EBADF) { + log() << "Port " << _port << " is no longer valid" << endl; return; } + else if (x == ECONNABORTED) { + log() << "Connection on port " << _port << " aborted" << endl; + continue; + } if ( x == 0 && inShutdown() ) { return; // socket closed } @@ -461,9 +465,13 @@ namespace mongo { int s = accept(socks[eventIndex], from.raw(), &from.addressSize); if ( s < 0 ) { int x = errno; // so no global issues - if ( x == ECONNABORTED || x == EBADF ) { + if (x == EBADF) { + log() << "Port " << _port << " is no longer valid" << endl; + continue; + } + else if (x == ECONNABORTED) { log() << "Listener on port " << _port << " aborted" << endl; - return; + continue; } if ( x == 0 && inShutdown() ) { return; // socket closed diff --git a/src/mongo/util/net/ssl_manager.cpp b/src/mongo/util/net/ssl_manager.cpp index 83ecd5923da..dd8b3a2fe6f 100644 --- a/src/mongo/util/net/ssl_manager.cpp +++ b/src/mongo/util/net/ssl_manager.cpp @@ -51,13 +51,9 @@ namespace mongo { SSLThreadInfo() { _id = ++_next; - CRYPTO_set_id_callback(_ssl_id_callback); - CRYPTO_set_locking_callback(_ssl_locking_callback); } - ~SSLThreadInfo() { - CRYPTO_set_id_callback(0); - } + ~SSLThreadInfo() {} unsigned long id() const { return _id; } @@ -150,14 +146,11 @@ namespace mongo { // Note: this is for blocking sockets only. SSL_CTX_set_mode(_context, SSL_MODE_AUTO_RETRY); - // Set context within which session can be reused - int status = SSL_CTX_set_session_id_context( - _context, - static_cast<unsigned char*>(static_cast<void*>(&_context)), - sizeof(_context)); - if (!status) { - uasserted(16768,"ssl initialization problem"); - } + // Disable session caching (see SERVER-10261) + SSL_CTX_set_session_cache_mode(_context, SSL_SESS_CACHE_OFF); + + CRYPTO_set_id_callback(_ssl_id_callback); + CRYPTO_set_locking_callback(_ssl_locking_callback); SSLThreadInfo::init(); SSLThreadInfo::get(); diff --git a/src/mongo/util/progress_meter.cpp b/src/mongo/util/progress_meter.cpp index 57ea9f728ec..d0f99ddff94 100644 --- a/src/mongo/util/progress_meter.cpp +++ b/src/mongo/util/progress_meter.cpp @@ -53,9 +53,13 @@ namespace mongo { if ( _total > 0 ) { int per = (int)( ( (double)_done * 100.0 ) / (double)_total ); - Nullstream& out = log() << "\t\t" << _name << ": " << _done - << '/' << _total << '\t' << per << '%'; + Nullstream& out = log(); + out << "\t\t" << _name << ": " << _done; + if (_showTotal) { + out << '/' << _total << '\t' << per << '%'; + } + if ( ! _units.empty() ) { out << "\t(" << _units << ")"; } diff --git a/src/mongo/util/progress_meter.h b/src/mongo/util/progress_meter.h index 88dbe29a0f5..baeb4680391 100644 --- a/src/mongo/util/progress_meter.h +++ b/src/mongo/util/progress_meter.h @@ -30,12 +30,13 @@ namespace mongo { int checkInterval = 100, std::string units = "", std::string name = "Progress") - : _units(units) - , _name(name) { + : _showTotal(true), + _units(units), + _name(name) { reset( total , secondsBetween , checkInterval ); } - ProgressMeter() : _active(0), _units(""), _name("Progress") {} + ProgressMeter() : _active(0), _showTotal(true), _units(""), _name("Progress") {} // typically you do ProgressMeterHolder void reset( unsigned long long total , int secondsBetween = 3 , int checkInterval = 100 ); @@ -65,6 +66,10 @@ namespace mongo { unsigned long long total() const { return _total; } + void showTotal(bool doShow) { + _showTotal = doShow; + } + std::string toString() const; bool operator==( const ProgressMeter& other ) const { return this == &other; } @@ -74,6 +79,7 @@ namespace mongo { bool _active; unsigned long long _total; + bool _showTotal; int _secondsBetween; int _checkInterval; diff --git a/src/mongo/util/version.cpp b/src/mongo/util/version.cpp index 25fe64b5811..a5622e4dbbb 100644 --- a/src/mongo/util/version.cpp +++ b/src/mongo/util/version.cpp @@ -47,7 +47,7 @@ namespace mongo { * 1.2.3-rc4-pre- * If you really need to do something else you'll need to fix _versionArray() */ - const char versionString[] = "2.4.6"; + const char versionString[] = "2.4.9"; // See unit test for example outputs BSONArray toVersionArray(const char* version){ diff --git a/src/third_party/v8/src/spaces.h b/src/third_party/v8/src/spaces.h index 6602c899dfb..d7a79c6f983 100644 --- a/src/third_party/v8/src/spaces.h +++ b/src/third_party/v8/src/spaces.h @@ -321,7 +321,8 @@ class MemoryChunk { Space* owner() const { if ((reinterpret_cast<intptr_t>(owner_) & kFailureTagMask) == kFailureTag) { - return reinterpret_cast<Space*>(owner_ - kFailureTag); + return reinterpret_cast<Space*>(reinterpret_cast<intptr_t>(owner_) - + kFailureTag); } else { return NULL; } |